LocalAI Setup: OpenAI Compatible Self-Hosted AI API

docker run -d \
  --name localai \
  -p 8080:8080 \
  -v localai_models:/build/models \
  localai/localai:latest-cpu

docker run -d \
  --name localai \
  --gpus all \
  -p 8080:8080 \
  -v localai_models:/build/models \
  localai/localai:latest-gpu-nvidia-cuda-12

version: '3.8'
services:
  localai:
    image: localai/localai:latest-gpu-nvidia-cuda-12
    container_name: localai
    ports:
      - "8080:8080"
    volumes:
      - localai_models:/build/models
      - ./config:/build/config
    environment:
      - THREADS=4
      - CONTEXT_SIZE=4096
      - GALLERIES=[{"name":"model-gallery","url":"github:mudler/LocalAI/gallery/index.yaml@master"}]
    deploy:
      resources:
        reservations:
          devices:
            - driver: nvidia
              count: all
              capabilities: [gpu]
    restart: unless-stopped
    healthcheck:
      test: ["CMD", "curl", "-f", "http://localhost:8080/readyz"]
      interval: 30s
      timeout: 10s
      retries: 3

volumes:
  localai_models:

# List available models
curl http://localhost:8080/models/available

# Load model (Llama 3)
curl http://localhost:8080/models/apply -d '{
  "id": "huggingface://TheBloke/Llama-2-7B-Chat-GGUF/llama-2-7b-chat.Q4_K_M.gguf",
  "name": "llama-3"
}'

# List loaded models
curl http://localhost:8080/v1/models

# Download GGUF model file
wget -O models/mistral-7b-instruct.gguf \
  "https://huggingface.co/TheBloke/Mistral-7B-Instruct-v0.2-GGUF/resolve/main/mistral-7b-instruct-v0.2.Q4_K_M.gguf"

# config/mistral.yaml
name: mistral
backend: llama-cpp
parameters:
  model: mistral-7b-instruct.gguf
  temperature: 0.7
  top_p: 0.9
  top_k: 40
  context_size: 4096
  threads: 4
  gpu_layers: 35
template:
  chat_message: |
    [INST] {{.Input}} [/INST]
  chat: |
    {{.Input}}

curl http://localhost:8080/v1/chat/completions \
  -H "Content-Type: application/json" \
  -d '{
    "model": "llama-3",
    "messages": [
      {"role": "system", "content": "You are a helpful assistant."},
      {"role": "user", "content": "What is Docker Compose?"}
    ],
    "temperature": 0.7,
    "max_tokens": 512
  }'

curl http://localhost:8080/v1/embeddings \
  -H "Content-Type: application/json" \
  -d '{
    "model": "text-embedding-ada-002",
    "input": "Server security best practices"
  }'

curl http://localhost:8080/v1/images/generations \
  -H "Content-Type: application/json" \
  -d '{
    "model": "stablediffusion",
    "prompt": "a futuristic server room, neon lights",
    "size": "512x512"
  }'

curl http://localhost:8080/v1/audio/transcriptions \
  -F "model=whisper-1" \
  -F "file=@audio.mp3"

curl http://localhost:8080/v1/audio/speech \
  -H "Content-Type: application/json" \
  -d '{
    "model": "tts-1",
    "input": "Hello, I am the LocalAI speech synthesis engine.",
    "voice": "alloy"
  }' --output speech.mp3

from openai import OpenAI

client = OpenAI(
    base_url="http://localhost:8080/v1",
    api_key="not-needed"
)

# Chat completion
response = client.chat.completions.create(
    model="llama-3",
    messages=[
        {"role": "system", "content": "You are a Linux expert."},
        {"role": "user", "content": "How to install Nginx?"}
    ],
    max_tokens=1024
)
print(response.choices[0].message.content)

# Embedding
embedding = client.embeddings.create(
    model="text-embedding-ada-002",
    input="Docker container management"
)
print(f"Embedding dimension: {len(embedding.data[0].embedding)}")

# config/models.yaml
- name: llama-3
  backend: llama-cpp
  parameters:
    model: llama-3-8b.gguf
    context_size: 4096
    gpu_layers: 35

- name: mistral
  backend: llama-cpp
  parameters:
    model: mistral-7b-instruct.gguf
    context_size: 4096
    gpu_layers: 35

- name: embedding
  backend: llama-cpp
  embeddings: true
  parameters:
    model: all-MiniLM-L6-v2.gguf

# Environment variables
export THREADS=8                    # CPU thread count
export CONTEXT_SIZE=4096            # Context window
export GPU_LAYERS=35                # Layers loaded to GPU
export PARALLEL_REQUESTS=true       # Parallel request support

# With Docker
docker run -d \
  --name localai \
  --gpus all \
  -p 8080:8080 \
  -e THREADS=8 \
  -e CONTEXT_SIZE=4096 \
  -v localai_models:/build/models \
  localai/localai:latest-gpu-nvidia-cuda-12