GPU Sunucuda AI Model Deployment: NVIDIA CUDA Rehberi

# Mevcut NVIDIA sürücülerini kaldır
sudo apt-get purge nvidia-* -y
sudo apt-get autoremove -y

# Sistem güncelleme
sudo apt-get update && sudo apt-get upgrade -y

# NVIDIA sürücü deposu ekleme
sudo apt-get install -y linux-headers-$(uname -r)
sudo apt-get install -y software-properties-common
sudo add-apt-repository ppa:graphics-drivers/ppa -y
sudo apt-get update

# Önerilen sürücüyü kontrol et
ubuntu-drivers devices

# NVIDIA sürücü kurulumu (önerilen versiyon)
sudo apt-get install -y nvidia-driver-550
sudo reboot

# Sürücü versiyonu ve GPU bilgisi
nvidia-smi

+-----------------------------------------------------------------------------+
| NVIDIA-SMI 550.54.15    Driver Version: 550.54.15    CUDA Version: 12.4    |
|-------------------------------+----------------------+----------------------+
| GPU  Name        Persistence-M| Bus-Id        Disp.A | Volatile Uncorr. ECC |
| Fan  Temp  Perf  Pwr:Usage/Cap|         Memory-Usage | GPU-Util  Compute M.  |
|===============================+======================+======================|
|   0  NVIDIA A100-SXM4-80GB On| 00000000:00:04.0 Off |                    0 |
| N/A   32C    P0    45W / 400W|      0MiB / 81920MiB |      0%      Default |
+-------------------------------+----------------------+----------------------+

# CUDA keyring ekleme
wget https://developer.download.nvidia.com/compute/cuda/repos/ubuntu2204/x86_64/cuda-keyring_1.1-1_all.deb
sudo dpkg -i cuda-keyring_1.1-1_all.deb
sudo apt-get update

# CUDA Toolkit kurulumu
sudo apt-get install -y cuda-toolkit-12-4

# PATH ayarları
echo 'export PATH=/usr/local/cuda-12.4/bin:$PATH' >> ~/.bashrc
echo 'export LD_LIBRARY_PATH=/usr/local/cuda-12.4/lib64:$LD_LIBRARY_PATH' >> ~/.bashrc
source ~/.bashrc

# Doğrulama
nvcc --version

# cuDNN kurulumu (CUDA 12.x için)
sudo apt-get install -y libcudnn8 libcudnn8-dev

# Doğrulama
cat /usr/include/cudnn_version.h | grep CUDNN_MAJOR -A 2

# NVIDIA Container Toolkit deposu
curl -fsSL https://nvidia.github.io/libnvidia-container/gpgkey | \
  sudo gpg --dearmor -o /usr/share/keyrings/nvidia-container-toolkit-keyring.gpg

curl -s -L https://nvidia.github.io/libnvidia-container/stable/deb/nvidia-container-toolkit.list | \
  sed 's#deb https://#deb [signed-by=/usr/share/keyrings/nvidia-container-toolkit-keyring.gpg] https://#g' | \
  sudo tee /etc/apt/sources.list.d/nvidia-container-toolkit.list

sudo apt-get update

# Kurulum
sudo apt-get install -y nvidia-container-toolkit

# Docker runtime yapılandırması
sudo nvidia-ctk runtime configure --runtime=docker
sudo systemctl restart docker

# Test
sudo docker run --rm --gpus all nvidia/cuda:12.4.0-base-ubuntu22.04 nvidia-smi

docker run -d \
  --name vllm \
  --gpus all \
  -p 8000:8000 \
  -v ~/.cache/huggingface:/root/.cache/huggingface \
  --env HUGGING_FACE_HUB_TOKEN=$HF_TOKEN \
  vllm/vllm-openai:latest \
  --model meta-llama/Meta-Llama-3.1-8B-Instruct \
  --max-model-len 4096 \
  --gpu-memory-utilization 0.9

version: '3.8'
services:
  vllm:
    image: vllm/vllm-openai:latest
    container_name: vllm-server
    ports:
      - "8000:8000"
    volumes:
      - huggingface_cache:/root/.cache/huggingface
    environment:
      - HUGGING_FACE_HUB_TOKEN=${HF_TOKEN}
    command: >
      --model meta-llama/Meta-Llama-3.1-8B-Instruct
      --max-model-len 4096
      --gpu-memory-utilization 0.9
      --tensor-parallel-size 1
      --dtype auto
    deploy:
      resources:
        reservations:
          devices:
            - driver: nvidia
              count: all
              capabilities: [gpu]
    restart: unless-stopped
    healthcheck:
      test: ["CMD", "curl", "-f", "http://localhost:8000/health"]
      interval: 30s
      timeout: 10s
      retries: 3

volumes:
  huggingface_cache:

# Chat completion
curl http://localhost:8000/v1/chat/completions \
  -H "Content-Type: application/json" \
  -d '{
    "model": "meta-llama/Meta-Llama-3.1-8B-Instruct",
    "messages": [
      {"role": "system", "content": "Sen yardımcı bir asistansın."},
      {"role": "user", "content": "Docker nedir?"}
    ],
    "max_tokens": 512,
    "temperature": 0.7
  }'

# Model listesi
curl http://localhost:8000/v1/models

from openai import OpenAI

client = OpenAI(
    base_url="http://localhost:8000/v1",
    api_key="not-needed"
)

response = client.chat.completions.create(
    model="meta-llama/Meta-Llama-3.1-8B-Instruct",
    messages=[
        {"role": "system", "content": "Sen bir Linux uzmanısın."},
        {"role": "user", "content": "RAID yapılandırması nasıl yapılır?"}
    ],
    max_tokens=1024,
    temperature=0.7
)

print(response.choices[0].message.content)

# Tensor parallelism ile çoklu GPU
docker run -d \
  --name vllm-multi \
  --gpus all \
  -p 8000:8000 \
  -v ~/.cache/huggingface:/root/.cache/huggingface \
  vllm/vllm-openai:latest \
  --model meta-llama/Meta-Llama-3.1-70B-Instruct \
  --tensor-parallel-size 4 \
  --max-model-len 4096 \
  --gpu-memory-utilization 0.9

# Anlık durum
nvidia-smi

# Sürekli izleme (her 1 saniye)
watch -n 1 nvidia-smi

# Sadece GPU kullanımı ve bellek
nvidia-smi --query-gpu=index,name,utilization.gpu,memory.used,memory.total,temperature.gpu --format=csv -l 1

# docker-compose.monitoring.yml
version: '3.8'
services:
  dcgm-exporter:
    image: nvcr.io/nvidia/k8s/dcgm-exporter:3.3.5-3.4.0-ubuntu22.04
    container_name: dcgm-exporter
    deploy:
      resources:
        reservations:
          devices:
            - driver: nvidia
              count: all
              capabilities: [gpu]
    ports:
      - "9400:9400"
    restart: unless-stopped

  prometheus:
    image: prom/prometheus:latest
    container_name: prometheus
    ports:
      - "9090:9090"
    volumes:
      - ./prometheus.yml:/etc/prometheus/prometheus.yml
    restart: unless-stopped

  grafana:
    image: grafana/grafana:latest
    container_name: grafana
    ports:
      - "3000:3000"
    environment:
      - GF_SECURITY_ADMIN_PASSWORD=admin
    restart: unless-stopped

# prometheus.yml
global:
  scrape_interval: 15s

scrape_configs:
  - job_name: 'dcgm'
    static_configs:
      - targets: ['dcgm-exporter:9400']

# Persistence mode aktifleştir (düşük latency)
sudo nvidia-smi -pm 1

# GPU clock hızını sabitle
sudo nvidia-smi -lgc 1410,1410

# Power limit ayarla
sudo nvidia-smi -pl 300

server {
    listen 443 ssl;
    server_name ai-api.example.com;

    ssl_certificate /etc/letsencrypt/live/ai-api.example.com/fullchain.pem;
    ssl_certificate_key /etc/letsencrypt/live/ai-api.example.com/privkey.pem;

    location / {
        proxy_pass http://127.0.0.1:8000;
        proxy_set_header Host $host;
        proxy_set_header X-Real-IP $remote_addr;
        proxy_read_timeout 300s;
        proxy_send_timeout 300s;
    }
}