Distributed
LLM Inference
Guide

# VRAM = weights + KV cache + activations + framework overhead
# Weights = params_B * bytes_per_param
# KV cache = 2 * num_layers * num_heads * head_dim * seq_len * batch * bytes

# Быстрый расчёт весов модели:
# FP16/BF16  → params_B * 2 GB
# INT8       → params_B * 1 GB
# INT4/AWQ   → params_B * 0.5 GB
# FP8        → params_B * 1 GB (лучше INT8 по качеству)

model_sizes = {
    "7B-FP16":  14,   # GB
    "7B-INT4":  4,    # GB (AWQ/GPTQ)
    "13B-FP16": 26,
    "13B-INT4": 7,
    "70B-FP16": 140,
    "70B-INT4": 35,   # влезает на 2×A100 40GB
    "405B-FP8": 405,  # требует 6-8 нод с A100 80GB
}

# Правило: добавить 30-50% к весам для KV cache при production нагрузке
# gpu_memory_utilization=0.90 — безопасный baseline для H100
# gpu_memory_utilization=0.95 — допустимо на B200/H200

Модель влезает на 1 GPU?
  └── Да → Data Parallelism (несколько реплик vLLM)
      Нет → Модель влезает на 1 ноду с N GPU?
              └── Да → Tensor Parallelism (TP=N)
                        └── GPU с NVLink? → Предпочесть TP
                        └── GPU без NVLink (L40S)? → Предпочесть PP
                  Нет → Multi-node: TP=GPU_per_node, PP=num_nodes
                        └── Для MoE моделей (Mixtral/DeepSeek):
                            TP + Expert Parallelism (EP)

# Золотое правило vLLM 2025+:
# TP размер должен быть степенью 2 и делить attention heads без остатка
# Total GPUs = tensor_parallel_size × pipeline_parallel_size

# Пример: 2 ноды по 4 GPU = 8 GPU total
vllm serve model \
  --tensor-parallel-size 4 \  # TP в пределах ноды
  --pipeline-parallel-size 2 \  # PP между нодами
  --distributed-executor-backend ray

# 1. Prerequisites
pip install vllm  # CUDA 12.1+ required

# 2. Запуск — 4 GPU, модель 70B INT4
vllm serve meta-llama/Llama-3.1-70B-Instruct \
  --tensor-parallel-size 4 \
  --quantization awq_marlin \
  --gpu-memory-utilization 0.90 \
  --max-model-len 32768 \
  --enable-prefix-caching \
  --enable-chunked-prefill \
  --max-num-seqs 256 \
  --port 8000 \
  --host 0.0.0.0 \
  --api-key $VLLM_API_KEY

# 3. Проверка
curl http://localhost:8000/v1/models \
  -H "Authorization: Bearer $VLLM_API_KEY"

# 4. Тест генерации
curl http://localhost:8000/v1/chat/completions \
  -H "Authorization: Bearer $VLLM_API_KEY" \
  -H "Content-Type: application/json" \
  -d '{"model":"meta-llama/Llama-3.1-70B-Instruct",
     "messages":[{"role":"user","content":"Hello"}],
     "max_tokens":100}'

# === НОДА 1 (head) — запустить первой ===
pip install vllm ray[default]

# Запуск Ray head
ray start --head \
  --port 6379 \
  --dashboard-host 0.0.0.0

# === НОДА 2 (worker) — подключиться к head ===
ray start \
  --address 'NODE1_IP:6379' \
  --block

# === НА HEAD НОДЕ: запуск vLLM ===
vllm serve meta-llama/Llama-3.1-405B-Instruct-FP8 \
  --tensor-parallel-size 4 \  # GPU на ноду
  --pipeline-parallel-size 2 \  # число нод
  --distributed-executor-backend ray \
  --gpu-memory-utilization 0.90 \
  --enable-prefix-caching \
  --port 8000

# Проверить Ray cluster:
ray status

services:
  vllm:
    image: vllm/vllm-openai:v0.8.5  # Пинить конкретный тег!
    runtime: nvidia
    environment:
      NVIDIA_VISIBLE_DEVICES: all
      HUGGING_FACE_HUB_TOKEN: ${HF_TOKEN}
      VLLM_API_KEY: ${VLLM_API_KEY}
    volumes:
      - /mnt/models:/models:ro  # NFS или local path
      - vllm-cache:/root/.cache
    command: >
      --model /models/llama-3.1-70b-instruct-awq
      --tensor-parallel-size 4
      --quantization awq_marlin
      --gpu-memory-utilization 0.90
      --max-model-len 32768
      --enable-prefix-caching
      --served-model-name llama-70b
    ports:
      - "8000:8000"
    healthcheck:
      test: ["CMD","curl","-f","http://localhost:8000/health"]
      interval: 30s
      timeout: 10s
      start_period: 120s  # Модели загружаются долго
      retries: 3
    restart: unless-stopped
    deploy:
      resources:
        reservations:
          devices:
          - driver: nvidia
            count: all
            capabilities: [gpu]

volumes:
  vllm-cache:

# Установка NVIDIA GPU Operator (обязателен для GPU scheduling)
helm repo add nvidia https://helm.ngc.nvidia.com/nvidia
helm repo update

helm install gpu-operator nvidia/gpu-operator \
  --namespace gpu-operator \
  --create-namespace \
  --set driver.enabled=false \  # Если драйвер уже установлен
  --set toolkit.enabled=true

# Проверить GPU resource
kubectl get nodes -o json | jq '.items[].status.capacity | select(."nvidia.com/gpu")'

apiVersion: apps/v1
kind: Deployment
metadata:
  name: vllm-llama70b
  namespace: inference
spec:
  replicas: 2
  selector:
    matchLabels:
      app: vllm-llama70b
  template:
    metadata:
      labels:
        app: vllm-llama70b
        model: llama-3.1-70b
    spec:
      nodeSelector:
        nvidia.com/gpu.product: A100-SXM4-80GB
      topologySpreadConstraints:
      - maxSkew: 1
        topologyKey: "kubernetes.io/hostname"
        whenUnsatisfiable: DoNotSchedule
      containers:
      - name: vllm
        image: vllm/vllm-openai:v0.8.5
        args:
        - --model=/models/llama-3.1-70b-awq
        - --tensor-parallel-size=4
        - --quantization=awq_marlin
        - --gpu-memory-utilization=0.90
        - --max-model-len=32768
        - --enable-prefix-caching
        - --port=8000
        env:
        - name: VLLM_API_KEY
          valueFrom:
            secretKeyRef:
              name: vllm-secrets
              key: api-key
        resources:
          requests:
            nvidia.com/gpu: "4"
            memory: 64Gi
            cpu: "8"
          limits:
            nvidia.com/gpu: "4"
            memory: 128Gi
            cpu: "16"
        volumeMounts:
        - name: model-storage
          mountPath: /models
          readOnly: true
        readinessProbe:
          httpGet: {path: /health, port: 8000}
          initialDelaySeconds: 120
          periodSeconds: 10
        livenessProbe:
          httpGet: {path: /health, port: 8000}
          initialDelaySeconds: 180
          periodSeconds: 30
          failureThreshold: 3
      volumes:
      - name: model-storage
        persistentVolumeClaim:
          claimName: model-pvc

apiVersion: keda.sh/v1alpha1
kind: ScaledObject
metadata:
  name: vllm-scaler
  namespace: inference
spec:
  scaleTargetRef:
    name: vllm-llama70b
  minReplicaCount: 1
  maxReplicaCount: 4
  triggers:
  - type: prometheus
    metadata:
      serverAddress: http://prometheus:9090
      # Scale по queue depth (in-flight requests)
      query: avg(vllm:num_requests_waiting)
      threshold: "10"  # >10 waiting → добавить реплику
  - type: prometheus
    metadata:
      serverAddress: http://prometheus:9090
      # Также scale по KV cache utilization
      query: avg(vllm:gpu_cache_usage_perc) * 100
      threshold: "80"  # >80% KV cache → scale up

# ray up ray_hybrid_cluster.yaml
cluster_name: hybrid-inference

provider:
  type: local  # Head на on-prem ВМ
  head_ip: 192.168.1.10

head_node_type:
  node_config: {}
  resources:
    CPU: 32
    GPU: 4

available_node_types:
  # On-prem ВМ workers
  onprem_gpu_worker:
    min_workers: 1
    max_workers: 3
    resources:
      CPU: 32
      GPU: 4
    node_config:
      NodeType: onprem

  # Cloud burst workers (spot)
  cloud_gpu_worker:
    min_workers: 0  # Scale to zero когда не нужно
    max_workers: 5
    resources:
      CPU: 48
      GPU: 8
    node_config:
      InstanceType: p4d.24xlarge  # 8× A100 40GB
      SpotInstances: true

setup_commands:
  - pip install vllm ray[default]
  - pip install ray[serve]

import ray
from ray import serve
from vllm.engine.arg_utils import AsyncEngineArgs
from vllm.engine.async_llm_engine import AsyncLLMEngine

ray.init("ray://head-node-ip:10001")

@serve.deployment(
    num_replicas=2,
    ray_actor_options={
        "num_gpus": 4,
        "resources": {"NodeType": 0.001}  # Предпочесть on-prem
    }
)
class LLMDeployment:
    def __init__(self, model_id: str, tp_size: int):
        args = AsyncEngineArgs(
            model=model_id,
            tensor_parallel_size=tp_size,
            gpu_memory_utilization=0.90,
            enable_prefix_caching=True,
            quantization="awq_marlin",
        )
        self.engine = AsyncLLMEngine.from_engine_args(args)

    async def generate(self, prompt: str, params: dict):
        # OpenAI-compatible generation
        results = []
        async for output in self.engine.generate(prompt, ...):
            results.append(output)
        return results

# Роутер: направлять запросы по model_name
@serve.deployment
class ModelRouter:
    def __init__(self):
        self.models = {
            "llama-70b":  LLMDeployment.bind("llama-70b-awq", 4),
            "llama-8b":   LLMDeployment.bind("llama-8b", 1),
        }

    async def __call__(self, request):
        model_name = request.json()["model"]
        return await self.models[model_name].remote(request)

serve.run(ModelRouter.bind())

scrape_configs:
- job_name: vllm
  scrape_interval: 15s
  static_configs:
  - targets: ['vllm-service:8000']  # vLLM metrics on /metrics

# Ключевые PromQL запросы для дашборда:

# TTFT P99
histogram_quantile(0.99,
  rate(vllm:time_to_first_token_seconds_bucket[5m]))

# Requests waiting (autoscaling trigger)
avg(vllm:num_requests_waiting)

# KV cache utilization
avg(vllm:gpu_cache_usage_perc) * 100

# Throughput (tokens/sec)
sum(rate(vllm:generation_tokens_total[1m]))