Fine-tuned vs RAG:
баланс производительности

services:
  # Vector Database
  qdrant:
    image: qdrant/qdrant:v1.10.0
    ports: ["6333:6333", "6334:6334"]
    volumes:
      - qdrant_data:/qdrant/storage
      - ./qdrant-config.yaml:/qdrant/config/production.yaml:ro
    environment:
      QDRANT__SERVICE__API_KEY: ${QDRANT_API_KEY}
    restart: unless-stopped

  # Embedding model (BGE-M3, self-hosted)
  embedding-service:
    image: ghcr.io/huggingface/text-embeddings-inference:latest
    command: --model-id BAAI/bge-m3 --port 8080
    deploy:
      resources:
        reservations:
          devices:
          - driver: nvidia
            count: 1
            capabilities: [gpu]
    ports: ["8080:8080"]

  # Reranker (BGE-Reranker, self-hosted)
  reranker-service:
    image: ghcr.io/huggingface/text-embeddings-inference:latest
    command: --model-id BAAI/bge-reranker-v2-m3 --port 8081
    deploy:
      resources:
        reservations:
          devices:
          - driver: nvidia
            count: 1
            capabilities: [gpu]
    ports: ["8081:8081"]

  # RAG API layer (LangChain/LlamaIndex приложение)
  rag-api:
    build: ./rag-service
    environment:
      QDRANT_URL: http://qdrant:6333
      QDRANT_API_KEY: ${QDRANT_API_KEY}
      EMBEDDING_URL: http://embedding-service:8080
      RERANKER_URL: http://reranker-service:8081
      LLM_URL: http://vllm:8000   # ваш vLLM endpoint
    ports: ["9000:9000"]
    restart: unless-stopped

volumes:
  qdrant_data:

from langchain.text_splitter import RecursiveCharacterTextSplitter
from qdrant_client import QdrantClient
from qdrant_client.models import Distance, VectorParams, PointStruct
import httpx, uuid

# Конфигурация
EMBEDDING_URL = "http://embedding-service:8080/embed"
QDRANT_URL    = "http://qdrant:6333"
CHUNK_SIZE    = 512   # оптимальный по 2026 бенчмаркам
CHUNK_OVERLAP = 64    # ~12.5% overlap

client = QdrantClient(url=QDRANT_URL, api_key="...")

def create_collection(name: str, dim: int = 1024):
    # BGE-M3 даёт 1024-dim embeddings
    client.create_collection(name, vectors_config=VectorParams(
        size=dim, distance=Distance.COSINE
    ))

def ingest_document(text: str, metadata: dict, collection: str):
    # 1. Chunking
    splitter = RecursiveCharacterTextSplitter(
        chunk_size=CHUNK_SIZE,
        chunk_overlap=CHUNK_OVERLAP,
        separators=["\n\n", "\n", ". ", " "]
    )
    chunks = splitter.split_text(text)
    
    # 2. Добавить breadcrumb prefix к каждому чанку
    breadcrumb = f"{metadata.get('title','')} → {metadata.get('section','')}"
    chunks_with_ctx = [f"{breadcrumb}\n{c}" for c in chunks]
    
    # 3. Embedding (batch)
    resp = httpx.post(EMBEDDING_URL, json={"inputs": chunks_with_ctx})
    embeddings = resp.json()
    
    # 4. Upsert в Qdrant
    points = [
        PointStruct(
            id=str(uuid.uuid4()),
            vector=emb,
            payload={"text": chunk, **metadata}
        )
        for chunk, emb in zip(chunks, embeddings)
    ]
    client.upsert(collection_name=collection, points=points)
    return len(points)

# Hybrid search с reranking
def retrieve_and_rerank(query: str, collection: str, top_k: int = 5):
    # 1. Embed query
    q_emb = httpx.post(EMBEDDING_URL, json={"inputs": [query]}).json()[0]
    
    # 2. Vector search (top 20 candidates)
    results = client.search(collection, query_vector=q_emb, limit=20)
    candidates = [r.payload["text"] for r in results]
    
    # 3. Rerank с BGE-Reranker
    rerank_resp = httpx.post(
        "http://reranker-service:8081/rerank",
        json={"query": query, "texts": candidates}
    ).json()
    
    # 4. Вернуть top_k по rerank score
    ranked = sorted(
        zip(candidates, rerank_resp["scores"]),
        key=lambda x: x[1], reverse=True
    )
    return [text for text, _ in ranked[:top_k]]

base_model: meta-llama/Llama-3.1-8B-Instruct
model_type: LlamaForCausalLM
tokenizer_type: AutoTokenizer

# QLoRA config
load_in_4bit: true
adapter: lora
lora_r: 16          # rank — баланс качество/VRAM
lora_alpha: 16     # обычно = lora_r
lora_dropout: 0.05
lora_target_modules:  # все linear слои = лучшее качество
  - q_proj
  - k_proj
  - v_proj
  - o_proj
  - gate_proj
  - up_proj
  - down_proj

# Данные
datasets:
  - path: data/train.jsonl
    type: chat_template
val_set_size: 0.1
sequence_len: 2048

# Training
micro_batch_size: 2
gradient_accumulation_steps: 4
num_epochs: 3          # 1–3 epochs, не переобучить
learning_rate: 2e-4   # безопасный LR для LoRA
lr_scheduler: cosine
warmup_steps: 10

# Оптимизации
flash_attention: true
gradient_checkpointing: true
bf16: true

# Output
output_dir: ./checkpoints/llama-8b-domain
save_safetensors: true

# Запуск: axolotl train axolotl_config.yml

# Каждая строка — один пример (conversations format)
{"conversations": [
  {"role": "system", "content": "You are a financial analyst..."},
  {"role": "user",   "content": "Analyze this P&L statement..."},
  {"role": "assistant", "content": "Based on the P&L data: ..."}
]}

# Минимальные требования к качеству датасета:
# 1. Разнообразие примеров — не одинаковые формулировки
# 2. Consistent system prompt — одинаковый во всех примерах
# 3. Качество ответов — лучше 500 хороших, чем 5000 плохих
# 4. Negative examples — примеры правильного отказа от нерелевантных запросов
# 5. Validation split — минимум 10%, лучше 15%

# Слить адаптер с базовой моделью
python -c "
from peft import PeftModel
from transformers import AutoModelForCausalLM, AutoTokenizer

base = AutoModelForCausalLM.from_pretrained(
    'meta-llama/Llama-3.1-8B-Instruct', torch_dtype='bfloat16')
model = PeftModel.from_pretrained(base, './checkpoints/llama-8b-domain')
merged = model.merge_and_unload()
merged.save_pretrained('./models/llama-8b-domain-merged')
AutoTokenizer.from_pretrained(
    'meta-llama/Llama-3.1-8B-Instruct').save_pretrained(
    './models/llama-8b-domain-merged')
print('Done')
"

# Запуск через vLLM (merged model = обычная модель)
vllm serve ./models/llama-8b-domain-merged \
  --tensor-parallel-size 1 \
  --gpu-memory-utilization 0.90 \
  --enable-prefix-caching \
  --port 8001  # другой порт — рядом с базовой моделью

# Multi-LoRA вариант (base + несколько адаптеров)
vllm serve meta-llama/Llama-3.1-8B-Instruct \
  --enable-lora \
  --lora-modules domain-finance=./adapters/finance \
                 domain-legal=./adapters/legal \
  --max-loras 4 \
  --max-lora-rank 16

from ragas import evaluate
from ragas.metrics import (
    faithfulness, answer_relevancy,
    context_recall, context_precision
)
from datasets import Dataset

# Тестовый набор: вопросы + ожидаемые ответы + ground truth
eval_data = {
    "question":  ["Какова политика возврата?", ...],
    "answer":    ["Возврат в течение 30 дней...", ...],  # ответ RAG
    "contexts":  [["Документ о возврате..."], ...],          # retrieved чанки
    "ground_truth": ["Правильный ответ...", ...]
}

results = evaluate(
    Dataset.from_dict(eval_data),
    metrics=[faithfulness, answer_relevancy, context_recall, context_precision]
)
print(results)
# {faithfulness: 0.87, answer_relevancy: 0.82, ...}

# Минимум 50 вопросов для значимой оценки.
# Запускать после каждого изменения в pipeline.