ч.2 разбор блоков
как устроен трансформера и BERT

def make_causal_mask(seq_len: int, device) -> torch.Tensor:
    """
    Возвращает булеву маску (seq_len, seq_len):
    True  → заблокировать (будущие позиции)
    False → разрешить
    """
    # torch.triu: верхний треугольник, diagonal=1 → исключаем диагональ
    mask = torch.triu(torch.ones(seq_len, seq_len, device=device), diagonal=1).bool()
    return mask  # (T, T)


# Пример: последовательность длиной 4
mask = make_causal_mask(4, 'cpu')
# tensor([[False,  True,  True,  True],
#         [False, False,  True,  True],
#         [False, False, False,  True],
#         [False, False, False, False]])

def sample_next_token(logits: torch.Tensor, temperature: float = 1.0, top_k: int = 50) -> int:
    """logits: (vocab_size,)"""
    logits = logits / max(temperature, 1e-8)           # масштабируем

    # Top-k фильтрация: оставляем только top_k токенов
    values, _ = torch.topk(logits, top_k)
    threshold = values[-1]
    logits = logits.masked_fill(logits < threshold, float('-inf'))

    probs = F.softmax(logits, dim=-1)
    next_token = torch.multinomial(probs, num_samples=1).item()
    return next_token

from transformers import pipeline

# Загружает: distilbert-base-uncased-finetuned-sst-2-english
# (или любой другой text-classification checkpoint)
classifier = pipeline(
    "sentiment-analysis",
    model="distilbert/distilbert-base-uncased-finetuned-sst-2-english",
    device=0,         # GPU; -1 для CPU
    truncation=True,
)

texts = [
    "Transformers make NLP incredibly accessible!",
    "The model was slow and the output was poor.",
]

results = classifier(texts, batch_size=8)

for text, r in zip(texts, results):
    print(f"{text[:40]}... → {r['label']} ({r['score']:.3f})")

# → Transformers make NLP... → POSITIVE (0.9998)
# → The model was slow...   → NEGATIVE (0.9996)

## Что происходит под капотом TextClassificationPipeline:
# 1. tokenizer(texts, ...) → input_ids, attention_mask
# 2. model(**inputs)       → BaseModelOutput(logits=...)
# 3. softmax(logits)       → вероятности по классам
# 4. argmax → id2label     → "POSITIVE" / "NEGATIVE"

from transformers import AutoTokenizer, AutoModelForCausalLM
import torch

model_id = "openai-community/gpt2"
tokenizer = AutoTokenizer.from_pretrained(model_id)
model     = AutoModelForCausalLM.from_pretrained(model_id)
model.eval()

prompt = "The transformer architecture revolutionized NLP because"
inputs = tokenizer(prompt, return_tensors="pt")

with torch.no_grad():
    output_ids = model.generate(
        **inputs,
        max_new_tokens=80,
        do_sample=True,
        temperature=0.8,
        top_p=0.9,           # nucleus sampling
        repetition_penalty=1.1,  # штраф за повторы
        pad_token_id=tokenizer.eos_token_id,
    )

generated = tokenizer.decode(output_ids[0], skip_special_tokens=True)
print(generated)

## Автоматически выполняется при каждом шаге generate():
# 1. Encoder-кэш KV не пересчитывается (KV-cache)
# 2. Causal mask применяется автоматически
# 3. Top-p / top-k / temperature применяются к logits
# 4. EOS-токен останавливает генерацию

from transformers import AutoTokenizer, AutoModel
import torch

tokenizer = AutoTokenizer.from_pretrained("bert-base-uncased")
model     = AutoModel.from_pretrained("bert-base-uncased")

text = "Hello, transformer world!"
enc  = tokenizer(text, return_tensors="pt")

# enc содержит: input_ids, attention_mask, token_type_ids
print(enc.input_ids)       # tensor([[101, 7592, 1010, 19081, 2088, 999, 102]])
print(tokenizer.convert_ids_to_tokens(enc.input_ids[0].tolist()))
# ['[CLS]', 'hello', ',', 'transform', '##er', 'world', '!', '[SEP]']

with torch.no_grad():
    outputs = model(**enc)

# last_hidden_state: (batch, seq_len, hidden_size) = (1, 8, 768)
# pooler_output:     (batch, hidden_size) — [CLS]-токен через linear+tanh

cls_emb = outputs.last_hidden_state[:, 0, :]   # [CLS] → sentence embedding
print(cls_emb.shape)  # torch.Size([1, 768])

import torch
import torch.nn as nn
from torch.utils.data import DataLoader, TensorDataset

# ── 1. Модель ──────────────────────────────────────────────
class LSTMClassifier(nn.Module):
    def __init__(self, vocab_size, embed_dim=128, hidden=256, num_layers=2, num_classes=2):
        super().__init__()
        self.embedding = nn.Embedding(vocab_size, embed_dim, padding_idx=0)
        self.lstm = nn.LSTM(
            embed_dim, hidden,
            num_layers=num_layers,
            batch_first=True,    # (batch, seq, features)
            dropout=0.3,
            bidirectional=True,  # BiLSTM: читает в обе стороны
        )
        self.fc = nn.Linear(hidden * 2, num_classes)  # *2 за bidirectional
        self.dropout = nn.Dropout(0.3)

    def forward(self, x):
        emb = self.dropout(self.embedding(x))           # (B, T, E)
        out, (h_n, _) = self.lstm(emb)                 # out: (B, T, 2H)
        # Берём последний скрытый слой обоих направлений
        h_fwd = h_n[-2]  # последний forward-слой
        h_bwd = h_n[-1]  # последний backward-слой
        h = torch.cat([h_fwd, h_bwd], dim=-1)           # (B, 2H)
        return self.fc(self.dropout(h))


# ── 2. Данные (заглушка) ───────────────────────────────────
VOCAB, SEQ, N = 10000, 128, 1000
X = torch.randint(1, VOCAB, (N, SEQ))
y = torch.randint(0, 2, (N,))
loader = DataLoader(TensorDataset(X, y), batch_size=64, shuffle=True)


# ── 3. Инициализация ───────────────────────────────────────
device = torch.device("cuda" if torch.cuda.is_available() else "cpu")
model  = LSTMClassifier(VOCAB).to(device)
optim  = torch.optim.AdamW(model.parameters(), lr=3e-4, weight_decay=1e-2)
sched  = torch.optim.lr_scheduler.CosineAnnealingLR(optim, T_max=5)
crit   = nn.CrossEntropyLoss()


# ── 4. Train-loop ──────────────────────────────────────────
for epoch in range(5):
    model.train()
    total_loss = 0.0

    for xb, yb in loader:
        xb, yb = xb.to(device), yb.to(device)

        optim.zero_grad()           # обнуляем накопленные градиенты
        logits = model(xb)          # forward pass
        loss   = crit(logits, yb)   # CrossEntropy = LogSoftmax + NLLLoss
        loss.backward()             # вычисляем градиенты (autograd)

        nn.utils.clip_grad_norm_(model.parameters(), max_norm=1.0)  # gradient clipping
        optim.step()               # обновляем веса
        total_loss += loss.item()

    sched.step()
    avg = total_loss / len(loader)
    print(f"Epoch {epoch+1:02d} | loss={avg:.4f} | lr={sched.get_last_lr()[0]:.2e}")


# ── 5. Инференс ────────────────────────────────────────────
model.eval()
with torch.no_grad():
    test_x = torch.randint(1, VOCAB, (4, SEQ)).to(device)
    preds  = model(test_x).argmax(dim=-1)
    print("Predicted classes:", preds.tolist())