| """ |
| Parametrisiertes GPT — Modelldefinition für pretrain.py, finetune_chat.py, |
| chat.py und sample_all.py (v3, eigener BPE-Tokenizer mit 8192 Vokabeln). |
| """ |
|
|
| import torch |
| import torch.nn as nn |
| from torch.nn import functional as F |
|
|
|
|
| class Head(nn.Module): |
| """ one head of self-attention """ |
|
|
| def __init__(self, n_embd, head_size, block_size, dropout): |
| super().__init__() |
| self.key = nn.Linear(n_embd, head_size, bias=False) |
| self.query = nn.Linear(n_embd, head_size, bias=False) |
| self.value = nn.Linear(n_embd, head_size, bias=False) |
| self.register_buffer('tril', torch.tril(torch.ones(block_size, block_size))) |
| self.dropout = nn.Dropout(dropout) |
|
|
| def forward(self, x): |
| B, T, C = x.shape |
| k = self.key(x) |
| q = self.query(x) |
| wei = q @ k.transpose(-2, -1) * k.shape[-1] ** -0.5 |
| wei = wei.masked_fill(self.tril[:T, :T] == 0, float('-inf')) |
| wei = F.softmax(wei, dim=-1) |
| wei = self.dropout(wei) |
| v = self.value(x) |
| return wei @ v |
|
|
|
|
| class MultiHeadAttention(nn.Module): |
| def __init__(self, n_embd, n_head, block_size, dropout): |
| super().__init__() |
| head_size = n_embd // n_head |
| self.heads = nn.ModuleList( |
| [Head(n_embd, head_size, block_size, dropout) for _ in range(n_head)]) |
| self.proj = nn.Linear(n_embd, n_embd) |
| self.dropout = nn.Dropout(dropout) |
|
|
| def forward(self, x): |
| out = torch.cat([h(x) for h in self.heads], dim=-1) |
| return self.dropout(self.proj(out)) |
|
|
|
|
| class FeedForward(nn.Module): |
| def __init__(self, n_embd, dropout): |
| super().__init__() |
| self.net = nn.Sequential( |
| nn.Linear(n_embd, 4 * n_embd), |
| nn.ReLU(), |
| nn.Linear(4 * n_embd, n_embd), |
| nn.Dropout(dropout), |
| ) |
|
|
| def forward(self, x): |
| return self.net(x) |
|
|
|
|
| class Block(nn.Module): |
| def __init__(self, n_embd, n_head, block_size, dropout): |
| super().__init__() |
| self.sa = MultiHeadAttention(n_embd, n_head, block_size, dropout) |
| self.ffwd = FeedForward(n_embd, dropout) |
| self.ln1 = nn.LayerNorm(n_embd) |
| self.ln2 = nn.LayerNorm(n_embd) |
|
|
| def forward(self, x): |
| x = x + self.sa(self.ln1(x)) |
| x = x + self.ffwd(self.ln2(x)) |
| return x |
|
|
|
|
| class GPTLanguageModel(nn.Module): |
|
|
| def __init__(self, vocab_size, n_embd, n_head, n_layer, block_size, dropout): |
| super().__init__() |
| self.block_size = block_size |
| self.token_embedding_table = nn.Embedding(vocab_size, n_embd) |
| self.position_embedding_table = nn.Embedding(block_size, n_embd) |
| self.blocks = nn.Sequential( |
| *[Block(n_embd, n_head, block_size, dropout) for _ in range(n_layer)]) |
| self.ln_f = nn.LayerNorm(n_embd) |
| self.lm_head = nn.Linear(n_embd, vocab_size, bias=False) |
| self.lm_head.weight = self.token_embedding_table.weight |
|
|
| self.apply(self._init_weights) |
|
|
| def _init_weights(self, module): |
| if isinstance(module, nn.Linear): |
| torch.nn.init.normal_(module.weight, mean=0.0, std=0.02) |
| if module.bias is not None: |
| torch.nn.init.zeros_(module.bias) |
| elif isinstance(module, nn.Embedding): |
| torch.nn.init.normal_(module.weight, mean=0.0, std=0.02) |
|
|
| def forward(self, idx, targets=None): |
| B, T = idx.shape |
| tok_emb = self.token_embedding_table(idx) |
| pos_emb = self.position_embedding_table(torch.arange(T, device=idx.device)) |
| x = tok_emb + pos_emb |
| x = self.blocks(x) |
| x = self.ln_f(x) |
| logits = self.lm_head(x) |
|
|
| if targets is None: |
| loss = None |
| else: |
| B, T, C = logits.shape |
| loss = F.cross_entropy(logits.view(B * T, C), targets.view(B * T)) |
| return logits, loss |
|
|
| @torch.no_grad() |
| def generate(self, idx, max_new_tokens, temperature=1.0, top_k=None, |
| stop_tokens=None, repetition_penalty=1.0): |
| """Sampelt Tokens; bricht ab, sobald ein stop_token erzeugt wurde |
| (das Stop-Token selbst wird nicht angehängt).""" |
| stop_tokens = set(stop_tokens or []) |
| for _ in range(max_new_tokens): |
| idx_cond = idx[:, -self.block_size:] |
| logits, _ = self(idx_cond) |
| logits = logits[:, -1, :] / max(temperature, 1e-5) |
| if repetition_penalty != 1.0: |
| recent = idx[0, -64:].tolist() |
| for t in set(recent): |
| logits[0, t] /= repetition_penalty |
| if top_k is not None: |
| v, _ = torch.topk(logits, min(top_k, logits.size(-1))) |
| logits[logits < v[:, [-1]]] = float('-inf') |
| probs = F.softmax(logits, dim=-1) |
| idx_next = torch.multinomial(probs, num_samples=1) |
| if idx_next.item() in stop_tokens: |
| break |
| idx = torch.cat((idx, idx_next), dim=1) |
| return idx |
|
|
|
|
| def get_device() -> str: |
| if torch.backends.mps.is_available(): |
| return 'mps' |
| if torch.cuda.is_available(): |
| return 'cuda' |
| return 'cpu' |
|
|
|
|
| def model_from_checkpoint(ckpt, device): |
| cfg = ckpt['config'] |
| model = GPTLanguageModel( |
| vocab_size=ckpt['vocab_size'], |
| n_embd=cfg['n_embd'], n_head=cfg['n_head'], n_layer=cfg['n_layer'], |
| block_size=cfg['block_size'], dropout=cfg['dropout']) |
| model.load_state_dict(ckpt['model_state_dict']) |
| return model.to(device) |
|
|
