import math import torch import torch.nn as nn import torch.nn.functional as F class CausalSelfAttention(nn.Module): def __init__(self, cfg): super().__init__() self.n_heads = cfg.n_heads self.d_head = cfg.d_model // cfg.n_heads self.qkv = nn.Linear(cfg.d_model, 3 * cfg.d_model, bias=False) self.proj = nn.Linear(cfg.d_model, cfg.d_model, bias=False) self.dropout = nn.Dropout(cfg.dropout) def forward(self, x): B, T, C = x.shape q, k, v = self.qkv(x).split(C, dim=2) q = q.view(B, T, self.n_heads, self.d_head).transpose(1, 2) k = k.view(B, T, self.n_heads, self.d_head).transpose(1, 2) v = v.view(B, T, self.n_heads, self.d_head).transpose(1, 2) y = F.scaled_dot_product_attention(q, k, v, is_causal=True, dropout_p=self.dropout.p if self.training else 0.0) y = y.transpose(1, 2).contiguous().view(B, T, C) return self.dropout(self.proj(y)) class FFN(nn.Module): def __init__(self, cfg): super().__init__() self.fc1 = nn.Linear(cfg.d_model, 4 * cfg.d_model) self.fc2 = nn.Linear(4 * cfg.d_model, cfg.d_model) self.dropout = nn.Dropout(cfg.dropout) def forward(self, x): return self.dropout(self.fc2(F.gelu(self.fc1(x)))) class Block(nn.Module): def __init__(self, cfg): super().__init__() self.ln1 = nn.LayerNorm(cfg.d_model) self.attn = CausalSelfAttention(cfg) self.ln2 = nn.LayerNorm(cfg.d_model) self.ffn = FFN(cfg) def forward(self, x): x = x + self.attn(self.ln1(x)) x = x + self.ffn(self.ln2(x)) return x class Amaia(nn.Module): def __init__(self, cfg): super().__init__() self.cfg = cfg self.tok_emb = nn.Embedding(cfg.vocab_size, cfg.d_model) self.pos_emb = nn.Embedding(cfg.max_seq, cfg.d_model) self.drop = nn.Dropout(cfg.dropout) self.blocks = nn.Sequential(*[Block(cfg) for _ in range(cfg.n_layers)]) self.ln_f = nn.LayerNorm(cfg.d_model) self.head = nn.Linear(cfg.d_model, cfg.vocab_size, bias=False) self.tok_emb.weight = self.head.weight self.apply(self._init_weights) def _init_weights(self, m): if isinstance(m, nn.Linear): nn.init.normal_(m.weight, std=0.02) if m.bias is not None: nn.init.zeros_(m.bias) elif isinstance(m, nn.Embedding): nn.init.normal_(m.weight, std=0.02) def forward(self, idx, targets=None): B, T = idx.shape pos = torch.arange(T, device=idx.device) x = self.drop(self.tok_emb(idx) + self.pos_emb(pos)) x = self.blocks(x) x = self.ln_f(x) logits = self.head(x) loss = None if targets is not None: loss = F.cross_entropy(logits.view(-1, logits.size(-1)), targets.view(-1)) return logits, loss def n_params(self): return sum(p.numel() for p in self.parameters()) @torch.no_grad() def generate(self, idx, max_new_tokens, temperature=0.8, top_p=0.92): for _ in range(max_new_tokens): idx_cond = idx[:, -self.cfg.max_seq:] logits, _ = self(idx_cond) logits = logits[:, -1, :] / temperature probs = F.softmax(logits, dim=-1) sorted_probs, sorted_idx = torch.sort(probs, descending=True) cumsum = torch.cumsum(sorted_probs, dim=-1) mask = cumsum - sorted_probs > top_p sorted_probs[mask] = 0.0 sorted_probs /= sorted_probs.sum() next_token = sorted_idx.gather(-1, torch.multinomial(sorted_probs, 1)) idx = torch.cat([idx, next_token], dim=1) return idx