# Student model builder for Kaggle kernels (mirrors student/model.py reference prototype). # The production dual-model system is Rust; this is the Python/torch training definition. import torch import torch.nn as nn def estimate_params(cfg): v = cfg["vocab"]; d = cfg["d_model"]; L = cfg["n_layers"]; ffn = cfg["ffn_ratio"] * d p = v * d per = (4 * d * d) + (2 * d * ffn) + (2 * 2 * d) p += L * per p += 2 * d return p def build_model(cfg): d, L, ffn = cfg["d_model"], cfg["n_layers"], cfg["ffn_ratio"] * cfg["d_model"] h, v, maxlen = cfg["n_heads"], cfg["vocab"], cfg["max_seq_len"] class Attn(nn.Module): def __init__(self): super().__init__() self.qkv = nn.Linear(d, 3 * d, bias=False) self.o = nn.Linear(d, d, bias=False) def forward(self, x): B, T, _ = x.shape q, k, vv = self.qkv(x).chunk(3, -1) q = q.view(B, T, h, d // h).transpose(1, 2) k = k.view(B, T, h, d // h).transpose(1, 2) vv = vv.view(B, T, h, d // h).transpose(1, 2) out = torch.nn.functional.scaled_dot_product_attention(q, k, vv) out = out.transpose(1, 2).reshape(B, T, d) return self.o(out) class Block(nn.Module): def __init__(self): super().__init__() self.ln1 = nn.LayerNorm(d) self.attn = Attn() self.ln2 = nn.LayerNorm(d) self.mlp = nn.Sequential(nn.Linear(d, ffn), nn.GELU(), nn.Linear(ffn, d)) def forward(self, x): x = x + self.attn(self.ln1(x)) x = x + self.mlp(self.ln2(x)) return x class SAN(nn.Module): def __init__(self): super().__init__() self.tok_emb = nn.Embedding(v, d) self.pos = nn.Parameter(torch.zeros(1, maxlen, d)) self.blocks = nn.ModuleList([Block() for _ in range(L)]) self.ln_f = nn.LayerNorm(d) self.head = nn.Linear(d, v, bias=False) if cfg.get("tied_embeddings", True): self.head.weight = self.tok_emb.weight def forward(self, idx): B, T = idx.shape x = self.tok_emb(idx) + self.pos[:, :T] for b in self.blocks: x = b(x) return self.ln_f(x) return SAN()