alter_ego / training /kernels /common /model_builder.py
geraldaton20's picture
initial upload: source code (no build artifacts)
f4f166e verified
Raw History Blame Contribute Delete
2.35 kB
# Student model builder for Kaggle kernels (mirrors student/model.py reference prototype).
# The production dual-model system is Rust; this is the Python/torch training definition.
import torch
import torch.nn as nn
def estimate_params(cfg):
v = cfg["vocab"]; d = cfg["d_model"]; L = cfg["n_layers"]; ffn = cfg["ffn_ratio"] * d
p = v * d
per = (4 * d * d) + (2 * d * ffn) + (2 * 2 * d)
p += L * per
p += 2 * d
return p
def build_model(cfg):
d, L, ffn = cfg["d_model"], cfg["n_layers"], cfg["ffn_ratio"] * cfg["d_model"]
h, v, maxlen = cfg["n_heads"], cfg["vocab"], cfg["max_seq_len"]
class Attn(nn.Module):
def __init__(self):
super().__init__()
self.qkv = nn.Linear(d, 3 * d, bias=False)
self.o = nn.Linear(d, d, bias=False)
def forward(self, x):
B, T, _ = x.shape
q, k, vv = self.qkv(x).chunk(3, -1)
q = q.view(B, T, h, d // h).transpose(1, 2)
k = k.view(B, T, h, d // h).transpose(1, 2)
vv = vv.view(B, T, h, d // h).transpose(1, 2)
out = torch.nn.functional.scaled_dot_product_attention(q, k, vv)
out = out.transpose(1, 2).reshape(B, T, d)
return self.o(out)
class Block(nn.Module):
def __init__(self):
super().__init__()
self.ln1 = nn.LayerNorm(d)
self.attn = Attn()
self.ln2 = nn.LayerNorm(d)
self.mlp = nn.Sequential(nn.Linear(d, ffn), nn.GELU(), nn.Linear(ffn, d))
def forward(self, x):
x = x + self.attn(self.ln1(x))
x = x + self.mlp(self.ln2(x))
return x
class SAN(nn.Module):
def __init__(self):
super().__init__()
self.tok_emb = nn.Embedding(v, d)
self.pos = nn.Parameter(torch.zeros(1, maxlen, d))
self.blocks = nn.ModuleList([Block() for _ in range(L)])
self.ln_f = nn.LayerNorm(d)
self.head = nn.Linear(d, v, bias=False)
if cfg.get("tied_embeddings", True):
self.head.weight = self.tok_emb.weight
def forward(self, idx):
B, T = idx.shape
x = self.tok_emb(idx) + self.pos[:, :T]
for b in self.blocks:
x = b(x)
return self.ln_f(x)
return SAN()