Download training/kernels/common/model_builder.py from geraldaton20/alter_ego: direct link, hf CLI and curl.
- Browser
- Download file 2.35 kB
-
https://huggingface.co/geraldaton20/alter_ego/resolve/main/training/kernels/common/model_builder.py
- Command line
-
hf download hf://geraldaton20/alter_ego/training/kernels/common/model_builder.py
-
curl -L -o model_builder.py https://huggingface.co/geraldaton20/alter_ego/resolve/main/training/kernels/common/model_builder.py
2.35 kB
| # Student model builder for Kaggle kernels (mirrors student/model.py reference prototype). | |
| # The production dual-model system is Rust; this is the Python/torch training definition. | |
| import torch | |
| import torch.nn as nn | |
| def estimate_params(cfg): | |
| v = cfg["vocab"]; d = cfg["d_model"]; L = cfg["n_layers"]; ffn = cfg["ffn_ratio"] * d | |
| p = v * d | |
| per = (4 * d * d) + (2 * d * ffn) + (2 * 2 * d) | |
| p += L * per | |
| p += 2 * d | |
| return p | |
| def build_model(cfg): | |
| d, L, ffn = cfg["d_model"], cfg["n_layers"], cfg["ffn_ratio"] * cfg["d_model"] | |
| h, v, maxlen = cfg["n_heads"], cfg["vocab"], cfg["max_seq_len"] | |
| class Attn(nn.Module): | |
| def __init__(self): | |
| super().__init__() | |
| self.qkv = nn.Linear(d, 3 * d, bias=False) | |
| self.o = nn.Linear(d, d, bias=False) | |
| def forward(self, x): | |
| B, T, _ = x.shape | |
| q, k, vv = self.qkv(x).chunk(3, -1) | |
| q = q.view(B, T, h, d // h).transpose(1, 2) | |
| k = k.view(B, T, h, d // h).transpose(1, 2) | |
| vv = vv.view(B, T, h, d // h).transpose(1, 2) | |
| out = torch.nn.functional.scaled_dot_product_attention(q, k, vv) | |
| out = out.transpose(1, 2).reshape(B, T, d) | |
| return self.o(out) | |
| class Block(nn.Module): | |
| def __init__(self): | |
| super().__init__() | |
| self.ln1 = nn.LayerNorm(d) | |
| self.attn = Attn() | |
| self.ln2 = nn.LayerNorm(d) | |
| self.mlp = nn.Sequential(nn.Linear(d, ffn), nn.GELU(), nn.Linear(ffn, d)) | |
| def forward(self, x): | |
| x = x + self.attn(self.ln1(x)) | |
| x = x + self.mlp(self.ln2(x)) | |
| return x | |
| class SAN(nn.Module): | |
| def __init__(self): | |
| super().__init__() | |
| self.tok_emb = nn.Embedding(v, d) | |
| self.pos = nn.Parameter(torch.zeros(1, maxlen, d)) | |
| self.blocks = nn.ModuleList([Block() for _ in range(L)]) | |
| self.ln_f = nn.LayerNorm(d) | |
| self.head = nn.Linear(d, v, bias=False) | |
| if cfg.get("tied_embeddings", True): | |
| self.head.weight = self.tok_emb.weight | |
| def forward(self, idx): | |
| B, T = idx.shape | |
| x = self.tok_emb(idx) + self.pos[:, :T] | |
| for b in self.blocks: | |
| x = b(x) | |
| return self.ln_f(x) | |
| return SAN() | |