Download model.py from KiteFishAI/NanoI1-155M-Base: direct link, hf CLI and curl.
- Browser
- Download file 13.2 kB
-
https://huggingface.co/KiteFishAI/NanoI1-155M-Base/resolve/main/model.py
- Command line
-
hf download hf://KiteFishAI/NanoI1-155M-Base/model.py
-
curl -L -o model.py https://huggingface.co/KiteFishAI/NanoI1-155M-Base/resolve/main/model.py
13.2 kB
| import math | |
| import torch | |
| import torch.nn as nn | |
| import torch.nn.functional as F | |
| class InputEmbedding(nn.Module): | |
| def __init__(self, vocab_size, embedding_dim): | |
| super().__init__() | |
| self.embedding = nn.Embedding(vocab_size, embedding_dim) | |
| def forward(self, x): | |
| return self.embedding(x) | |
| class RMSNorm(nn.Module): | |
| def __init__(self, embed_dim, rms_norm_eps=1e-9): | |
| super().__init__() | |
| self.rms_norm_eps = rms_norm_eps | |
| self.weight = nn.Parameter(torch.ones(embed_dim)) | |
| def forward(self, x): | |
| variance = x.pow(2).mean(dim=-1, keepdim=True) | |
| std = torch.sqrt(variance + self.rms_norm_eps) | |
| x_norm = x / std | |
| return x_norm * self.weight | |
| class RoPE(nn.Module): | |
| def __init__(self, embed_dim, context_len, rope_theta): | |
| super().__init__() | |
| self.context_len = context_len | |
| self.embed_dim = embed_dim | |
| N = rope_theta | |
| inv_freq = (1. / N ** (torch.arange(0, embed_dim, 2).float() / embed_dim)) | |
| inv_freq = torch.cat((inv_freq, inv_freq), dim=-1) | |
| positions = torch.arange(context_len) | |
| rotation_angle = positions.unsqueeze(-1) * inv_freq.unsqueeze(0) | |
| # non-persistent: recomputed at init, kept out of checkpoints | |
| self.register_buffer('cos', torch.cos(rotation_angle), persistent=False) | |
| self.register_buffer('sin', torch.sin(rotation_angle), persistent=False) | |
| def forward(self, x, pos_offset=0): | |
| seq_len = x.size(2) | |
| x1, x2 = x.chunk(2, dim=-1) | |
| adj_cos = self.cos[pos_offset: pos_offset + seq_len].unsqueeze(0).unsqueeze(0).to(x.dtype) | |
| adj_sin = self.sin[pos_offset: pos_offset + seq_len].unsqueeze(0).unsqueeze(0).to(x.dtype) | |
| rotate = torch.cat((-x2, x1), dim=-1) | |
| x_adjusted = (x * adj_cos) + ( rotate * adj_sin ) | |
| return x_adjusted | |
| class MultiHeadLatentAttention(nn.Module): | |
| def __init__(self, embed_dim, context_len, num_heads, head_dim=None, qk_norm=False, latent_dim=None, | |
| rope_theta=10000, rms_norm_eps=1e-6, rope=None): | |
| super().__init__() | |
| self.embed_dim = embed_dim | |
| self.context_len = context_len | |
| self.num_heads = num_heads | |
| self.qk_norm = qk_norm | |
| if head_dim is None: | |
| head_dim = embed_dim // num_heads | |
| self.head_dim = head_dim | |
| self.hidden_dim = self.head_dim * num_heads | |
| self.latent_dim = latent_dim if latent_dim is not None else max(16, embed_dim // 8) | |
| self.w_q = nn.Linear(embed_dim, self.hidden_dim) | |
| self.w_dkv = nn.Linear(embed_dim, self.latent_dim) | |
| self.w_uk = nn.Linear(self.latent_dim, self.hidden_dim) | |
| self.w_uv = nn.Linear(self.latent_dim, self.hidden_dim) | |
| self.w_o = nn.Linear(self.hidden_dim, embed_dim) | |
| # one RoPE table shared by all layers (a 131072-long table per layer would waste ~770 MB) | |
| self.rope = rope if rope is not None else RoPE(self.head_dim, context_len, rope_theta) | |
| if self.qk_norm: | |
| self.q_norm = RMSNorm(self.head_dim, rms_norm_eps) | |
| self.k_norm = RMSNorm(self.head_dim, rms_norm_eps) | |
| else: | |
| self.q_norm = self.k_norm = None | |
| self.register_buffer("cache_ckv", None, persistent=False) | |
| self.ptr_current_pos = 0 | |
| def reset_cache(self): | |
| self.cache_ckv = None | |
| self.ptr_current_pos = 0 | |
| def _get_causal_mask(seq_len, kv_len, device): | |
| q_pos = torch.arange(seq_len, device=device).unsqueeze(1) + (kv_len - seq_len) | |
| k_pos = torch.arange(kv_len, device=device).unsqueeze(0) | |
| return (k_pos <= q_pos).float() | |
| def forward(self, x, mask=None, use_cache=False): | |
| b, seq_len, embed_dim = x.shape | |
| q = self.w_q(x) | |
| ckv = self.w_dkv(x) | |
| if use_cache: | |
| if self.cache_ckv is None: | |
| updated_cache = ckv | |
| else: | |
| updated_cache = torch.cat((self.cache_ckv, ckv), dim=1) | |
| self.cache_ckv = updated_cache | |
| pos_offset = self.ptr_current_pos | |
| self.ptr_current_pos += seq_len | |
| else: | |
| updated_cache = ckv | |
| pos_offset = 0 | |
| kv_len = updated_cache.shape[1] | |
| k = self.w_uk(updated_cache) | |
| v = self.w_uv(updated_cache) | |
| q = q.view(b, seq_len, self.num_heads, self.head_dim).transpose(1, 2) | |
| k = k.view(b, kv_len, self.num_heads, self.head_dim).transpose(1, 2) | |
| v = v.view(b, kv_len, self.num_heads, self.head_dim).transpose(1, 2) | |
| if self.qk_norm: | |
| q = self.q_norm(q) | |
| k = self.k_norm(k) | |
| q = self.rope(q, pos_offset=pos_offset) | |
| k = self.rope(k, pos_offset=0) | |
| # attn_score = (q @ k.transpose(2, 3)) / math.sqrt(self.head_dim) | |
| # if mask is None: | |
| # mask = self._get_causal_mask(seq_len, kv_len, x.device) | |
| # attn_score = attn_score.masked_fill(mask == 0, -1e9) | |
| # weights = F.softmax(attn_score, dim=-1) | |
| # | |
| # context_vec = torch.matmul(weights, v).transpose(1, 2).contiguous() | |
| if mask is None and seq_len == kv_len: | |
| context_vec = F.scaled_dot_product_attention( | |
| q, k, v, | |
| attn_mask=None, | |
| is_causal=True, | |
| dropout_p=0.0, | |
| ) | |
| else: | |
| attn_mask = self._get_causal_mask(seq_len, kv_len, x.device).bool() | |
| if mask is not None: | |
| # mask: [b, kv_len] key-padding mask, True = real token (batched generation with left padding). | |
| # Every query may also see itself, so rows that are all padding stay finite (never NaN); | |
| # real queries never attend to pad keys, so padding cannot leak into real outputs. | |
| q_pos = torch.arange(seq_len, device=x.device).unsqueeze(1) + (kv_len - seq_len) | |
| k_pos = torch.arange(kv_len, device=x.device).unsqueeze(0) | |
| attn_mask = (attn_mask[None, None] & mask[:, None, None, :].bool()) | (k_pos == q_pos)[None, None] | |
| context_vec = F.scaled_dot_product_attention( | |
| q, k, v, | |
| attn_mask=attn_mask, | |
| is_causal=False, | |
| dropout_p=0.0, | |
| ) | |
| context_vec = context_vec.transpose(1, 2).contiguous() | |
| context_vec = context_vec.view(b, seq_len, self.hidden_dim) | |
| logits = self.w_o(context_vec) | |
| return logits | |
| class SkipConnection(nn.Module): | |
| def __init__(self): | |
| super().__init__() | |
| def forward(self, x1, x2): | |
| return x1+x2 | |
| class FeedForwardLayer(nn.Module): | |
| def __init__(self, embed_dim, hidden_dim): | |
| super().__init__() | |
| self.fc1 = nn.Linear(embed_dim, hidden_dim) | |
| self.fc2 = nn.Linear(embed_dim, hidden_dim) | |
| self.fc3 = nn.Linear(hidden_dim, embed_dim) | |
| def forward(self, x): | |
| up = self.fc1(x) | |
| gate = self.fc2(x) | |
| down = F.silu(up) * gate | |
| output = self.fc3(down) | |
| return output | |
| class OutputLinearLayer(nn.Module): | |
| def __init__(self, embedding_dim, vocab_size): | |
| super().__init__() | |
| self.linear = nn.Linear(embedding_dim, vocab_size) | |
| def forward(self, x): | |
| return self.linear(x) | |
| class TransformerBlock(nn.Module): | |
| def __init__(self, embed_dim, | |
| context_len, | |
| num_heads, | |
| head_dim=None, | |
| qk_norm=False, | |
| latent_dim=None, | |
| ffn_hidden_dim=None, | |
| rope_theta=10000, | |
| rms_norm_eps=1e-6, | |
| rope=None): | |
| super().__init__() | |
| self.embed_dim = embed_dim | |
| self.num_heads = num_heads | |
| if ffn_hidden_dim is None: | |
| ffn_hidden_dim = embed_dim | |
| self.rms_norm1 = RMSNorm(embed_dim, rms_norm_eps) | |
| self.mla_block = MultiHeadLatentAttention(embed_dim, context_len, num_heads, head_dim, qk_norm, latent_dim, | |
| rope_theta, rms_norm_eps, rope) | |
| self.skip_connection = SkipConnection() | |
| self.rms_norm2 = RMSNorm(embed_dim, rms_norm_eps) | |
| self.ffn = FeedForwardLayer(embed_dim, ffn_hidden_dim) | |
| def forward(self, x, mask=None, use_cache=False): | |
| residual_1 = x | |
| x = self.rms_norm1(x) | |
| x = self.mla_block(x, mask, use_cache) | |
| x = self.skip_connection(x, residual_1) | |
| residual_2 = x | |
| x = self.rms_norm2(x) | |
| x = self.ffn(x) | |
| x = self.skip_connection(x, residual_2) | |
| return x | |
| class NanoI1Model(nn.Module): | |
| def __init__(self, vocab_size, | |
| embed_dim, | |
| context_len, | |
| num_heads, | |
| head_dim=None, | |
| qk_norm=False, | |
| latent_dim=None, | |
| ffn_hidden_dim=None, | |
| transformer_layers=4, | |
| rope_theta=10000, | |
| rms_norm_eps=1e-6, | |
| tie_embeddings=False): | |
| super().__init__() | |
| self.config = dict(vocab_size=vocab_size, embed_dim=embed_dim, context_len=context_len, | |
| num_heads=num_heads, head_dim=head_dim, qk_norm=qk_norm, latent_dim=latent_dim, | |
| ffn_hidden_dim=ffn_hidden_dim, transformer_layers=transformer_layers, | |
| rope_theta=rope_theta, rms_norm_eps=rms_norm_eps, tie_embeddings=tie_embeddings) | |
| hd = head_dim if head_dim is not None else embed_dim // num_heads | |
| self.rope = RoPE(hd, context_len, rope_theta) | |
| self.input_embedding = nn.Embedding(vocab_size, embed_dim) | |
| self.transformer_blocks = nn.ModuleList([ | |
| TransformerBlock(embed_dim, context_len, num_heads, head_dim, qk_norm, latent_dim, ffn_hidden_dim, | |
| rope_theta, rms_norm_eps, self.rope) for _ in range(transformer_layers) | |
| ]) | |
| self.final_norm = RMSNorm(embed_dim, rms_norm_eps) | |
| self.output_linear = OutputLinearLayer(embed_dim, vocab_size) | |
| if tie_embeddings: | |
| self.output_linear.linear.weight = self.input_embedding.weight | |
| self.apply(self._init_weights) | |
| # GPT-2 / OLMo-style scaled init for projections that write into the residual stream | |
| std_res = 0.02 / math.sqrt(2 * transformer_layers) | |
| for blk in self.transformer_blocks: | |
| nn.init.normal_(blk.mla_block.w_o.weight, mean=0.0, std=std_res) | |
| nn.init.normal_(blk.ffn.fc3.weight, mean=0.0, std=std_res) | |
| def _init_weights(m): | |
| if isinstance(m, nn.Linear): | |
| nn.init.normal_(m.weight, mean=0.0, std=0.02) | |
| if m.bias is not None: | |
| nn.init.zeros_(m.bias) | |
| elif isinstance(m, nn.Embedding): | |
| nn.init.normal_(m.weight, mean=0.0, std=0.02) | |
| def from_config(cls, path_or_dict, **overrides): | |
| cfg = load_config(path_or_dict) if isinstance(path_or_dict, str) else dict(path_or_dict) | |
| cfg.update(overrides) | |
| return cls(**cfg) | |
| def num_params(self, non_embedding=True): | |
| n = sum(p.numel() for p in self.parameters()) | |
| if non_embedding: | |
| n -= self.input_embedding.weight.numel() | |
| return n | |
| def reset_cache(self): | |
| for blk in self.transformer_blocks: | |
| blk.mla_block.reset_cache() | |
| def forward(self, x, mask=None, use_cache=False, targets=None, reduction="mean"): | |
| """Returns logits; or, when `targets` is given, the next-token loss. | |
| Targets equal to -100 are ignored (SFT: only assistant tokens are learned), and the output | |
| projection is computed only for the kept positions. With self.fused_loss set (fused_loss.py) | |
| the full logits are never materialised. reduction: 'mean' over kept tokens, or 'sum'.""" | |
| x = self.input_embedding(x) | |
| for block in self.transformer_blocks: | |
| x = block(x, mask, use_cache) | |
| x = self.final_norm(x) | |
| if targets is not None: | |
| return self._loss(x, targets, reduction) | |
| x = self.output_linear(x) | |
| return x | |
| def _loss(self, h, targets, reduction): | |
| h = h.reshape(-1, h.size(-1)) | |
| t = targets.reshape(-1) | |
| keep = t != -100 | |
| if not bool(keep.all()): | |
| h, t = h[keep], t[keep] | |
| lin = self.output_linear.linear | |
| if t.numel() == 0: # nothing to learn in this batch: zero loss that still touches every param (DDP) | |
| zero = h.sum() * 0.0 + lin.weight.sum() * 0.0 | |
| return zero + lin.bias.sum() * 0.0 if lin.bias is not None else zero | |
| fused = getattr(self, "fused_loss", None) | |
| if fused is not None: | |
| loss = fused(lin.weight, h, t, lin.bias) | |
| else: | |
| loss = F.cross_entropy(self.output_linear(h).float(), t) | |
| return loss * t.numel() if reduction == "sum" else loss | |
| def load_config(path): | |
| """config.json loader that tolerates // line comments.""" | |
| import json | |
| with open(path) as f: | |
| text = "\n".join(l for l in f if not l.lstrip().startswith("//")) | |
| return json.loads(text) | |