import math import torch import torch.nn as nn import torch.nn.functional as F class InputEmbedding(nn.Module): def __init__(self, vocab_size, embedding_dim): super().__init__() self.embedding = nn.Embedding(vocab_size, embedding_dim) def forward(self, x): return self.embedding(x) class RMSNorm(nn.Module): def __init__(self, embed_dim, rms_norm_eps=1e-9): super().__init__() self.rms_norm_eps = rms_norm_eps self.weight = nn.Parameter(torch.ones(embed_dim)) def forward(self, x): variance = x.pow(2).mean(dim=-1, keepdim=True) std = torch.sqrt(variance + self.rms_norm_eps) x_norm = x / std return x_norm * self.weight class RoPE(nn.Module): def __init__(self, embed_dim, context_len, rope_theta): super().__init__() self.context_len = context_len self.embed_dim = embed_dim N = rope_theta inv_freq = (1. / N ** (torch.arange(0, embed_dim, 2).float() / embed_dim)) inv_freq = torch.cat((inv_freq, inv_freq), dim=-1) positions = torch.arange(context_len) rotation_angle = positions.unsqueeze(-1) * inv_freq.unsqueeze(0) # non-persistent: recomputed at init, kept out of checkpoints self.register_buffer('cos', torch.cos(rotation_angle), persistent=False) self.register_buffer('sin', torch.sin(rotation_angle), persistent=False) def forward(self, x, pos_offset=0): seq_len = x.size(2) x1, x2 = x.chunk(2, dim=-1) adj_cos = self.cos[pos_offset: pos_offset + seq_len].unsqueeze(0).unsqueeze(0).to(x.dtype) adj_sin = self.sin[pos_offset: pos_offset + seq_len].unsqueeze(0).unsqueeze(0).to(x.dtype) rotate = torch.cat((-x2, x1), dim=-1) x_adjusted = (x * adj_cos) + ( rotate * adj_sin ) return x_adjusted class MultiHeadLatentAttention(nn.Module): def __init__(self, embed_dim, context_len, num_heads, head_dim=None, qk_norm=False, latent_dim=None, rope_theta=10000, rms_norm_eps=1e-6, rope=None): super().__init__() self.embed_dim = embed_dim self.context_len = context_len self.num_heads = num_heads self.qk_norm = qk_norm if head_dim is None: head_dim = embed_dim // num_heads self.head_dim = head_dim self.hidden_dim = self.head_dim * num_heads self.latent_dim = latent_dim if latent_dim is not None else max(16, embed_dim // 8) self.w_q = nn.Linear(embed_dim, self.hidden_dim) self.w_dkv = nn.Linear(embed_dim, self.latent_dim) self.w_uk = nn.Linear(self.latent_dim, self.hidden_dim) self.w_uv = nn.Linear(self.latent_dim, self.hidden_dim) self.w_o = nn.Linear(self.hidden_dim, embed_dim) # one RoPE table shared by all layers (a 131072-long table per layer would waste ~770 MB) self.rope = rope if rope is not None else RoPE(self.head_dim, context_len, rope_theta) if self.qk_norm: self.q_norm = RMSNorm(self.head_dim, rms_norm_eps) self.k_norm = RMSNorm(self.head_dim, rms_norm_eps) else: self.q_norm = self.k_norm = None self.register_buffer("cache_ckv", None, persistent=False) self.ptr_current_pos = 0 def reset_cache(self): self.cache_ckv = None self.ptr_current_pos = 0 @staticmethod def _get_causal_mask(seq_len, kv_len, device): q_pos = torch.arange(seq_len, device=device).unsqueeze(1) + (kv_len - seq_len) k_pos = torch.arange(kv_len, device=device).unsqueeze(0) return (k_pos <= q_pos).float() def forward(self, x, mask=None, use_cache=False): b, seq_len, embed_dim = x.shape q = self.w_q(x) ckv = self.w_dkv(x) if use_cache: if self.cache_ckv is None: updated_cache = ckv else: updated_cache = torch.cat((self.cache_ckv, ckv), dim=1) self.cache_ckv = updated_cache pos_offset = self.ptr_current_pos self.ptr_current_pos += seq_len else: updated_cache = ckv pos_offset = 0 kv_len = updated_cache.shape[1] k = self.w_uk(updated_cache) v = self.w_uv(updated_cache) q = q.view(b, seq_len, self.num_heads, self.head_dim).transpose(1, 2) k = k.view(b, kv_len, self.num_heads, self.head_dim).transpose(1, 2) v = v.view(b, kv_len, self.num_heads, self.head_dim).transpose(1, 2) if self.qk_norm: q = self.q_norm(q) k = self.k_norm(k) q = self.rope(q, pos_offset=pos_offset) k = self.rope(k, pos_offset=0) # attn_score = (q @ k.transpose(2, 3)) / math.sqrt(self.head_dim) # if mask is None: # mask = self._get_causal_mask(seq_len, kv_len, x.device) # attn_score = attn_score.masked_fill(mask == 0, -1e9) # weights = F.softmax(attn_score, dim=-1) # # context_vec = torch.matmul(weights, v).transpose(1, 2).contiguous() if mask is None and seq_len == kv_len: context_vec = F.scaled_dot_product_attention( q, k, v, attn_mask=None, is_causal=True, dropout_p=0.0, ) else: attn_mask = self._get_causal_mask(seq_len, kv_len, x.device).bool() if mask is not None: # mask: [b, kv_len] key-padding mask, True = real token (batched generation with left padding). # Every query may also see itself, so rows that are all padding stay finite (never NaN); # real queries never attend to pad keys, so padding cannot leak into real outputs. q_pos = torch.arange(seq_len, device=x.device).unsqueeze(1) + (kv_len - seq_len) k_pos = torch.arange(kv_len, device=x.device).unsqueeze(0) attn_mask = (attn_mask[None, None] & mask[:, None, None, :].bool()) | (k_pos == q_pos)[None, None] context_vec = F.scaled_dot_product_attention( q, k, v, attn_mask=attn_mask, is_causal=False, dropout_p=0.0, ) context_vec = context_vec.transpose(1, 2).contiguous() context_vec = context_vec.view(b, seq_len, self.hidden_dim) logits = self.w_o(context_vec) return logits class SkipConnection(nn.Module): def __init__(self): super().__init__() def forward(self, x1, x2): return x1+x2 class FeedForwardLayer(nn.Module): def __init__(self, embed_dim, hidden_dim): super().__init__() self.fc1 = nn.Linear(embed_dim, hidden_dim) self.fc2 = nn.Linear(embed_dim, hidden_dim) self.fc3 = nn.Linear(hidden_dim, embed_dim) def forward(self, x): up = self.fc1(x) gate = self.fc2(x) down = F.silu(up) * gate output = self.fc3(down) return output class OutputLinearLayer(nn.Module): def __init__(self, embedding_dim, vocab_size): super().__init__() self.linear = nn.Linear(embedding_dim, vocab_size) def forward(self, x): return self.linear(x) class TransformerBlock(nn.Module): def __init__(self, embed_dim, context_len, num_heads, head_dim=None, qk_norm=False, latent_dim=None, ffn_hidden_dim=None, rope_theta=10000, rms_norm_eps=1e-6, rope=None): super().__init__() self.embed_dim = embed_dim self.num_heads = num_heads if ffn_hidden_dim is None: ffn_hidden_dim = embed_dim self.rms_norm1 = RMSNorm(embed_dim, rms_norm_eps) self.mla_block = MultiHeadLatentAttention(embed_dim, context_len, num_heads, head_dim, qk_norm, latent_dim, rope_theta, rms_norm_eps, rope) self.skip_connection = SkipConnection() self.rms_norm2 = RMSNorm(embed_dim, rms_norm_eps) self.ffn = FeedForwardLayer(embed_dim, ffn_hidden_dim) def forward(self, x, mask=None, use_cache=False): residual_1 = x x = self.rms_norm1(x) x = self.mla_block(x, mask, use_cache) x = self.skip_connection(x, residual_1) residual_2 = x x = self.rms_norm2(x) x = self.ffn(x) x = self.skip_connection(x, residual_2) return x class NanoI1Model(nn.Module): def __init__(self, vocab_size, embed_dim, context_len, num_heads, head_dim=None, qk_norm=False, latent_dim=None, ffn_hidden_dim=None, transformer_layers=4, rope_theta=10000, rms_norm_eps=1e-6, tie_embeddings=False): super().__init__() self.config = dict(vocab_size=vocab_size, embed_dim=embed_dim, context_len=context_len, num_heads=num_heads, head_dim=head_dim, qk_norm=qk_norm, latent_dim=latent_dim, ffn_hidden_dim=ffn_hidden_dim, transformer_layers=transformer_layers, rope_theta=rope_theta, rms_norm_eps=rms_norm_eps, tie_embeddings=tie_embeddings) hd = head_dim if head_dim is not None else embed_dim // num_heads self.rope = RoPE(hd, context_len, rope_theta) self.input_embedding = nn.Embedding(vocab_size, embed_dim) self.transformer_blocks = nn.ModuleList([ TransformerBlock(embed_dim, context_len, num_heads, head_dim, qk_norm, latent_dim, ffn_hidden_dim, rope_theta, rms_norm_eps, self.rope) for _ in range(transformer_layers) ]) self.final_norm = RMSNorm(embed_dim, rms_norm_eps) self.output_linear = OutputLinearLayer(embed_dim, vocab_size) if tie_embeddings: self.output_linear.linear.weight = self.input_embedding.weight self.apply(self._init_weights) # GPT-2 / OLMo-style scaled init for projections that write into the residual stream std_res = 0.02 / math.sqrt(2 * transformer_layers) for blk in self.transformer_blocks: nn.init.normal_(blk.mla_block.w_o.weight, mean=0.0, std=std_res) nn.init.normal_(blk.ffn.fc3.weight, mean=0.0, std=std_res) @staticmethod def _init_weights(m): if isinstance(m, nn.Linear): nn.init.normal_(m.weight, mean=0.0, std=0.02) if m.bias is not None: nn.init.zeros_(m.bias) elif isinstance(m, nn.Embedding): nn.init.normal_(m.weight, mean=0.0, std=0.02) @classmethod def from_config(cls, path_or_dict, **overrides): cfg = load_config(path_or_dict) if isinstance(path_or_dict, str) else dict(path_or_dict) cfg.update(overrides) return cls(**cfg) def num_params(self, non_embedding=True): n = sum(p.numel() for p in self.parameters()) if non_embedding: n -= self.input_embedding.weight.numel() return n def reset_cache(self): for blk in self.transformer_blocks: blk.mla_block.reset_cache() def forward(self, x, mask=None, use_cache=False, targets=None, reduction="mean"): """Returns logits; or, when `targets` is given, the next-token loss. Targets equal to -100 are ignored (SFT: only assistant tokens are learned), and the output projection is computed only for the kept positions. With self.fused_loss set (fused_loss.py) the full logits are never materialised. reduction: 'mean' over kept tokens, or 'sum'.""" x = self.input_embedding(x) for block in self.transformer_blocks: x = block(x, mask, use_cache) x = self.final_norm(x) if targets is not None: return self._loss(x, targets, reduction) x = self.output_linear(x) return x @torch.compiler.disable def _loss(self, h, targets, reduction): h = h.reshape(-1, h.size(-1)) t = targets.reshape(-1) keep = t != -100 if not bool(keep.all()): h, t = h[keep], t[keep] lin = self.output_linear.linear if t.numel() == 0: # nothing to learn in this batch: zero loss that still touches every param (DDP) zero = h.sum() * 0.0 + lin.weight.sum() * 0.0 return zero + lin.bias.sum() * 0.0 if lin.bias is not None else zero fused = getattr(self, "fused_loss", None) if fused is not None: loss = fused(lin.weight, h, t, lin.bias) else: loss = F.cross_entropy(self.output_linear(h).float(), t) return loss * t.numel() if reduction == "sum" else loss def load_config(path): """config.json loader that tolerates // line comments.""" import json with open(path) as f: text = "\n".join(l for l in f if not l.lstrip().startswith("//")) return json.loads(text)