NanoI1-155M-Base / model.py
anuj0456's picture
Upload folder using huggingface_hub
8a73bb7 verified
Raw History Blame Contribute Delete
13.2 kB
import math
import torch
import torch.nn as nn
import torch.nn.functional as F
class InputEmbedding(nn.Module):
def __init__(self, vocab_size, embedding_dim):
super().__init__()
self.embedding = nn.Embedding(vocab_size, embedding_dim)
def forward(self, x):
return self.embedding(x)
class RMSNorm(nn.Module):
def __init__(self, embed_dim, rms_norm_eps=1e-9):
super().__init__()
self.rms_norm_eps = rms_norm_eps
self.weight = nn.Parameter(torch.ones(embed_dim))
def forward(self, x):
variance = x.pow(2).mean(dim=-1, keepdim=True)
std = torch.sqrt(variance + self.rms_norm_eps)
x_norm = x / std
return x_norm * self.weight
class RoPE(nn.Module):
def __init__(self, embed_dim, context_len, rope_theta):
super().__init__()
self.context_len = context_len
self.embed_dim = embed_dim
N = rope_theta
inv_freq = (1. / N ** (torch.arange(0, embed_dim, 2).float() / embed_dim))
inv_freq = torch.cat((inv_freq, inv_freq), dim=-1)
positions = torch.arange(context_len)
rotation_angle = positions.unsqueeze(-1) * inv_freq.unsqueeze(0)
# non-persistent: recomputed at init, kept out of checkpoints
self.register_buffer('cos', torch.cos(rotation_angle), persistent=False)
self.register_buffer('sin', torch.sin(rotation_angle), persistent=False)
def forward(self, x, pos_offset=0):
seq_len = x.size(2)
x1, x2 = x.chunk(2, dim=-1)
adj_cos = self.cos[pos_offset: pos_offset + seq_len].unsqueeze(0).unsqueeze(0).to(x.dtype)
adj_sin = self.sin[pos_offset: pos_offset + seq_len].unsqueeze(0).unsqueeze(0).to(x.dtype)
rotate = torch.cat((-x2, x1), dim=-1)
x_adjusted = (x * adj_cos) + ( rotate * adj_sin )
return x_adjusted
class MultiHeadLatentAttention(nn.Module):
def __init__(self, embed_dim, context_len, num_heads, head_dim=None, qk_norm=False, latent_dim=None,
rope_theta=10000, rms_norm_eps=1e-6, rope=None):
super().__init__()
self.embed_dim = embed_dim
self.context_len = context_len
self.num_heads = num_heads
self.qk_norm = qk_norm
if head_dim is None:
head_dim = embed_dim // num_heads
self.head_dim = head_dim
self.hidden_dim = self.head_dim * num_heads
self.latent_dim = latent_dim if latent_dim is not None else max(16, embed_dim // 8)
self.w_q = nn.Linear(embed_dim, self.hidden_dim)
self.w_dkv = nn.Linear(embed_dim, self.latent_dim)
self.w_uk = nn.Linear(self.latent_dim, self.hidden_dim)
self.w_uv = nn.Linear(self.latent_dim, self.hidden_dim)
self.w_o = nn.Linear(self.hidden_dim, embed_dim)
# one RoPE table shared by all layers (a 131072-long table per layer would waste ~770 MB)
self.rope = rope if rope is not None else RoPE(self.head_dim, context_len, rope_theta)
if self.qk_norm:
self.q_norm = RMSNorm(self.head_dim, rms_norm_eps)
self.k_norm = RMSNorm(self.head_dim, rms_norm_eps)
else:
self.q_norm = self.k_norm = None
self.register_buffer("cache_ckv", None, persistent=False)
self.ptr_current_pos = 0
def reset_cache(self):
self.cache_ckv = None
self.ptr_current_pos = 0
@staticmethod
def _get_causal_mask(seq_len, kv_len, device):
q_pos = torch.arange(seq_len, device=device).unsqueeze(1) + (kv_len - seq_len)
k_pos = torch.arange(kv_len, device=device).unsqueeze(0)
return (k_pos <= q_pos).float()
def forward(self, x, mask=None, use_cache=False):
b, seq_len, embed_dim = x.shape
q = self.w_q(x)
ckv = self.w_dkv(x)
if use_cache:
if self.cache_ckv is None:
updated_cache = ckv
else:
updated_cache = torch.cat((self.cache_ckv, ckv), dim=1)
self.cache_ckv = updated_cache
pos_offset = self.ptr_current_pos
self.ptr_current_pos += seq_len
else:
updated_cache = ckv
pos_offset = 0
kv_len = updated_cache.shape[1]
k = self.w_uk(updated_cache)
v = self.w_uv(updated_cache)
q = q.view(b, seq_len, self.num_heads, self.head_dim).transpose(1, 2)
k = k.view(b, kv_len, self.num_heads, self.head_dim).transpose(1, 2)
v = v.view(b, kv_len, self.num_heads, self.head_dim).transpose(1, 2)
if self.qk_norm:
q = self.q_norm(q)
k = self.k_norm(k)
q = self.rope(q, pos_offset=pos_offset)
k = self.rope(k, pos_offset=0)
# attn_score = (q @ k.transpose(2, 3)) / math.sqrt(self.head_dim)
# if mask is None:
# mask = self._get_causal_mask(seq_len, kv_len, x.device)
# attn_score = attn_score.masked_fill(mask == 0, -1e9)
# weights = F.softmax(attn_score, dim=-1)
#
# context_vec = torch.matmul(weights, v).transpose(1, 2).contiguous()
if mask is None and seq_len == kv_len:
context_vec = F.scaled_dot_product_attention(
q, k, v,
attn_mask=None,
is_causal=True,
dropout_p=0.0,
)
else:
attn_mask = self._get_causal_mask(seq_len, kv_len, x.device).bool()
if mask is not None:
# mask: [b, kv_len] key-padding mask, True = real token (batched generation with left padding).
# Every query may also see itself, so rows that are all padding stay finite (never NaN);
# real queries never attend to pad keys, so padding cannot leak into real outputs.
q_pos = torch.arange(seq_len, device=x.device).unsqueeze(1) + (kv_len - seq_len)
k_pos = torch.arange(kv_len, device=x.device).unsqueeze(0)
attn_mask = (attn_mask[None, None] & mask[:, None, None, :].bool()) | (k_pos == q_pos)[None, None]
context_vec = F.scaled_dot_product_attention(
q, k, v,
attn_mask=attn_mask,
is_causal=False,
dropout_p=0.0,
)
context_vec = context_vec.transpose(1, 2).contiguous()
context_vec = context_vec.view(b, seq_len, self.hidden_dim)
logits = self.w_o(context_vec)
return logits
class SkipConnection(nn.Module):
def __init__(self):
super().__init__()
def forward(self, x1, x2):
return x1+x2
class FeedForwardLayer(nn.Module):
def __init__(self, embed_dim, hidden_dim):
super().__init__()
self.fc1 = nn.Linear(embed_dim, hidden_dim)
self.fc2 = nn.Linear(embed_dim, hidden_dim)
self.fc3 = nn.Linear(hidden_dim, embed_dim)
def forward(self, x):
up = self.fc1(x)
gate = self.fc2(x)
down = F.silu(up) * gate
output = self.fc3(down)
return output
class OutputLinearLayer(nn.Module):
def __init__(self, embedding_dim, vocab_size):
super().__init__()
self.linear = nn.Linear(embedding_dim, vocab_size)
def forward(self, x):
return self.linear(x)
class TransformerBlock(nn.Module):
def __init__(self, embed_dim,
context_len,
num_heads,
head_dim=None,
qk_norm=False,
latent_dim=None,
ffn_hidden_dim=None,
rope_theta=10000,
rms_norm_eps=1e-6,
rope=None):
super().__init__()
self.embed_dim = embed_dim
self.num_heads = num_heads
if ffn_hidden_dim is None:
ffn_hidden_dim = embed_dim
self.rms_norm1 = RMSNorm(embed_dim, rms_norm_eps)
self.mla_block = MultiHeadLatentAttention(embed_dim, context_len, num_heads, head_dim, qk_norm, latent_dim,
rope_theta, rms_norm_eps, rope)
self.skip_connection = SkipConnection()
self.rms_norm2 = RMSNorm(embed_dim, rms_norm_eps)
self.ffn = FeedForwardLayer(embed_dim, ffn_hidden_dim)
def forward(self, x, mask=None, use_cache=False):
residual_1 = x
x = self.rms_norm1(x)
x = self.mla_block(x, mask, use_cache)
x = self.skip_connection(x, residual_1)
residual_2 = x
x = self.rms_norm2(x)
x = self.ffn(x)
x = self.skip_connection(x, residual_2)
return x
class NanoI1Model(nn.Module):
def __init__(self, vocab_size,
embed_dim,
context_len,
num_heads,
head_dim=None,
qk_norm=False,
latent_dim=None,
ffn_hidden_dim=None,
transformer_layers=4,
rope_theta=10000,
rms_norm_eps=1e-6,
tie_embeddings=False):
super().__init__()
self.config = dict(vocab_size=vocab_size, embed_dim=embed_dim, context_len=context_len,
num_heads=num_heads, head_dim=head_dim, qk_norm=qk_norm, latent_dim=latent_dim,
ffn_hidden_dim=ffn_hidden_dim, transformer_layers=transformer_layers,
rope_theta=rope_theta, rms_norm_eps=rms_norm_eps, tie_embeddings=tie_embeddings)
hd = head_dim if head_dim is not None else embed_dim // num_heads
self.rope = RoPE(hd, context_len, rope_theta)
self.input_embedding = nn.Embedding(vocab_size, embed_dim)
self.transformer_blocks = nn.ModuleList([
TransformerBlock(embed_dim, context_len, num_heads, head_dim, qk_norm, latent_dim, ffn_hidden_dim,
rope_theta, rms_norm_eps, self.rope) for _ in range(transformer_layers)
])
self.final_norm = RMSNorm(embed_dim, rms_norm_eps)
self.output_linear = OutputLinearLayer(embed_dim, vocab_size)
if tie_embeddings:
self.output_linear.linear.weight = self.input_embedding.weight
self.apply(self._init_weights)
# GPT-2 / OLMo-style scaled init for projections that write into the residual stream
std_res = 0.02 / math.sqrt(2 * transformer_layers)
for blk in self.transformer_blocks:
nn.init.normal_(blk.mla_block.w_o.weight, mean=0.0, std=std_res)
nn.init.normal_(blk.ffn.fc3.weight, mean=0.0, std=std_res)
@staticmethod
def _init_weights(m):
if isinstance(m, nn.Linear):
nn.init.normal_(m.weight, mean=0.0, std=0.02)
if m.bias is not None:
nn.init.zeros_(m.bias)
elif isinstance(m, nn.Embedding):
nn.init.normal_(m.weight, mean=0.0, std=0.02)
@classmethod
def from_config(cls, path_or_dict, **overrides):
cfg = load_config(path_or_dict) if isinstance(path_or_dict, str) else dict(path_or_dict)
cfg.update(overrides)
return cls(**cfg)
def num_params(self, non_embedding=True):
n = sum(p.numel() for p in self.parameters())
if non_embedding:
n -= self.input_embedding.weight.numel()
return n
def reset_cache(self):
for blk in self.transformer_blocks:
blk.mla_block.reset_cache()
def forward(self, x, mask=None, use_cache=False, targets=None, reduction="mean"):
"""Returns logits; or, when `targets` is given, the next-token loss.
Targets equal to -100 are ignored (SFT: only assistant tokens are learned), and the output
projection is computed only for the kept positions. With self.fused_loss set (fused_loss.py)
the full logits are never materialised. reduction: 'mean' over kept tokens, or 'sum'."""
x = self.input_embedding(x)
for block in self.transformer_blocks:
x = block(x, mask, use_cache)
x = self.final_norm(x)
if targets is not None:
return self._loss(x, targets, reduction)
x = self.output_linear(x)
return x
@torch.compiler.disable
def _loss(self, h, targets, reduction):
h = h.reshape(-1, h.size(-1))
t = targets.reshape(-1)
keep = t != -100
if not bool(keep.all()):
h, t = h[keep], t[keep]
lin = self.output_linear.linear
if t.numel() == 0: # nothing to learn in this batch: zero loss that still touches every param (DDP)
zero = h.sum() * 0.0 + lin.weight.sum() * 0.0
return zero + lin.bias.sum() * 0.0 if lin.bias is not None else zero
fused = getattr(self, "fused_loss", None)
if fused is not None:
loss = fused(lin.weight, h, t, lin.bias)
else:
loss = F.cross_entropy(self.output_linear(h).float(), t)
return loss * t.numel() if reduction == "sum" else loss
def load_config(path):
"""config.json loader that tolerates // line comments."""
import json
with open(path) as f:
text = "\n".join(l for l in f if not l.lstrip().startswith("//"))
return json.loads(text)