from __future__ import annotations import math from dataclasses import dataclass import torch import torch.nn as nn import torch.nn.functional as F import torch.utils.checkpoint try: from flash_attn import flash_attn_func as _flash_attn_func HAS_FLASH_ATTN = True except Exception: # A package can be discoverable while its CUDA extension is ABI-incompatible # with the installed Torch. Treat that as unavailable so startup falls # back safely to PyTorch SDPA instead of crashing on the first batch. _flash_attn_func = None HAS_FLASH_ATTN = False try: import liger_kernel # noqa: F401 HAS_LIGER = True except Exception: HAS_LIGER = False @dataclass class ModelConfig: vocab_size: int = 32_768 sequence_len: int = 2_048 n_layer: int = 21 n_head: int = 10 n_kv_head: int = 10 n_embd: int = 1_280 mlp_ratio: float = 4.0 rope_base: float = 10_000.0 dropout: float = 0.0 use_qk_norm: bool = True use_liger: bool = True attention_backend: str = "auto" # auto, flash_attn, sdpa compute_dtype: str = "bfloat16" @property def head_dim(self) -> int: if self.n_embd % self.n_head: raise ValueError("n_embd must be divisible by n_head") return self.n_embd // self.n_head @property def intermediate_size(self) -> int: return int(self.n_embd * self.mlp_ratio) def _rmsnorm_cls(use_liger: bool): if use_liger and HAS_LIGER: try: from liger_kernel.transformers import LigerRMSNorm return LigerRMSNorm except (ImportError, AttributeError): pass return None class RMSNorm(nn.Module): """Parameterless RMSNorm executed in the activation dtype.""" def __init__(self, dim: int, eps: float = 1e-6, use_liger: bool = False): super().__init__() self.dim = dim self.eps = eps liger = _rmsnorm_cls(use_liger) self._impl = liger(dim, eps=eps) if liger is not None else None def forward(self, x: torch.Tensor) -> torch.Tensor: if self._impl is not None: return self._impl(x) return F.rms_norm(x, (self.dim,), eps=self.eps) class Linear(nn.Linear): """Keep fp32 master weights while running matmuls in activation dtype.""" def forward(self, x: torch.Tensor) -> torch.Tensor: weight = self.weight.to(dtype=x.dtype) bias = self.bias.to(dtype=x.dtype) if self.bias is not None else None return F.linear(x, weight, bias) def _rotate_half(x: torch.Tensor) -> torch.Tensor: x1, x2 = x[..., : x.shape[-1] // 2], x[..., x.shape[-1] // 2 :] return torch.cat((-x2, x1), dim=-1) def _apply_rope(x: torch.Tensor, cos: torch.Tensor, sin: torch.Tensor) -> torch.Tensor: return (x * cos) + (_rotate_half(x) * sin) class CausalSelfAttention(nn.Module): def __init__(self, config: ModelConfig): super().__init__() if config.n_head % config.n_kv_head: raise ValueError("n_head must be divisible by n_kv_head") self.n_head = config.n_head self.n_kv_head = config.n_kv_head self.head_dim = config.head_dim self.dropout = config.dropout self.backend = config.attention_backend if self.backend == "auto": self.backend = "flash_attn" if HAS_FLASH_ATTN else "sdpa" self.qkv = Linear(config.n_embd, (config.n_head + 2 * config.n_kv_head) * self.head_dim, bias=False) self.proj = Linear(config.n_embd, config.n_embd, bias=False) self.q_norm = RMSNorm(self.head_dim, use_liger=config.use_liger) if config.use_qk_norm else nn.Identity() self.k_norm = RMSNorm(self.head_dim, use_liger=config.use_liger) if config.use_qk_norm else nn.Identity() def forward(self, x: torch.Tensor, cos: torch.Tensor, sin: torch.Tensor) -> torch.Tensor: batch, length, _ = x.shape qkv = self.qkv(x).view(batch, length, self.n_head + 2 * self.n_kv_head, self.head_dim) q = qkv[:, :, : self.n_head] k = qkv[:, :, self.n_head : self.n_head + self.n_kv_head] v = qkv[:, :, self.n_head + self.n_kv_head :] q = _apply_rope(self.q_norm(q), cos[:, :length], sin[:, :length]) k = _apply_rope(self.k_norm(k), cos[:, :length], sin[:, :length]) if self.n_kv_head != self.n_head: repeats = self.n_head // self.n_kv_head k = k.repeat_interleave(repeats, dim=2) v = v.repeat_interleave(repeats, dim=2) if self.backend == "flash_attn" and HAS_FLASH_ATTN and x.is_cuda and x.dtype in (torch.float16, torch.bfloat16): y = _flash_attn_func(q, k, v, dropout_p=self.dropout if self.training else 0.0, causal=True) else: q_t = q.transpose(1, 2) k_t = k.transpose(1, 2) v_t = v.transpose(1, 2) y = F.scaled_dot_product_attention( q_t, k_t, v_t, dropout_p=self.dropout if self.training else 0.0, is_causal=True, ).transpose(1, 2) return self.proj(y.reshape(batch, length, -1)) class MLP(nn.Module): def __init__(self, config: ModelConfig): super().__init__() hidden = config.intermediate_size self.fc = Linear(config.n_embd, hidden, bias=False) self.proj = Linear(hidden, config.n_embd, bias=False) def forward(self, x: torch.Tensor) -> torch.Tensor: # ReLU-squared is the dense FNN used by nanochat and avoids a gated # third matrix multiply while retaining a useful smooth positive gate. return self.proj(F.relu(self.fc(x)).square()) class Block(nn.Module): def __init__(self, config: ModelConfig): super().__init__() self.attn_norm = RMSNorm(config.n_embd, use_liger=config.use_liger) self.attn = CausalSelfAttention(config) self.mlp_norm = RMSNorm(config.n_embd, use_liger=config.use_liger) self.mlp = MLP(config) def forward(self, x: torch.Tensor, cos: torch.Tensor, sin: torch.Tensor) -> torch.Tensor: x = x + self.attn(self.attn_norm(x), cos, sin) x = x + self.mlp(self.mlp_norm(x)) return x class GPT(nn.Module): def __init__(self, config: ModelConfig): super().__init__() self.config = config self.activation_checkpointing = False self.token_embedding = nn.Embedding(config.vocab_size, config.n_embd) self.blocks = nn.ModuleList([Block(config) for _ in range(config.n_layer)]) self.final_norm = RMSNorm(config.n_embd, use_liger=config.use_liger) self.lm_head = Linear(config.n_embd, config.vocab_size, bias=False) inv_freq = 1.0 / (config.rope_base ** (torch.arange(0, config.head_dim, 2).float() / config.head_dim)) positions = torch.arange(config.sequence_len).float() freqs = torch.outer(positions, inv_freq) emb = torch.cat((freqs, freqs), dim=-1) self.register_buffer("rope_cos", emb.cos()[None, :, None, :], persistent=False) self.register_buffer("rope_sin", emb.sin()[None, :, None, :], persistent=False) self.apply(self._init_weights) def _init_weights(self, module: nn.Module) -> None: if isinstance(module, nn.Linear): std = 0.02 if module is self.lm_head: std = 0.02 nn.init.normal_(module.weight, mean=0.0, std=std) elif isinstance(module, nn.Embedding): nn.init.normal_(module.weight, mean=0.0, std=0.02) def forward( self, input_ids: torch.Tensor, targets: torch.Tensor | None = None, loss_mask: torch.Tensor | None = None, return_logits: bool = True, ): if input_ids.shape[1] > self.config.sequence_len: raise ValueError(f"sequence length {input_ids.shape[1]} exceeds {self.config.sequence_len}") x = self.token_embedding(input_ids) compute_dtype = getattr(torch, self.config.compute_dtype) if x.dtype != compute_dtype: x = x.to(compute_dtype) cos = self.rope_cos.to(device=x.device, dtype=x.dtype) sin = self.rope_sin.to(device=x.device, dtype=x.dtype) for block in self.blocks: if self.training and self.activation_checkpointing: x = torch.utils.checkpoint.checkpoint(block, x, cos, sin, use_reentrant=False) else: x = block(x, cos, sin) logits = self.lm_head(self.final_norm(x)) if targets is None: return logits if loss_mask is None: loss = F.cross_entropy(logits.reshape(-1, logits.size(-1)).float(), targets.reshape(-1)) else: if loss_mask.shape != targets.shape: raise ValueError("loss_mask must have the same shape as targets") # The SFT mask is binary. Convert masked positions to the native # cross-entropy ignore index so PyTorch can use its fused mean # reduction instead of materializing a loss value for every # vocabulary row and then multiplying by the mask. This keeps the # assistant-only objective identical while avoiding a substantial # SFT throughput penalty. masked_targets = targets.masked_fill(~loss_mask, -100) loss = F.cross_entropy( logits.reshape(-1, logits.size(-1)).float(), masked_targets.reshape(-1), ignore_index=-100, reduction="sum", ) loss = loss / loss_mask.sum().to(dtype=loss.dtype).clamp_min(1.0) return (logits, loss) if return_logits else loss def num_parameters(self, non_embedding: bool = False) -> int: total = sum(p.numel() for p in self.parameters()) if non_embedding: total -= self.token_embedding.weight.numel() + self.lm_head.weight.numel() return total def architecture_summary(config: ModelConfig) -> dict: matrix_per_layer = ( (config.n_head + 2 * config.n_kv_head) * config.head_dim * config.n_embd + config.n_embd * config.n_embd + config.n_embd * config.intermediate_size + config.intermediate_size * config.n_embd ) norm_parameters = (2 * config.n_layer + 1) * config.n_embd if config.use_liger else 0 parameter_count = 2 * config.vocab_size * config.n_embd + config.n_layer * matrix_per_layer + norm_parameters return { "parameters": parameter_count, "non_embedding_parameters": parameter_count - 2 * config.vocab_size * config.n_embd, "attention_backend": config.attention_backend, "selected_attention_backend": ( "flash_attn" if config.attention_backend == "auto" and HAS_FLASH_ATTN else "sdpa" ), "flash_attn_available": HAS_FLASH_ATTN, "liger_available": HAS_LIGER, "config": config.__dict__, }