sripkunda's picture
Publish model checkpoint
fde9066 verified
Raw History Blame Contribute Delete
10.9 kB
from __future__ import annotations
import math
from dataclasses import dataclass
import torch
import torch.nn as nn
import torch.nn.functional as F
import torch.utils.checkpoint
try:
from flash_attn import flash_attn_func as _flash_attn_func
HAS_FLASH_ATTN = True
except Exception:
# A package can be discoverable while its CUDA extension is ABI-incompatible
# with the installed Torch. Treat that as unavailable so startup falls
# back safely to PyTorch SDPA instead of crashing on the first batch.
_flash_attn_func = None
HAS_FLASH_ATTN = False
try:
import liger_kernel # noqa: F401
HAS_LIGER = True
except Exception:
HAS_LIGER = False
@dataclass
class ModelConfig:
vocab_size: int = 32_768
sequence_len: int = 2_048
n_layer: int = 21
n_head: int = 10
n_kv_head: int = 10
n_embd: int = 1_280
mlp_ratio: float = 4.0
rope_base: float = 10_000.0
dropout: float = 0.0
use_qk_norm: bool = True
use_liger: bool = True
attention_backend: str = "auto" # auto, flash_attn, sdpa
compute_dtype: str = "bfloat16"
@property
def head_dim(self) -> int:
if self.n_embd % self.n_head:
raise ValueError("n_embd must be divisible by n_head")
return self.n_embd // self.n_head
@property
def intermediate_size(self) -> int:
return int(self.n_embd * self.mlp_ratio)
def _rmsnorm_cls(use_liger: bool):
if use_liger and HAS_LIGER:
try:
from liger_kernel.transformers import LigerRMSNorm
return LigerRMSNorm
except (ImportError, AttributeError):
pass
return None
class RMSNorm(nn.Module):
"""Parameterless RMSNorm executed in the activation dtype."""
def __init__(self, dim: int, eps: float = 1e-6, use_liger: bool = False):
super().__init__()
self.dim = dim
self.eps = eps
liger = _rmsnorm_cls(use_liger)
self._impl = liger(dim, eps=eps) if liger is not None else None
def forward(self, x: torch.Tensor) -> torch.Tensor:
if self._impl is not None:
return self._impl(x)
return F.rms_norm(x, (self.dim,), eps=self.eps)
class Linear(nn.Linear):
"""Keep fp32 master weights while running matmuls in activation dtype."""
def forward(self, x: torch.Tensor) -> torch.Tensor:
weight = self.weight.to(dtype=x.dtype)
bias = self.bias.to(dtype=x.dtype) if self.bias is not None else None
return F.linear(x, weight, bias)
def _rotate_half(x: torch.Tensor) -> torch.Tensor:
x1, x2 = x[..., : x.shape[-1] // 2], x[..., x.shape[-1] // 2 :]
return torch.cat((-x2, x1), dim=-1)
def _apply_rope(x: torch.Tensor, cos: torch.Tensor, sin: torch.Tensor) -> torch.Tensor:
return (x * cos) + (_rotate_half(x) * sin)
class CausalSelfAttention(nn.Module):
def __init__(self, config: ModelConfig):
super().__init__()
if config.n_head % config.n_kv_head:
raise ValueError("n_head must be divisible by n_kv_head")
self.n_head = config.n_head
self.n_kv_head = config.n_kv_head
self.head_dim = config.head_dim
self.dropout = config.dropout
self.backend = config.attention_backend
if self.backend == "auto":
self.backend = "flash_attn" if HAS_FLASH_ATTN else "sdpa"
self.qkv = Linear(config.n_embd, (config.n_head + 2 * config.n_kv_head) * self.head_dim, bias=False)
self.proj = Linear(config.n_embd, config.n_embd, bias=False)
self.q_norm = RMSNorm(self.head_dim, use_liger=config.use_liger) if config.use_qk_norm else nn.Identity()
self.k_norm = RMSNorm(self.head_dim, use_liger=config.use_liger) if config.use_qk_norm else nn.Identity()
def forward(self, x: torch.Tensor, cos: torch.Tensor, sin: torch.Tensor) -> torch.Tensor:
batch, length, _ = x.shape
qkv = self.qkv(x).view(batch, length, self.n_head + 2 * self.n_kv_head, self.head_dim)
q = qkv[:, :, : self.n_head]
k = qkv[:, :, self.n_head : self.n_head + self.n_kv_head]
v = qkv[:, :, self.n_head + self.n_kv_head :]
q = _apply_rope(self.q_norm(q), cos[:, :length], sin[:, :length])
k = _apply_rope(self.k_norm(k), cos[:, :length], sin[:, :length])
if self.n_kv_head != self.n_head:
repeats = self.n_head // self.n_kv_head
k = k.repeat_interleave(repeats, dim=2)
v = v.repeat_interleave(repeats, dim=2)
if self.backend == "flash_attn" and HAS_FLASH_ATTN and x.is_cuda and x.dtype in (torch.float16, torch.bfloat16):
y = _flash_attn_func(q, k, v, dropout_p=self.dropout if self.training else 0.0, causal=True)
else:
q_t = q.transpose(1, 2)
k_t = k.transpose(1, 2)
v_t = v.transpose(1, 2)
y = F.scaled_dot_product_attention(
q_t,
k_t,
v_t,
dropout_p=self.dropout if self.training else 0.0,
is_causal=True,
).transpose(1, 2)
return self.proj(y.reshape(batch, length, -1))
class MLP(nn.Module):
def __init__(self, config: ModelConfig):
super().__init__()
hidden = config.intermediate_size
self.fc = Linear(config.n_embd, hidden, bias=False)
self.proj = Linear(hidden, config.n_embd, bias=False)
def forward(self, x: torch.Tensor) -> torch.Tensor:
# ReLU-squared is the dense FNN used by nanochat and avoids a gated
# third matrix multiply while retaining a useful smooth positive gate.
return self.proj(F.relu(self.fc(x)).square())
class Block(nn.Module):
def __init__(self, config: ModelConfig):
super().__init__()
self.attn_norm = RMSNorm(config.n_embd, use_liger=config.use_liger)
self.attn = CausalSelfAttention(config)
self.mlp_norm = RMSNorm(config.n_embd, use_liger=config.use_liger)
self.mlp = MLP(config)
def forward(self, x: torch.Tensor, cos: torch.Tensor, sin: torch.Tensor) -> torch.Tensor:
x = x + self.attn(self.attn_norm(x), cos, sin)
x = x + self.mlp(self.mlp_norm(x))
return x
class GPT(nn.Module):
def __init__(self, config: ModelConfig):
super().__init__()
self.config = config
self.activation_checkpointing = False
self.token_embedding = nn.Embedding(config.vocab_size, config.n_embd)
self.blocks = nn.ModuleList([Block(config) for _ in range(config.n_layer)])
self.final_norm = RMSNorm(config.n_embd, use_liger=config.use_liger)
self.lm_head = Linear(config.n_embd, config.vocab_size, bias=False)
inv_freq = 1.0 / (config.rope_base ** (torch.arange(0, config.head_dim, 2).float() / config.head_dim))
positions = torch.arange(config.sequence_len).float()
freqs = torch.outer(positions, inv_freq)
emb = torch.cat((freqs, freqs), dim=-1)
self.register_buffer("rope_cos", emb.cos()[None, :, None, :], persistent=False)
self.register_buffer("rope_sin", emb.sin()[None, :, None, :], persistent=False)
self.apply(self._init_weights)
def _init_weights(self, module: nn.Module) -> None:
if isinstance(module, nn.Linear):
std = 0.02
if module is self.lm_head:
std = 0.02
nn.init.normal_(module.weight, mean=0.0, std=std)
elif isinstance(module, nn.Embedding):
nn.init.normal_(module.weight, mean=0.0, std=0.02)
def forward(
self,
input_ids: torch.Tensor,
targets: torch.Tensor | None = None,
loss_mask: torch.Tensor | None = None,
return_logits: bool = True,
):
if input_ids.shape[1] > self.config.sequence_len:
raise ValueError(f"sequence length {input_ids.shape[1]} exceeds {self.config.sequence_len}")
x = self.token_embedding(input_ids)
compute_dtype = getattr(torch, self.config.compute_dtype)
if x.dtype != compute_dtype:
x = x.to(compute_dtype)
cos = self.rope_cos.to(device=x.device, dtype=x.dtype)
sin = self.rope_sin.to(device=x.device, dtype=x.dtype)
for block in self.blocks:
if self.training and self.activation_checkpointing:
x = torch.utils.checkpoint.checkpoint(block, x, cos, sin, use_reentrant=False)
else:
x = block(x, cos, sin)
logits = self.lm_head(self.final_norm(x))
if targets is None:
return logits
if loss_mask is None:
loss = F.cross_entropy(logits.reshape(-1, logits.size(-1)).float(), targets.reshape(-1))
else:
if loss_mask.shape != targets.shape:
raise ValueError("loss_mask must have the same shape as targets")
# The SFT mask is binary. Convert masked positions to the native
# cross-entropy ignore index so PyTorch can use its fused mean
# reduction instead of materializing a loss value for every
# vocabulary row and then multiplying by the mask. This keeps the
# assistant-only objective identical while avoiding a substantial
# SFT throughput penalty.
masked_targets = targets.masked_fill(~loss_mask, -100)
loss = F.cross_entropy(
logits.reshape(-1, logits.size(-1)).float(),
masked_targets.reshape(-1),
ignore_index=-100,
reduction="sum",
)
loss = loss / loss_mask.sum().to(dtype=loss.dtype).clamp_min(1.0)
return (logits, loss) if return_logits else loss
def num_parameters(self, non_embedding: bool = False) -> int:
total = sum(p.numel() for p in self.parameters())
if non_embedding:
total -= self.token_embedding.weight.numel() + self.lm_head.weight.numel()
return total
def architecture_summary(config: ModelConfig) -> dict:
matrix_per_layer = (
(config.n_head + 2 * config.n_kv_head) * config.head_dim * config.n_embd
+ config.n_embd * config.n_embd
+ config.n_embd * config.intermediate_size
+ config.intermediate_size * config.n_embd
)
norm_parameters = (2 * config.n_layer + 1) * config.n_embd if config.use_liger else 0
parameter_count = 2 * config.vocab_size * config.n_embd + config.n_layer * matrix_per_layer + norm_parameters
return {
"parameters": parameter_count,
"non_embedding_parameters": parameter_count - 2 * config.vocab_size * config.n_embd,
"attention_backend": config.attention_backend,
"selected_attention_backend": (
"flash_attn" if config.attention_backend == "auto" and HAS_FLASH_ATTN else "sdpa"
),
"flash_attn_available": HAS_FLASH_ATTN,
"liger_available": HAS_LIGER,
"config": config.__dict__,
}