NoTokenLM-MicroGen / modeling_notokengen.py
omurberaisik's picture
Publish shared configuration/modeling code
6c298b4 verified
Raw History Blame Contribute Delete
11.4 kB
"""
modeling_notokengen.py
Shared HuggingFace-compatible model class for every version in the
NoTokenLM-MicroGen series (2.5 through 3.7). Wraps the original from-scratch
architectures (byte-level, RoPE, SwiGLU, tied embeddings) in a
transformers.PreTrainedModel so every checkpoint can be loaded the standard
way:
from transformers import AutoModelForCausalLM, AutoTokenizer
model = AutoModelForCausalLM.from_pretrained(
"omurberaisik/NoTokenLM-MicroGen",
subfolder="3.6",
trust_remote_code=True,
)
There is no tokenizer - input/output is raw UTF-8 bytes (vocab_size=256).
Use the `generate_bytes` helper on the model, or feed byte ids directly.
"""
import math
import torch
import torch.nn as nn
import torch.nn.functional as F
from transformers import PreTrainedModel
from transformers.modeling_outputs import CausalLMOutput
from .configuration_notokengen import NoTokenGenConfig
# ============================================================
# Building blocks
# ============================================================
class RMSNorm(nn.Module):
def __init__(self, dim, eps=1e-6):
super().__init__()
self.weight = nn.Parameter(torch.ones(dim))
self.eps = eps
def forward(self, x):
norm = x.norm(dim=-1, keepdim=True) * (x.shape[-1] ** -0.5)
return x / (norm + self.eps) * self.weight
# ---------- "tiny" family (used only by 2.5) ----------
class TinyAttention(nn.Module):
def __init__(self, d_model):
super().__init__()
self.d_model = d_model
self.qkv = nn.Linear(d_model, d_model * 3, bias=False)
self.proj = nn.Linear(d_model, d_model, bias=False)
def forward(self, x):
B, T, C = x.shape
q, k, v = self.qkv(x).chunk(3, dim=-1)
att = (q @ k.transpose(-2, -1)) / math.sqrt(C)
mask = torch.triu(torch.ones(T, T, device=x.device), diagonal=1).bool()
att = att.masked_fill(mask, float("-inf"))
att = att.softmax(dim=-1)
return self.proj(att @ v)
class TinyMLP(nn.Module):
def __init__(self, d_model, mult=1):
super().__init__()
hidden = d_model * mult
self.fc1 = nn.Linear(d_model, hidden, bias=False)
self.fc2 = nn.Linear(hidden, d_model, bias=False)
def forward(self, x):
return self.fc2(torch.relu(self.fc1(x)))
class TinyBackbone(nn.Module):
"""Used only by 2.5: single layer, single head, learned positional embedding."""
def __init__(self, vocab_size, d_model, max_len):
super().__init__()
self.tok_emb = nn.Embedding(vocab_size, d_model)
self.pos_emb = nn.Embedding(max_len, d_model)
self.norm1 = RMSNorm(d_model)
self.attn = TinyAttention(d_model)
self.norm2 = RMSNorm(d_model)
self.mlp = TinyMLP(d_model, mult=1)
self.norm_out = RMSNorm(d_model)
self.max_len = max_len
def forward(self, idx):
B, T = idx.shape
pos = torch.arange(T, device=idx.device)
x = self.tok_emb(idx) + self.pos_emb(pos)
x = x + self.attn(self.norm1(x))
x = x + self.mlp(self.norm2(x))
x = self.norm_out(x)
return x @ self.tok_emb.weight.T
# ---------- RoPE+SwiGLU family (used by 2.6 through 3.7) ----------
def precompute_rope(dim, max_len, base=10000.0):
inv_freq = 1.0 / (base ** (torch.arange(0, dim, 2).float() / dim))
t = torch.arange(max_len).float()
freqs = torch.outer(t, inv_freq)
return torch.cos(freqs), torch.sin(freqs)
def apply_rope(x, cos, sin):
x1, x2 = x[..., ::2], x[..., 1::2]
cos = cos[None, :, None, :]
sin = sin[None, :, None, :]
rx1 = x1 * cos - x2 * sin
rx2 = x1 * sin + x2 * cos
return torch.stack([rx1, rx2], dim=-1).flatten(-2)
class MultiHeadAttentionRoPE(nn.Module):
"""Manual (non-SDPA) attention - used by 2.6 only."""
def __init__(self, d_model, n_heads):
super().__init__()
assert d_model % n_heads == 0
self.n_heads = n_heads
self.head_dim = d_model // n_heads
self.qkv = nn.Linear(d_model, d_model * 3, bias=False)
self.proj = nn.Linear(d_model, d_model, bias=False)
def forward(self, x, cos, sin):
B, T, C = x.shape
qkv = self.qkv(x).view(B, T, 3, self.n_heads, self.head_dim)
q, k, v = qkv.unbind(2)
q = apply_rope(q, cos, sin)
k = apply_rope(k, cos, sin)
q, k, v = q.transpose(1, 2), k.transpose(1, 2), v.transpose(1, 2)
att = (q @ k.transpose(-2, -1)) / math.sqrt(self.head_dim)
mask = torch.triu(torch.ones(T, T, device=x.device), diagonal=1).bool()
att = att.masked_fill(mask, float("-inf"))
att = att.softmax(dim=-1)
out = (att @ v).transpose(1, 2).reshape(B, T, C)
return self.proj(out)
class FlashAttentionRoPE(nn.Module):
"""PyTorch native SDPA - used from 2.7 onward."""
def __init__(self, d_model, n_heads):
super().__init__()
assert d_model % n_heads == 0
self.n_heads = n_heads
self.head_dim = d_model // n_heads
self.qkv = nn.Linear(d_model, d_model * 3, bias=False)
self.proj = nn.Linear(d_model, d_model, bias=False)
def forward(self, x, cos, sin):
B, T, C = x.shape
qkv = self.qkv(x).view(B, T, 3, self.n_heads, self.head_dim)
q, k, v = qkv.unbind(2)
q = apply_rope(q, cos, sin)
k = apply_rope(k, cos, sin)
q, k, v = q.transpose(1, 2), k.transpose(1, 2), v.transpose(1, 2)
out = F.scaled_dot_product_attention(q, k, v, is_causal=True)
out = out.transpose(1, 2).reshape(B, T, C)
return self.proj(out)
class SwiGLU(nn.Module):
def __init__(self, d_model, hidden):
super().__init__()
self.gate = nn.Linear(d_model, hidden, bias=False)
self.up = nn.Linear(d_model, hidden, bias=False)
self.down = nn.Linear(hidden, d_model, bias=False)
def forward(self, x):
return self.down(F.silu(self.gate(x)) * self.up(x))
class Block(nn.Module):
def __init__(self, d_model, n_heads, mlp_hidden, attn_cls):
super().__init__()
self.norm1 = RMSNorm(d_model)
self.attn = attn_cls(d_model, n_heads)
self.norm2 = RMSNorm(d_model)
self.mlp = SwiGLU(d_model, mlp_hidden)
def forward(self, x, cos, sin):
x = x + self.attn(self.norm1(x), cos, sin)
x = x + self.mlp(self.norm2(x))
return x
ATTN_CLASSES = {
"manual": MultiHeadAttentionRoPE,
"flash": FlashAttentionRoPE,
}
class RoPEBackbone(nn.Module):
"""Shared skeleton for 2.6, 2.7, 3.5, 3.6, 3.7 - only dims and attn differ."""
def __init__(self, vocab_size, d_model, n_heads, n_layers, mlp_hidden, max_len, attn):
super().__init__()
attn_cls = ATTN_CLASSES[attn]
self.tok_emb = nn.Embedding(vocab_size, d_model)
nn.init.normal_(self.tok_emb.weight, mean=0.0, std=0.02)
self.blocks = nn.ModuleList(
[Block(d_model, n_heads, mlp_hidden, attn_cls) for _ in range(n_layers)]
)
self.norm_out = RMSNorm(d_model)
head_dim = d_model // n_heads
cos, sin = precompute_rope(head_dim, max_len)
self.register_buffer("rope_cos", cos, persistent=False)
self.register_buffer("rope_sin", sin, persistent=False)
def forward(self, idx):
B, T = idx.shape
x = self.tok_emb(idx)
cos, sin = self.rope_cos[:T], self.rope_sin[:T]
# Defensive guard: on some environments the non-persistent RoPE buffers
# have been observed to end up containing NaN despite being computed
# cleanly at __init__ time (likely due to post_init()/from_pretrained()
# touching buffers unexpectedly). If that happens, recompute them fresh
# on the fly rather than silently propagating NaN through every layer.
if torch.isnan(cos).any() or torch.isnan(sin).any():
head_dim = self.rope_cos.shape[-1] * 2
max_len = self.rope_cos.shape[0]
fresh_cos, fresh_sin = precompute_rope(head_dim, max_len)
fresh_cos, fresh_sin = fresh_cos.to(idx.device), fresh_sin.to(idx.device)
self.rope_cos.copy_(fresh_cos)
self.rope_sin.copy_(fresh_sin)
cos, sin = self.rope_cos[:T], self.rope_sin[:T]
for block in self.blocks:
x = block(x, cos, sin)
x = self.norm_out(x)
return x @ self.tok_emb.weight.T
# ============================================================
# HuggingFace-compatible wrapper
# ============================================================
class NoTokenGenForCausalLM(PreTrainedModel):
config_class = NoTokenGenConfig
def _init_weights(self, module):
# No-op on purpose: every checkpoint's real weights are loaded via
# from_pretrained's own state_dict loading, which happens AFTER __init__
# runs. HF's PreTrainedModel.post_init() calls self.apply(self._init_weights)
# if this method is left undefined, using its own default initialization
# scheme - on some environments this has been observed to also touch
# registered buffers (like our precomputed RoPE cos/sin tables), corrupting
# them into NaN before the real weights are ever loaded. Defining this as
# a no-op guarantees post_init() never touches any parameter or buffer.
pass
def __init__(self, config: NoTokenGenConfig):
super().__init__(config)
if config.kind == "tiny":
self.backbone = TinyBackbone(
vocab_size=config.vocab_size,
d_model=config.d_model,
max_len=config.max_len,
)
else:
self.backbone = RoPEBackbone(
vocab_size=config.vocab_size,
d_model=config.d_model,
n_heads=config.n_heads,
n_layers=config.n_layers,
mlp_hidden=config.mlp_hidden,
max_len=config.max_len,
attn=config.attn,
)
self.post_init()
def forward(self, input_ids, labels=None, **kwargs):
logits = self.backbone(input_ids)
loss = None
if labels is not None:
loss = F.cross_entropy(
logits.view(-1, logits.size(-1)), labels.view(-1)
)
return CausalLMOutput(loss=loss, logits=logits)
@torch.no_grad()
def generate_bytes(self, prompt: str, n_new_bytes: int = 200, temperature: float = 0.5) -> str:
"""Convenience helper: raw-byte generation without a tokenizer.
This series has no tokenizer - text goes in and comes out as UTF-8 bytes.
`generate_bytes` is the simplest way to use these models; `forward()` is
also available directly for anyone who wants to feed byte ids themselves.
"""
self.eval()
device = next(self.parameters()).device
max_len = self.config.max_len
ids = list(prompt.encode("utf-8"))
for _ in range(n_new_bytes):
context = ids[-max_len:]
x = torch.tensor([context], dtype=torch.long, device=device)
logits = self.forward(x).logits[0, -1]
probs = torch.softmax(logits / temperature, dim=-1)
next_id = torch.multinomial(probs, 1).item()
ids.append(next_id)
return bytes(ids).decode("utf-8", errors="replace")