""" modeling_notokengen.py Shared HuggingFace-compatible model class for every version in the NoTokenLM-MicroGen series (2.5 through 3.7). Wraps the original from-scratch architectures (byte-level, RoPE, SwiGLU, tied embeddings) in a transformers.PreTrainedModel so every checkpoint can be loaded the standard way: from transformers import AutoModelForCausalLM, AutoTokenizer model = AutoModelForCausalLM.from_pretrained( "omurberaisik/NoTokenLM-MicroGen", subfolder="3.6", trust_remote_code=True, ) There is no tokenizer - input/output is raw UTF-8 bytes (vocab_size=256). Use the `generate_bytes` helper on the model, or feed byte ids directly. """ import math import torch import torch.nn as nn import torch.nn.functional as F from transformers import PreTrainedModel from transformers.modeling_outputs import CausalLMOutput from .configuration_notokengen import NoTokenGenConfig # ============================================================ # Building blocks # ============================================================ class RMSNorm(nn.Module): def __init__(self, dim, eps=1e-6): super().__init__() self.weight = nn.Parameter(torch.ones(dim)) self.eps = eps def forward(self, x): norm = x.norm(dim=-1, keepdim=True) * (x.shape[-1] ** -0.5) return x / (norm + self.eps) * self.weight # ---------- "tiny" family (used only by 2.5) ---------- class TinyAttention(nn.Module): def __init__(self, d_model): super().__init__() self.d_model = d_model self.qkv = nn.Linear(d_model, d_model * 3, bias=False) self.proj = nn.Linear(d_model, d_model, bias=False) def forward(self, x): B, T, C = x.shape q, k, v = self.qkv(x).chunk(3, dim=-1) att = (q @ k.transpose(-2, -1)) / math.sqrt(C) mask = torch.triu(torch.ones(T, T, device=x.device), diagonal=1).bool() att = att.masked_fill(mask, float("-inf")) att = att.softmax(dim=-1) return self.proj(att @ v) class TinyMLP(nn.Module): def __init__(self, d_model, mult=1): super().__init__() hidden = d_model * mult self.fc1 = nn.Linear(d_model, hidden, bias=False) self.fc2 = nn.Linear(hidden, d_model, bias=False) def forward(self, x): return self.fc2(torch.relu(self.fc1(x))) class TinyBackbone(nn.Module): """Used only by 2.5: single layer, single head, learned positional embedding.""" def __init__(self, vocab_size, d_model, max_len): super().__init__() self.tok_emb = nn.Embedding(vocab_size, d_model) self.pos_emb = nn.Embedding(max_len, d_model) self.norm1 = RMSNorm(d_model) self.attn = TinyAttention(d_model) self.norm2 = RMSNorm(d_model) self.mlp = TinyMLP(d_model, mult=1) self.norm_out = RMSNorm(d_model) self.max_len = max_len def forward(self, idx): B, T = idx.shape pos = torch.arange(T, device=idx.device) x = self.tok_emb(idx) + self.pos_emb(pos) x = x + self.attn(self.norm1(x)) x = x + self.mlp(self.norm2(x)) x = self.norm_out(x) return x @ self.tok_emb.weight.T # ---------- RoPE+SwiGLU family (used by 2.6 through 3.7) ---------- def precompute_rope(dim, max_len, base=10000.0): inv_freq = 1.0 / (base ** (torch.arange(0, dim, 2).float() / dim)) t = torch.arange(max_len).float() freqs = torch.outer(t, inv_freq) return torch.cos(freqs), torch.sin(freqs) def apply_rope(x, cos, sin): x1, x2 = x[..., ::2], x[..., 1::2] cos = cos[None, :, None, :] sin = sin[None, :, None, :] rx1 = x1 * cos - x2 * sin rx2 = x1 * sin + x2 * cos return torch.stack([rx1, rx2], dim=-1).flatten(-2) class MultiHeadAttentionRoPE(nn.Module): """Manual (non-SDPA) attention - used by 2.6 only.""" def __init__(self, d_model, n_heads): super().__init__() assert d_model % n_heads == 0 self.n_heads = n_heads self.head_dim = d_model // n_heads self.qkv = nn.Linear(d_model, d_model * 3, bias=False) self.proj = nn.Linear(d_model, d_model, bias=False) def forward(self, x, cos, sin): B, T, C = x.shape qkv = self.qkv(x).view(B, T, 3, self.n_heads, self.head_dim) q, k, v = qkv.unbind(2) q = apply_rope(q, cos, sin) k = apply_rope(k, cos, sin) q, k, v = q.transpose(1, 2), k.transpose(1, 2), v.transpose(1, 2) att = (q @ k.transpose(-2, -1)) / math.sqrt(self.head_dim) mask = torch.triu(torch.ones(T, T, device=x.device), diagonal=1).bool() att = att.masked_fill(mask, float("-inf")) att = att.softmax(dim=-1) out = (att @ v).transpose(1, 2).reshape(B, T, C) return self.proj(out) class FlashAttentionRoPE(nn.Module): """PyTorch native SDPA - used from 2.7 onward.""" def __init__(self, d_model, n_heads): super().__init__() assert d_model % n_heads == 0 self.n_heads = n_heads self.head_dim = d_model // n_heads self.qkv = nn.Linear(d_model, d_model * 3, bias=False) self.proj = nn.Linear(d_model, d_model, bias=False) def forward(self, x, cos, sin): B, T, C = x.shape qkv = self.qkv(x).view(B, T, 3, self.n_heads, self.head_dim) q, k, v = qkv.unbind(2) q = apply_rope(q, cos, sin) k = apply_rope(k, cos, sin) q, k, v = q.transpose(1, 2), k.transpose(1, 2), v.transpose(1, 2) out = F.scaled_dot_product_attention(q, k, v, is_causal=True) out = out.transpose(1, 2).reshape(B, T, C) return self.proj(out) class SwiGLU(nn.Module): def __init__(self, d_model, hidden): super().__init__() self.gate = nn.Linear(d_model, hidden, bias=False) self.up = nn.Linear(d_model, hidden, bias=False) self.down = nn.Linear(hidden, d_model, bias=False) def forward(self, x): return self.down(F.silu(self.gate(x)) * self.up(x)) class Block(nn.Module): def __init__(self, d_model, n_heads, mlp_hidden, attn_cls): super().__init__() self.norm1 = RMSNorm(d_model) self.attn = attn_cls(d_model, n_heads) self.norm2 = RMSNorm(d_model) self.mlp = SwiGLU(d_model, mlp_hidden) def forward(self, x, cos, sin): x = x + self.attn(self.norm1(x), cos, sin) x = x + self.mlp(self.norm2(x)) return x ATTN_CLASSES = { "manual": MultiHeadAttentionRoPE, "flash": FlashAttentionRoPE, } class RoPEBackbone(nn.Module): """Shared skeleton for 2.6, 2.7, 3.5, 3.6, 3.7 - only dims and attn differ.""" def __init__(self, vocab_size, d_model, n_heads, n_layers, mlp_hidden, max_len, attn): super().__init__() attn_cls = ATTN_CLASSES[attn] self.tok_emb = nn.Embedding(vocab_size, d_model) nn.init.normal_(self.tok_emb.weight, mean=0.0, std=0.02) self.blocks = nn.ModuleList( [Block(d_model, n_heads, mlp_hidden, attn_cls) for _ in range(n_layers)] ) self.norm_out = RMSNorm(d_model) head_dim = d_model // n_heads cos, sin = precompute_rope(head_dim, max_len) self.register_buffer("rope_cos", cos, persistent=False) self.register_buffer("rope_sin", sin, persistent=False) def forward(self, idx): B, T = idx.shape x = self.tok_emb(idx) cos, sin = self.rope_cos[:T], self.rope_sin[:T] # Defensive guard: on some environments the non-persistent RoPE buffers # have been observed to end up containing NaN despite being computed # cleanly at __init__ time (likely due to post_init()/from_pretrained() # touching buffers unexpectedly). If that happens, recompute them fresh # on the fly rather than silently propagating NaN through every layer. if torch.isnan(cos).any() or torch.isnan(sin).any(): head_dim = self.rope_cos.shape[-1] * 2 max_len = self.rope_cos.shape[0] fresh_cos, fresh_sin = precompute_rope(head_dim, max_len) fresh_cos, fresh_sin = fresh_cos.to(idx.device), fresh_sin.to(idx.device) self.rope_cos.copy_(fresh_cos) self.rope_sin.copy_(fresh_sin) cos, sin = self.rope_cos[:T], self.rope_sin[:T] for block in self.blocks: x = block(x, cos, sin) x = self.norm_out(x) return x @ self.tok_emb.weight.T # ============================================================ # HuggingFace-compatible wrapper # ============================================================ class NoTokenGenForCausalLM(PreTrainedModel): config_class = NoTokenGenConfig def _init_weights(self, module): # No-op on purpose: every checkpoint's real weights are loaded via # from_pretrained's own state_dict loading, which happens AFTER __init__ # runs. HF's PreTrainedModel.post_init() calls self.apply(self._init_weights) # if this method is left undefined, using its own default initialization # scheme - on some environments this has been observed to also touch # registered buffers (like our precomputed RoPE cos/sin tables), corrupting # them into NaN before the real weights are ever loaded. Defining this as # a no-op guarantees post_init() never touches any parameter or buffer. pass def __init__(self, config: NoTokenGenConfig): super().__init__(config) if config.kind == "tiny": self.backbone = TinyBackbone( vocab_size=config.vocab_size, d_model=config.d_model, max_len=config.max_len, ) else: self.backbone = RoPEBackbone( vocab_size=config.vocab_size, d_model=config.d_model, n_heads=config.n_heads, n_layers=config.n_layers, mlp_hidden=config.mlp_hidden, max_len=config.max_len, attn=config.attn, ) self.post_init() def forward(self, input_ids, labels=None, **kwargs): logits = self.backbone(input_ids) loss = None if labels is not None: loss = F.cross_entropy( logits.view(-1, logits.size(-1)), labels.view(-1) ) return CausalLMOutput(loss=loss, logits=logits) @torch.no_grad() def generate_bytes(self, prompt: str, n_new_bytes: int = 200, temperature: float = 0.5) -> str: """Convenience helper: raw-byte generation without a tokenizer. This series has no tokenizer - text goes in and comes out as UTF-8 bytes. `generate_bytes` is the simplest way to use these models; `forward()` is also available directly for anyone who wants to feed byte ids themselves. """ self.eval() device = next(self.parameters()).device max_len = self.config.max_len ids = list(prompt.encode("utf-8")) for _ in range(n_new_bytes): context = ids[-max_len:] x = torch.tensor([context], dtype=torch.long, device=device) logits = self.forward(x).logits[0, -1] probs = torch.softmax(logits / temperature, dim=-1) next_id = torch.multinomial(probs, 1).item() ids.append(next_id) return bytes(ids).decode("utf-8", errors="replace")