"""Boris-2 for Hugging Face transformers (OpenCerebral). A faithful re-implementation of the training model (model.py / ngram.py in the training repo) in plain PyTorch, so it runs anywhere transformers does: * 31 pre-norm blocks, d_model 512, 8 heads of 64, SwiGLU (inner 1664) * Canon: a causal 3-tap depthwise convolution over the fused Q|K|V projection * RoPE (theta 10,000, rotate-half layout), positions restart at each document * an n-gram embedding table (exact bigram/trigram lookup, rank 64) added to the residual stream before block 2 * tied input/output embeddings, fp32 residual stream Document boundaries. Training packed several documents per row; attention, Canon and the n-gram lookup were all masked at document boundaries and positions restarted at 0. Here a new document starts at position 0, after every EOS when `config.eos_resets_context` is set, and around padding (attention_mask == 0). A plain single-document input behaves exactly like an ordinary causal LM. No KV cache: generation recomputes the prefix every step. At 144M parameters this is fast enough for sampling and evaluation. """ import math import torch import torch.nn as nn import torch.nn.functional as F import transformers from transformers import PreTrainedModel from transformers.modeling_outputs import CausalLMOutputWithPast try: from transformers import GenerationMixin except ImportError: # very old transformers: PreTrainedModel already includes it GenerationMixin = object from .configuration_boris2 import Boris2Config def _rmsnorm(h, w, eps): f = h.float() return f * torch.rsqrt(f.pow(2).mean(-1, keepdim=True) + eps) * w.float() def _lin(x, layer): """Linear in the weight's dtype, result back in fp32 for the residual stream.""" return F.linear(x.to(layer.weight.dtype), layer.weight).float() class Boris2RMSNorm(nn.Module): def __init__(self, dim, eps): super().__init__() self.weight = nn.Parameter(torch.ones(dim)) self.eps = eps def forward(self, h): return _rmsnorm(h, self.weight, self.eps) class Boris2Attention(nn.Module): def __init__(self, cfg): super().__init__() self.n_head, self.n_kv = cfg.num_attention_heads, cfg.num_key_value_heads self.hd = cfg.hidden_size // cfg.num_attention_heads c = (self.n_head + 2 * self.n_kv) * self.hd self.qkv_proj = nn.Linear(cfg.hidden_size, c, bias=False) self.o_proj = nn.Linear(self.n_head * self.hd, cfg.hidden_size, bias=False) # (C, 3): column 0 weights token t, column 1 t-1, column 2 t-2. Zero = identity. self.canon = nn.Parameter(torch.zeros(c, cfg.canon_kernel)) def forward(self, x, m1, m2, cos, sin, mask): B, T, _ = x.shape qkv = _lin(x, self.qkv_proj) w = self.canon.float() s1 = F.pad(qkv, (0, 0, 1, 0))[:, :-1] * m1 s2 = F.pad(qkv, (0, 0, 2, 0))[:, :-2] * m2 qkv = qkv + qkv * w[:, 0] + s1 * w[:, 1] + s2 * w[:, 2] qd, kd = self.n_head * self.hd, self.n_kv * self.hd q = qkv[..., :qd].view(B, T, self.n_head, self.hd) k = qkv[..., qd:qd + kd].view(B, T, self.n_kv, self.hd) v = qkv[..., qd + kd:].view(B, T, self.n_kv, self.hd) def rope(t): a, b = t.chunk(2, dim=-1) return torch.cat([a * cos - b * sin, a * sin + b * cos], -1) dt = self.qkv_proj.weight.dtype q, k, v = (t.transpose(1, 2).to(dt) for t in (rope(q), rope(k), v)) if self.n_kv != self.n_head: k = k.repeat_interleave(self.n_head // self.n_kv, 1) v = v.repeat_interleave(self.n_head // self.n_kv, 1) o = F.scaled_dot_product_attention(q, k, v, attn_mask=mask) return _lin(o.transpose(1, 2).reshape(B, T, self.n_head * self.hd), self.o_proj) class Boris2MLP(nn.Module): def __init__(self, cfg): super().__init__() self.gate_up_proj = nn.Linear(cfg.hidden_size, 2 * cfg.intermediate_size, bias=False) self.down_proj = nn.Linear(cfg.intermediate_size, cfg.hidden_size, bias=False) def forward(self, x): g, u = _lin(x, self.gate_up_proj).chunk(2, dim=-1) return _lin(F.silu(g) * u, self.down_proj) class Boris2Block(nn.Module): def __init__(self, cfg): super().__init__() self.input_layernorm = Boris2RMSNorm(cfg.hidden_size, cfg.rms_norm_eps) self.self_attn = Boris2Attention(cfg) self.post_attention_layernorm = Boris2RMSNorm(cfg.hidden_size, cfg.rms_norm_eps) self.mlp = Boris2MLP(cfg) def forward(self, h, m1, m2, cos, sin, mask): h = h + self.self_attn(self.input_layernorm(h), m1, m2, cos, sin, mask) return h + self.mlp(self.post_attention_layernorm(h)) class Boris2NGram(nn.Module): """Exact lookup on (t-1, t) and (t-2, t-1, t); trigram wins where both hit. Keys are sorted int64 (bigram a*V+b, trigram (a*V+b)*V+c); a miss is the OOV row.""" def __init__(self, cfg): super().__init__() self.V, self.oov = cfg.vocab_size, cfg.ngram_oov_row self.table = nn.Parameter(torch.zeros(cfg.ngram_rows, cfg.ngram_rank)) self.proj = nn.Parameter(torch.zeros(cfg.hidden_size, cfg.ngram_rank)) self.register_buffer("bigram_keys", torch.zeros(cfg.ngram_bigram_entries, dtype=torch.int64)) self.register_buffer("bigram_rows", torch.zeros(cfg.ngram_bigram_entries, dtype=torch.int64)) self.register_buffer("trigram_keys", torch.zeros(cfg.ngram_trigram_entries, dtype=torch.int64)) self.register_buffer("trigram_rows", torch.zeros(cfg.ngram_trigram_entries, dtype=torch.int64)) def _lookup(self, keys, rows, q): i = torch.searchsorted(keys, q.clamp_min(0)).clamp_max(keys.numel() - 1) return torch.where(keys[i] == q, rows[i], torch.full_like(q, self.oov)) def forward(self, idx, boundary): V = self.V i0 = idx.long() i1, i2 = torch.roll(i0, 1, 1), torch.roll(i0, 2, 1) ok1 = ~boundary ok2 = ~(boundary | torch.roll(boundary, 1, 1)) ok1[:, 0] = False ok2[:, :2] = False none = torch.full_like(i0, -1) bi = self._lookup(self.bigram_keys, self.bigram_rows, torch.where(ok1, i1 * V + i0, none)) tri = self._lookup(self.trigram_keys, self.trigram_rows, torch.where(ok2, (i2 * V + i1) * V + i0, none)) row = torch.where(tri != self.oov, tri, bi) vec = F.embedding(row, self.table) return F.linear(vec.to(self.proj.dtype), self.proj).float() class Boris2PreTrainedModel(PreTrainedModel): config_class = Boris2Config base_model_prefix = "model" _no_split_modules = ["Boris2Block"] _supports_cache_class = False _supports_static_cache = False def _init_weights(self, module): if isinstance(module, (nn.Linear, nn.Embedding)): nn.init.normal_(module.weight, std=0.02) class Boris2Model(Boris2PreTrainedModel): def __init__(self, config): super().__init__(config) self.embed_tokens = nn.Embedding(config.vocab_size, config.hidden_size) self.layers = nn.ModuleList([Boris2Block(config) for _ in range(config.num_hidden_layers)]) self.norm = Boris2RMSNorm(config.hidden_size, config.rms_norm_eps) self.ngram = Boris2NGram(config) self.post_init() def get_input_embeddings(self): return self.embed_tokens def set_input_embeddings(self, value): self.embed_tokens = value def _documents(self, input_ids, attention_mask): """boundary[b, t]: token t starts a new document. Returns boundary, positions.""" B, T = input_ids.shape dev = input_ids.device boundary = torch.zeros(B, T, dtype=torch.bool, device=dev) boundary[:, 0] = True if self.config.eos_resets_context and self.config.eos_token_id is not None: boundary[:, 1:] |= input_ids[:, :-1] == self.config.eos_token_id if attention_mask is not None: pad = attention_mask == 0 boundary |= pad # each pad is its own segment boundary[:, 1:] |= pad[:, :-1] # text after padding starts fresh t = torch.arange(T, device=dev).expand(B, T) start = torch.cummax(torch.where(boundary, t, torch.zeros_like(t)), dim=1).values return boundary, t - start def forward(self, input_ids, attention_mask=None): B, T = input_ids.shape boundary, pos = self._documents(input_ids, attention_mask) seg = torch.cumsum(boundary.long(), 1) causal = torch.ones(T, T, dtype=torch.bool, device=input_ids.device).tril() mask = (causal[None] & (seg[:, :, None] == seg[:, None, :]))[:, None] # (B,1,T,T) hd = self.config.hidden_size // self.config.num_attention_heads inv = 1.0 / (self.config.rope_theta ** (torch.arange(0, hd, 2, device=input_ids.device).float() / hd)) f = pos.float()[..., None] * inv cos, sin = f.cos()[:, :, None], f.sin()[:, :, None] # (B,T,1,hd/2) m1 = (~boundary).float()[:, :, None] m2 = (~(boundary | F.pad(boundary, (1, 0))[:, :-1])).float()[:, :, None] h = self.embed_tokens(input_ids).float() for i, blk in enumerate(self.layers): if i == self.config.ngram_insert_after: h = h + self.ngram(input_ids, boundary) h = blk(h, m1, m2, cos, sin, mask) return self.norm(h) class Boris2ForCausalLM(Boris2PreTrainedModel, GenerationMixin): # transformers 5 maps tied keys to their source; 4.x takes a list. _tied_weights_keys = ({"lm_head.weight": "model.embed_tokens.weight"} if int(transformers.__version__.split(".")[0]) >= 5 else ["lm_head.weight"]) def __init__(self, config): super().__init__(config) self.model = Boris2Model(config) self.lm_head = nn.Linear(config.hidden_size, config.vocab_size, bias=False) self.post_init() def get_input_embeddings(self): return self.model.embed_tokens def set_input_embeddings(self, value): self.model.embed_tokens = value def get_output_embeddings(self): return self.lm_head def set_output_embeddings(self, value): self.lm_head = value def forward(self, input_ids=None, attention_mask=None, labels=None, logits_to_keep=0, return_dict=True, **kwargs): h = self.model(input_ids, attention_mask) if logits_to_keep: h = h[:, -logits_to_keep:] logits = F.linear(h, self.lm_head.weight.float()) loss = None if labels is not None: loss = F.cross_entropy(logits[:, :-1].reshape(-1, logits.shape[-1]), labels[:, 1:].reshape(-1), ignore_index=-100) if not return_dict: return (loss, logits) if loss is not None else (logits,) return CausalLMOutputWithPast(loss=loss, logits=logits) def prepare_inputs_for_generation(self, input_ids, attention_mask=None, **kwargs): # No KV cache: always pass the whole sequence. return {"input_ids": input_ids, "attention_mask": attention_mask}