youssK
/

Boris-2 / modeling_boris2.py
youssK's picture KlondikeDev's picture
Duplicate from opencerebral/Boris-2
c5cfc0e
Raw History Blame Contribute Delete
11.3 kB
"""Boris-2 for Hugging Face transformers (OpenCerebral).
A faithful re-implementation of the training model (model.py / ngram.py in the
training repo) in plain PyTorch, so it runs anywhere transformers does:
* 31 pre-norm blocks, d_model 512, 8 heads of 64, SwiGLU (inner 1664)
* Canon: a causal 3-tap depthwise convolution over the fused Q|K|V projection
* RoPE (theta 10,000, rotate-half layout), positions restart at each document
* an n-gram embedding table (exact bigram/trigram lookup, rank 64) added to the
residual stream before block 2
* tied input/output embeddings, fp32 residual stream
Document boundaries. Training packed several documents per row; attention, Canon
and the n-gram lookup were all masked at document boundaries and positions
restarted at 0. Here a new document starts at position 0, after every EOS when
`config.eos_resets_context` is set, and around padding (attention_mask == 0).
A plain single-document input behaves exactly like an ordinary causal LM.
No KV cache: generation recomputes the prefix every step. At 144M parameters
this is fast enough for sampling and evaluation.
"""
import math
import torch
import torch.nn as nn
import torch.nn.functional as F
import transformers
from transformers import PreTrainedModel
from transformers.modeling_outputs import CausalLMOutputWithPast
try:
from transformers import GenerationMixin
except ImportError: # very old transformers: PreTrainedModel already includes it
GenerationMixin = object
from .configuration_boris2 import Boris2Config
def _rmsnorm(h, w, eps):
f = h.float()
return f * torch.rsqrt(f.pow(2).mean(-1, keepdim=True) + eps) * w.float()
def _lin(x, layer):
"""Linear in the weight's dtype, result back in fp32 for the residual stream."""
return F.linear(x.to(layer.weight.dtype), layer.weight).float()
class Boris2RMSNorm(nn.Module):
def __init__(self, dim, eps):
super().__init__()
self.weight = nn.Parameter(torch.ones(dim))
self.eps = eps
def forward(self, h):
return _rmsnorm(h, self.weight, self.eps)
class Boris2Attention(nn.Module):
def __init__(self, cfg):
super().__init__()
self.n_head, self.n_kv = cfg.num_attention_heads, cfg.num_key_value_heads
self.hd = cfg.hidden_size // cfg.num_attention_heads
c = (self.n_head + 2 * self.n_kv) * self.hd
self.qkv_proj = nn.Linear(cfg.hidden_size, c, bias=False)
self.o_proj = nn.Linear(self.n_head * self.hd, cfg.hidden_size, bias=False)
# (C, 3): column 0 weights token t, column 1 t-1, column 2 t-2. Zero = identity.
self.canon = nn.Parameter(torch.zeros(c, cfg.canon_kernel))
def forward(self, x, m1, m2, cos, sin, mask):
B, T, _ = x.shape
qkv = _lin(x, self.qkv_proj)
w = self.canon.float()
s1 = F.pad(qkv, (0, 0, 1, 0))[:, :-1] * m1
s2 = F.pad(qkv, (0, 0, 2, 0))[:, :-2] * m2
qkv = qkv + qkv * w[:, 0] + s1 * w[:, 1] + s2 * w[:, 2]
qd, kd = self.n_head * self.hd, self.n_kv * self.hd
q = qkv[..., :qd].view(B, T, self.n_head, self.hd)
k = qkv[..., qd:qd + kd].view(B, T, self.n_kv, self.hd)
v = qkv[..., qd + kd:].view(B, T, self.n_kv, self.hd)
def rope(t):
a, b = t.chunk(2, dim=-1)
return torch.cat([a * cos - b * sin, a * sin + b * cos], -1)
dt = self.qkv_proj.weight.dtype
q, k, v = (t.transpose(1, 2).to(dt) for t in (rope(q), rope(k), v))
if self.n_kv != self.n_head:
k = k.repeat_interleave(self.n_head // self.n_kv, 1)
v = v.repeat_interleave(self.n_head // self.n_kv, 1)
o = F.scaled_dot_product_attention(q, k, v, attn_mask=mask)
return _lin(o.transpose(1, 2).reshape(B, T, self.n_head * self.hd), self.o_proj)
class Boris2MLP(nn.Module):
def __init__(self, cfg):
super().__init__()
self.gate_up_proj = nn.Linear(cfg.hidden_size, 2 * cfg.intermediate_size, bias=False)
self.down_proj = nn.Linear(cfg.intermediate_size, cfg.hidden_size, bias=False)
def forward(self, x):
g, u = _lin(x, self.gate_up_proj).chunk(2, dim=-1)
return _lin(F.silu(g) * u, self.down_proj)
class Boris2Block(nn.Module):
def __init__(self, cfg):
super().__init__()
self.input_layernorm = Boris2RMSNorm(cfg.hidden_size, cfg.rms_norm_eps)
self.self_attn = Boris2Attention(cfg)
self.post_attention_layernorm = Boris2RMSNorm(cfg.hidden_size, cfg.rms_norm_eps)
self.mlp = Boris2MLP(cfg)
def forward(self, h, m1, m2, cos, sin, mask):
h = h + self.self_attn(self.input_layernorm(h), m1, m2, cos, sin, mask)
return h + self.mlp(self.post_attention_layernorm(h))
class Boris2NGram(nn.Module):
"""Exact lookup on (t-1, t) and (t-2, t-1, t); trigram wins where both hit.
Keys are sorted int64 (bigram a*V+b, trigram (a*V+b)*V+c); a miss is the OOV row."""
def __init__(self, cfg):
super().__init__()
self.V, self.oov = cfg.vocab_size, cfg.ngram_oov_row
self.table = nn.Parameter(torch.zeros(cfg.ngram_rows, cfg.ngram_rank))
self.proj = nn.Parameter(torch.zeros(cfg.hidden_size, cfg.ngram_rank))
self.register_buffer("bigram_keys", torch.zeros(cfg.ngram_bigram_entries, dtype=torch.int64))
self.register_buffer("bigram_rows", torch.zeros(cfg.ngram_bigram_entries, dtype=torch.int64))
self.register_buffer("trigram_keys", torch.zeros(cfg.ngram_trigram_entries, dtype=torch.int64))
self.register_buffer("trigram_rows", torch.zeros(cfg.ngram_trigram_entries, dtype=torch.int64))
def _lookup(self, keys, rows, q):
i = torch.searchsorted(keys, q.clamp_min(0)).clamp_max(keys.numel() - 1)
return torch.where(keys[i] == q, rows[i], torch.full_like(q, self.oov))
def forward(self, idx, boundary):
V = self.V
i0 = idx.long()
i1, i2 = torch.roll(i0, 1, 1), torch.roll(i0, 2, 1)
ok1 = ~boundary
ok2 = ~(boundary | torch.roll(boundary, 1, 1))
ok1[:, 0] = False
ok2[:, :2] = False
none = torch.full_like(i0, -1)
bi = self._lookup(self.bigram_keys, self.bigram_rows, torch.where(ok1, i1 * V + i0, none))
tri = self._lookup(self.trigram_keys, self.trigram_rows,
torch.where(ok2, (i2 * V + i1) * V + i0, none))
row = torch.where(tri != self.oov, tri, bi)
vec = F.embedding(row, self.table)
return F.linear(vec.to(self.proj.dtype), self.proj).float()
class Boris2PreTrainedModel(PreTrainedModel):
config_class = Boris2Config
base_model_prefix = "model"
_no_split_modules = ["Boris2Block"]
_supports_cache_class = False
_supports_static_cache = False
def _init_weights(self, module):
if isinstance(module, (nn.Linear, nn.Embedding)):
nn.init.normal_(module.weight, std=0.02)
class Boris2Model(Boris2PreTrainedModel):
def __init__(self, config):
super().__init__(config)
self.embed_tokens = nn.Embedding(config.vocab_size, config.hidden_size)
self.layers = nn.ModuleList([Boris2Block(config) for _ in range(config.num_hidden_layers)])
self.norm = Boris2RMSNorm(config.hidden_size, config.rms_norm_eps)
self.ngram = Boris2NGram(config)
self.post_init()
def get_input_embeddings(self):
return self.embed_tokens
def set_input_embeddings(self, value):
self.embed_tokens = value
def _documents(self, input_ids, attention_mask):
"""boundary[b, t]: token t starts a new document. Returns boundary, positions."""
B, T = input_ids.shape
dev = input_ids.device
boundary = torch.zeros(B, T, dtype=torch.bool, device=dev)
boundary[:, 0] = True
if self.config.eos_resets_context and self.config.eos_token_id is not None:
boundary[:, 1:] |= input_ids[:, :-1] == self.config.eos_token_id
if attention_mask is not None:
pad = attention_mask == 0
boundary |= pad # each pad is its own segment
boundary[:, 1:] |= pad[:, :-1] # text after padding starts fresh
t = torch.arange(T, device=dev).expand(B, T)
start = torch.cummax(torch.where(boundary, t, torch.zeros_like(t)), dim=1).values
return boundary, t - start
def forward(self, input_ids, attention_mask=None):
B, T = input_ids.shape
boundary, pos = self._documents(input_ids, attention_mask)
seg = torch.cumsum(boundary.long(), 1)
causal = torch.ones(T, T, dtype=torch.bool, device=input_ids.device).tril()
mask = (causal[None] & (seg[:, :, None] == seg[:, None, :]))[:, None] # (B,1,T,T)
hd = self.config.hidden_size // self.config.num_attention_heads
inv = 1.0 / (self.config.rope_theta ** (torch.arange(0, hd, 2, device=input_ids.device).float() / hd))
f = pos.float()[..., None] * inv
cos, sin = f.cos()[:, :, None], f.sin()[:, :, None] # (B,T,1,hd/2)
m1 = (~boundary).float()[:, :, None]
m2 = (~(boundary | F.pad(boundary, (1, 0))[:, :-1])).float()[:, :, None]
h = self.embed_tokens(input_ids).float()
for i, blk in enumerate(self.layers):
if i == self.config.ngram_insert_after:
h = h + self.ngram(input_ids, boundary)
h = blk(h, m1, m2, cos, sin, mask)
return self.norm(h)
class Boris2ForCausalLM(Boris2PreTrainedModel, GenerationMixin):
# transformers 5 maps tied keys to their source; 4.x takes a list.
_tied_weights_keys = ({"lm_head.weight": "model.embed_tokens.weight"}
if int(transformers.__version__.split(".")[0]) >= 5
else ["lm_head.weight"])
def __init__(self, config):
super().__init__(config)
self.model = Boris2Model(config)
self.lm_head = nn.Linear(config.hidden_size, config.vocab_size, bias=False)
self.post_init()
def get_input_embeddings(self):
return self.model.embed_tokens
def set_input_embeddings(self, value):
self.model.embed_tokens = value
def get_output_embeddings(self):
return self.lm_head
def set_output_embeddings(self, value):
self.lm_head = value
def forward(self, input_ids=None, attention_mask=None, labels=None,
logits_to_keep=0, return_dict=True, **kwargs):
h = self.model(input_ids, attention_mask)
if logits_to_keep:
h = h[:, -logits_to_keep:]
logits = F.linear(h, self.lm_head.weight.float())
loss = None
if labels is not None:
loss = F.cross_entropy(logits[:, :-1].reshape(-1, logits.shape[-1]),
labels[:, 1:].reshape(-1), ignore_index=-100)
if not return_dict:
return (loss, logits) if loss is not None else (logits,)
return CausalLMOutputWithPast(loss=loss, logits=logits)
def prepare_inputs_for_generation(self, input_ids, attention_mask=None, **kwargs):
# No KV cache: always pass the whole sequence.
return {"input_ids": input_ids, "attention_mask": attention_mask}