"""ChatGBA for transformers: AutoModelForCausalLM with trust_remote_code. The same model as reference.js and the Game Boy Advance engine: byte tokens; blocks with RMSNorm, canon convolutions, grouped-query attention with QK-norm and interleaved RoPE (most blocks see only the last 64 bytes), and a SwiGLU MLP; a tied embedding table. There is no KV cache: every call runs the last contextLength bytes from scratch. The model was fine-tuned on windows that start anywhere in a conversation, so a long chat simply keeps its latest contextLength bytes, a sliding context. At 4 million parameters, that is fast enough. """ import torch import torch.nn.functional as F from torch import nn from transformers import GenerationMixin, PreTrainedModel from transformers.modeling_outputs import CausalLMOutput from .configuration_gabgpt import GabGPTConfig RMS_EPSILON = 1e-5 def rope(x, base): """Rotate adjacent pairs (2i, 2i+1) by position x base^(-2i/head_dim).""" length, head_dim = x.shape[-2], x.shape[-1] inverse = base ** (-torch.arange(0, head_dim, 2, dtype=torch.float32, device=x.device) / head_dim) angles = torch.arange(length, dtype=torch.float32, device=x.device)[:, None] * inverse cos, sin = angles.cos().to(x.dtype), angles.sin().to(x.dtype) x0, x1 = x[..., 0::2], x[..., 1::2] return torch.stack([x0 * cos - x1 * sin, x0 * sin + x1 * cos], dim=-1).flatten(-2) def causal_conv(x, weight): """out[t] = sum over j of weight[j] * x[t - j], per feature.""" out = weight[0] * x for j in range(1, weight.shape[0]): out = out + weight[j] * F.pad(x, (0, 0, j, 0))[:, : x.shape[1]] return out class GabGPTBlock(nn.Module): def __init__(self, c, window): super().__init__() d, h, kv, hd = c.featureDim, c.numHeads, c.numKVHeads, c.headDim self.heads, self.kv_heads, self.head_dim, self.rope_base, self.window = h, kv, hd, c.ropeBase, window self.attention_norm = nn.Parameter(torch.ones(d)) self.conv_a = nn.Parameter(torch.zeros(c.convKernel, d)) self.wq = nn.Parameter(torch.zeros(d, h * hd)) self.wk = nn.Parameter(torch.zeros(d, kv * hd)) self.wv = nn.Parameter(torch.zeros(d, kv * hd)) self.wo = nn.Parameter(torch.zeros(h * hd, d)) self.q_norm = nn.Parameter(torch.ones(h, hd)) self.k_norm = nn.Parameter(torch.ones(kv, hd)) self.mlp_norm = nn.Parameter(torch.ones(d)) self.conv_c = nn.Parameter(torch.zeros(c.convKernel, d)) self.gate = nn.Parameter(torch.zeros(d, c.mlpHiddenDim)) self.up = nn.Parameter(torch.zeros(d, c.mlpHiddenDim)) self.down = nn.Parameter(torch.zeros(c.mlpHiddenDim, d)) def forward(self, x): batch, length, d = x.shape a = F.rms_norm(x, (d,), self.attention_norm, RMS_EPSILON) a = a + causal_conv(a, self.conv_a) q = (a @ self.wq).view(batch, length, self.heads, self.head_dim).transpose(1, 2) k = (a @ self.wk).view(batch, length, self.kv_heads, self.head_dim).transpose(1, 2) v = (a @ self.wv).view(batch, length, self.kv_heads, self.head_dim).transpose(1, 2) q = F.rms_norm(q, (self.head_dim,), None, RMS_EPSILON) * self.q_norm[:, None, :] k = F.rms_norm(k, (self.head_dim,), None, RMS_EPSILON) * self.k_norm[:, None, :] q, k = rope(q, self.rope_base), rope(k, self.rope_base) if self.kv_heads != self.heads: group = self.heads // self.kv_heads k, v = k.repeat_interleave(group, dim=1), v.repeat_interleave(group, dim=1) if self.window and self.window < length: i = torch.arange(length, device=x.device) distance = i[:, None] - i[None, :] mixed = F.scaled_dot_product_attention(q, k, v, attn_mask=(distance >= 0) & (distance < self.window), scale=self.head_dim ** -0.5) else: mixed = F.scaled_dot_product_attention(q, k, v, is_causal=True, scale=self.head_dim ** -0.5) x = x + mixed.transpose(1, 2).reshape(batch, length, -1) @ self.wo m = F.rms_norm(x, (d,), self.mlp_norm, RMS_EPSILON) m = m + causal_conv(m, self.conv_c) return x + (F.silu(m @ self.gate) * (m @ self.up)) @ self.down class GabGPTModel(nn.Module): def __init__(self, c): super().__init__() self.c = c self.token_embeddings = nn.Parameter(torch.zeros(c.vocabSize, c.featureDim)) self.blocks = nn.ModuleList(GabGPTBlock(c, w) for w in c.attentionWindows) self.final_norm = nn.Parameter(torch.ones(c.featureDim)) def forward(self, ids): c = self.c x = self.token_embeddings[ids] if c.loopCount: blocks = list(self.blocks) prelude = blocks[: c.numPreludeBlocks] core = blocks[c.numPreludeBlocks : c.numPreludeBlocks + c.numCoreBlocks] coda = blocks[c.numPreludeBlocks + c.numCoreBlocks :] for block in prelude: x = block(x) injected = x for loop in range(c.loopCount): if loop > 0: x = x + injected for block in core: x = block(x) for block in coda: x = block(x) else: for block in self.blocks: x = block(x) x = F.rms_norm(x, (c.featureDim,), self.final_norm, RMS_EPSILON) return x @ self.token_embeddings.T # tied unembedding class GabGPTForCausalLM(PreTrainedModel, GenerationMixin): config_class = GabGPTConfig base_model_prefix = "model" _no_split_modules = ["GabGPTBlock"] _supports_cache_class = False _is_stateful = True # no cache to reorder: beam search is not supported def __init__(self, config): super().__init__(config) self.model = GabGPTModel(config) self.post_init() def _init_weights(self, module): pass # weights always come from the checkpoint def get_input_embeddings(self): return None def forward(self, input_ids=None, labels=None, attention_mask=None, **kwargs): """input_ids [batch, length]: byte values. Only the last contextLength bytes are used. attention_mask is ignored: pad on the left only if all rows are the same length, or generate one row at a time.""" ids = input_ids[:, -self.config.contextLength :] logits = self.model(ids) loss = None if labels is not None: labels = labels[:, -ids.shape[1] :] loss = F.cross_entropy(logits[:, :-1].flatten(0, 1).float(), labels[:, 1:].flatten(), ignore_index=-100) return CausalLMOutput(loss=loss, logits=logits) def prepare_inputs_for_generation(self, input_ids, **kwargs): return {"input_ids": input_ids[:, -self.config.contextLength :]}