ChatGBA / modeling_gabgpt.py
gszauer's picture
ChatGBA: chat model, Game Boy Advance cartridge and source, Transformers port, reference.js
546acd4 verified
Raw History Blame Contribute Delete
6.87 kB
"""ChatGBA for transformers: AutoModelForCausalLM with trust_remote_code.
The same model as reference.js and the Game Boy Advance engine: byte tokens;
blocks with RMSNorm, canon convolutions, grouped-query attention with QK-norm
and interleaved RoPE (most blocks see only the last 64 bytes), and a SwiGLU
MLP; a tied embedding table.
There is no KV cache: every call runs the last contextLength bytes from
scratch. The model was fine-tuned on windows that start anywhere in a
conversation, so a long chat simply keeps its latest contextLength bytes, a
sliding context. At 4 million parameters, that is fast enough.
"""
import torch
import torch.nn.functional as F
from torch import nn
from transformers import GenerationMixin, PreTrainedModel
from transformers.modeling_outputs import CausalLMOutput
from .configuration_gabgpt import GabGPTConfig
RMS_EPSILON = 1e-5
def rope(x, base):
"""Rotate adjacent pairs (2i, 2i+1) by position x base^(-2i/head_dim)."""
length, head_dim = x.shape[-2], x.shape[-1]
inverse = base ** (-torch.arange(0, head_dim, 2, dtype=torch.float32, device=x.device) / head_dim)
angles = torch.arange(length, dtype=torch.float32, device=x.device)[:, None] * inverse
cos, sin = angles.cos().to(x.dtype), angles.sin().to(x.dtype)
x0, x1 = x[..., 0::2], x[..., 1::2]
return torch.stack([x0 * cos - x1 * sin, x0 * sin + x1 * cos], dim=-1).flatten(-2)
def causal_conv(x, weight):
"""out[t] = sum over j of weight[j] * x[t - j], per feature."""
out = weight[0] * x
for j in range(1, weight.shape[0]):
out = out + weight[j] * F.pad(x, (0, 0, j, 0))[:, : x.shape[1]]
return out
class GabGPTBlock(nn.Module):
def __init__(self, c, window):
super().__init__()
d, h, kv, hd = c.featureDim, c.numHeads, c.numKVHeads, c.headDim
self.heads, self.kv_heads, self.head_dim, self.rope_base, self.window = h, kv, hd, c.ropeBase, window
self.attention_norm = nn.Parameter(torch.ones(d))
self.conv_a = nn.Parameter(torch.zeros(c.convKernel, d))
self.wq = nn.Parameter(torch.zeros(d, h * hd))
self.wk = nn.Parameter(torch.zeros(d, kv * hd))
self.wv = nn.Parameter(torch.zeros(d, kv * hd))
self.wo = nn.Parameter(torch.zeros(h * hd, d))
self.q_norm = nn.Parameter(torch.ones(h, hd))
self.k_norm = nn.Parameter(torch.ones(kv, hd))
self.mlp_norm = nn.Parameter(torch.ones(d))
self.conv_c = nn.Parameter(torch.zeros(c.convKernel, d))
self.gate = nn.Parameter(torch.zeros(d, c.mlpHiddenDim))
self.up = nn.Parameter(torch.zeros(d, c.mlpHiddenDim))
self.down = nn.Parameter(torch.zeros(c.mlpHiddenDim, d))
def forward(self, x):
batch, length, d = x.shape
a = F.rms_norm(x, (d,), self.attention_norm, RMS_EPSILON)
a = a + causal_conv(a, self.conv_a)
q = (a @ self.wq).view(batch, length, self.heads, self.head_dim).transpose(1, 2)
k = (a @ self.wk).view(batch, length, self.kv_heads, self.head_dim).transpose(1, 2)
v = (a @ self.wv).view(batch, length, self.kv_heads, self.head_dim).transpose(1, 2)
q = F.rms_norm(q, (self.head_dim,), None, RMS_EPSILON) * self.q_norm[:, None, :]
k = F.rms_norm(k, (self.head_dim,), None, RMS_EPSILON) * self.k_norm[:, None, :]
q, k = rope(q, self.rope_base), rope(k, self.rope_base)
if self.kv_heads != self.heads:
group = self.heads // self.kv_heads
k, v = k.repeat_interleave(group, dim=1), v.repeat_interleave(group, dim=1)
if self.window and self.window < length:
i = torch.arange(length, device=x.device)
distance = i[:, None] - i[None, :]
mixed = F.scaled_dot_product_attention(q, k, v, attn_mask=(distance >= 0) & (distance < self.window),
scale=self.head_dim ** -0.5)
else:
mixed = F.scaled_dot_product_attention(q, k, v, is_causal=True, scale=self.head_dim ** -0.5)
x = x + mixed.transpose(1, 2).reshape(batch, length, -1) @ self.wo
m = F.rms_norm(x, (d,), self.mlp_norm, RMS_EPSILON)
m = m + causal_conv(m, self.conv_c)
return x + (F.silu(m @ self.gate) * (m @ self.up)) @ self.down
class GabGPTModel(nn.Module):
def __init__(self, c):
super().__init__()
self.c = c
self.token_embeddings = nn.Parameter(torch.zeros(c.vocabSize, c.featureDim))
self.blocks = nn.ModuleList(GabGPTBlock(c, w) for w in c.attentionWindows)
self.final_norm = nn.Parameter(torch.ones(c.featureDim))
def forward(self, ids):
c = self.c
x = self.token_embeddings[ids]
if c.loopCount:
blocks = list(self.blocks)
prelude = blocks[: c.numPreludeBlocks]
core = blocks[c.numPreludeBlocks : c.numPreludeBlocks + c.numCoreBlocks]
coda = blocks[c.numPreludeBlocks + c.numCoreBlocks :]
for block in prelude:
x = block(x)
injected = x
for loop in range(c.loopCount):
if loop > 0:
x = x + injected
for block in core:
x = block(x)
for block in coda:
x = block(x)
else:
for block in self.blocks:
x = block(x)
x = F.rms_norm(x, (c.featureDim,), self.final_norm, RMS_EPSILON)
return x @ self.token_embeddings.T # tied unembedding
class GabGPTForCausalLM(PreTrainedModel, GenerationMixin):
config_class = GabGPTConfig
base_model_prefix = "model"
_no_split_modules = ["GabGPTBlock"]
_supports_cache_class = False
_is_stateful = True # no cache to reorder: beam search is not supported
def __init__(self, config):
super().__init__(config)
self.model = GabGPTModel(config)
self.post_init()
def _init_weights(self, module):
pass # weights always come from the checkpoint
def get_input_embeddings(self):
return None
def forward(self, input_ids=None, labels=None, attention_mask=None, **kwargs):
"""input_ids [batch, length]: byte values. Only the last contextLength
bytes are used. attention_mask is ignored: pad on the left only if all
rows are the same length, or generate one row at a time."""
ids = input_ids[:, -self.config.contextLength :]
logits = self.model(ids)
loss = None
if labels is not None:
labels = labels[:, -ids.shape[1] :]
loss = F.cross_entropy(logits[:, :-1].flatten(0, 1).float(), labels[:, 1:].flatten(), ignore_index=-100)
return CausalLMOutput(loss=loss, logits=logits)
def prepare_inputs_for_generation(self, input_ids, **kwargs):
return {"input_ids": input_ids[:, -self.config.contextLength :]}