Text Generation
Transformers
Safetensors
English
gabgpt
gift-of-gab
small-language-model
trained-from-scratch
byte-level
game-boy-advance
browser
conversational
custom_code
Instructions to use gszauer/ChatGBA with libraries, inference providers, notebooks, and local apps. Follow these links to get started.
- Libraries
- Transformers
How to use gszauer/ChatGBA with Transformers:
# Use a pipeline as a high-level helper from transformers import pipeline pipe = pipeline("text-generation", model="gszauer/ChatGBA", trust_remote_code=True) messages = [ {"role": "user", "content": "Who are you?"}, ] pipe(messages)# pip install -U transformers accelerate # Load model directly from transformers import AutoModelForCausalLM model = AutoModelForCausalLM.from_pretrained("gszauer/ChatGBA", trust_remote_code=True, device_map="auto") - Notebooks
- Google Colab
- Kaggle
- Local Apps Settings
- vLLM
How to use gszauer/ChatGBA with vLLM:
Install from pip and serve model
# Install vLLM from pip: pip install vllm # Start the vLLM server: vllm serve "gszauer/ChatGBA" # Call the server using curl (OpenAI-compatible API): curl -X POST "http://localhost:8000/v1/chat/completions" \ -H "Content-Type: application/json" \ --data '{ "model": "gszauer/ChatGBA", "messages": [ { "role": "user", "content": "What is the capital of France?" } ] }'Use Docker
docker model run hf.co/gszauer/ChatGBA
- SGLang
How to use gszauer/ChatGBA with SGLang:
Install from pip and serve model
# Install SGLang from pip: pip install sglang # Start the SGLang server: python3 -m sglang.launch_server \ --model-path "gszauer/ChatGBA" \ --host 0.0.0.0 \ --port 30000 # Call the server using curl (OpenAI-compatible API): curl -X POST "http://localhost:30000/v1/chat/completions" \ -H "Content-Type: application/json" \ --data '{ "model": "gszauer/ChatGBA", "messages": [ { "role": "user", "content": "What is the capital of France?" } ] }'Use Docker images
docker run --gpus all \ --shm-size 32g \ -p 30000:30000 \ -v ~/.cache/huggingface:/root/.cache/huggingface \ --env "HF_TOKEN=<secret>" \ --ipc=host \ lmsysorg/sglang:latest \ python3 -m sglang.launch_server \ --model-path "gszauer/ChatGBA" \ --host 0.0.0.0 \ --port 30000 # Call the server using curl (OpenAI-compatible API): curl -X POST "http://localhost:30000/v1/chat/completions" \ -H "Content-Type: application/json" \ --data '{ "model": "gszauer/ChatGBA", "messages": [ { "role": "user", "content": "What is the capital of France?" } ] }' - Docker Model Runner
How to use gszauer/ChatGBA with Docker Model Runner:
docker model run hf.co/gszauer/ChatGBA
Download modeling_gabgpt.py from gszauer/ChatGBA: direct link, hf CLI and curl.
- Browser
- Download file 6.87 kB
-
https://huggingface.co/gszauer/ChatGBA/resolve/main/modeling_gabgpt.py
- Command line
-
hf download hf://gszauer/ChatGBA/modeling_gabgpt.py
-
curl -L -o modeling_gabgpt.py https://huggingface.co/gszauer/ChatGBA/resolve/main/modeling_gabgpt.py
6.87 kB
| """ChatGBA for transformers: AutoModelForCausalLM with trust_remote_code. | |
| The same model as reference.js and the Game Boy Advance engine: byte tokens; | |
| blocks with RMSNorm, canon convolutions, grouped-query attention with QK-norm | |
| and interleaved RoPE (most blocks see only the last 64 bytes), and a SwiGLU | |
| MLP; a tied embedding table. | |
| There is no KV cache: every call runs the last contextLength bytes from | |
| scratch. The model was fine-tuned on windows that start anywhere in a | |
| conversation, so a long chat simply keeps its latest contextLength bytes, a | |
| sliding context. At 4 million parameters, that is fast enough. | |
| """ | |
| import torch | |
| import torch.nn.functional as F | |
| from torch import nn | |
| from transformers import GenerationMixin, PreTrainedModel | |
| from transformers.modeling_outputs import CausalLMOutput | |
| from .configuration_gabgpt import GabGPTConfig | |
| RMS_EPSILON = 1e-5 | |
| def rope(x, base): | |
| """Rotate adjacent pairs (2i, 2i+1) by position x base^(-2i/head_dim).""" | |
| length, head_dim = x.shape[-2], x.shape[-1] | |
| inverse = base ** (-torch.arange(0, head_dim, 2, dtype=torch.float32, device=x.device) / head_dim) | |
| angles = torch.arange(length, dtype=torch.float32, device=x.device)[:, None] * inverse | |
| cos, sin = angles.cos().to(x.dtype), angles.sin().to(x.dtype) | |
| x0, x1 = x[..., 0::2], x[..., 1::2] | |
| return torch.stack([x0 * cos - x1 * sin, x0 * sin + x1 * cos], dim=-1).flatten(-2) | |
| def causal_conv(x, weight): | |
| """out[t] = sum over j of weight[j] * x[t - j], per feature.""" | |
| out = weight[0] * x | |
| for j in range(1, weight.shape[0]): | |
| out = out + weight[j] * F.pad(x, (0, 0, j, 0))[:, : x.shape[1]] | |
| return out | |
| class GabGPTBlock(nn.Module): | |
| def __init__(self, c, window): | |
| super().__init__() | |
| d, h, kv, hd = c.featureDim, c.numHeads, c.numKVHeads, c.headDim | |
| self.heads, self.kv_heads, self.head_dim, self.rope_base, self.window = h, kv, hd, c.ropeBase, window | |
| self.attention_norm = nn.Parameter(torch.ones(d)) | |
| self.conv_a = nn.Parameter(torch.zeros(c.convKernel, d)) | |
| self.wq = nn.Parameter(torch.zeros(d, h * hd)) | |
| self.wk = nn.Parameter(torch.zeros(d, kv * hd)) | |
| self.wv = nn.Parameter(torch.zeros(d, kv * hd)) | |
| self.wo = nn.Parameter(torch.zeros(h * hd, d)) | |
| self.q_norm = nn.Parameter(torch.ones(h, hd)) | |
| self.k_norm = nn.Parameter(torch.ones(kv, hd)) | |
| self.mlp_norm = nn.Parameter(torch.ones(d)) | |
| self.conv_c = nn.Parameter(torch.zeros(c.convKernel, d)) | |
| self.gate = nn.Parameter(torch.zeros(d, c.mlpHiddenDim)) | |
| self.up = nn.Parameter(torch.zeros(d, c.mlpHiddenDim)) | |
| self.down = nn.Parameter(torch.zeros(c.mlpHiddenDim, d)) | |
| def forward(self, x): | |
| batch, length, d = x.shape | |
| a = F.rms_norm(x, (d,), self.attention_norm, RMS_EPSILON) | |
| a = a + causal_conv(a, self.conv_a) | |
| q = (a @ self.wq).view(batch, length, self.heads, self.head_dim).transpose(1, 2) | |
| k = (a @ self.wk).view(batch, length, self.kv_heads, self.head_dim).transpose(1, 2) | |
| v = (a @ self.wv).view(batch, length, self.kv_heads, self.head_dim).transpose(1, 2) | |
| q = F.rms_norm(q, (self.head_dim,), None, RMS_EPSILON) * self.q_norm[:, None, :] | |
| k = F.rms_norm(k, (self.head_dim,), None, RMS_EPSILON) * self.k_norm[:, None, :] | |
| q, k = rope(q, self.rope_base), rope(k, self.rope_base) | |
| if self.kv_heads != self.heads: | |
| group = self.heads // self.kv_heads | |
| k, v = k.repeat_interleave(group, dim=1), v.repeat_interleave(group, dim=1) | |
| if self.window and self.window < length: | |
| i = torch.arange(length, device=x.device) | |
| distance = i[:, None] - i[None, :] | |
| mixed = F.scaled_dot_product_attention(q, k, v, attn_mask=(distance >= 0) & (distance < self.window), | |
| scale=self.head_dim ** -0.5) | |
| else: | |
| mixed = F.scaled_dot_product_attention(q, k, v, is_causal=True, scale=self.head_dim ** -0.5) | |
| x = x + mixed.transpose(1, 2).reshape(batch, length, -1) @ self.wo | |
| m = F.rms_norm(x, (d,), self.mlp_norm, RMS_EPSILON) | |
| m = m + causal_conv(m, self.conv_c) | |
| return x + (F.silu(m @ self.gate) * (m @ self.up)) @ self.down | |
| class GabGPTModel(nn.Module): | |
| def __init__(self, c): | |
| super().__init__() | |
| self.c = c | |
| self.token_embeddings = nn.Parameter(torch.zeros(c.vocabSize, c.featureDim)) | |
| self.blocks = nn.ModuleList(GabGPTBlock(c, w) for w in c.attentionWindows) | |
| self.final_norm = nn.Parameter(torch.ones(c.featureDim)) | |
| def forward(self, ids): | |
| c = self.c | |
| x = self.token_embeddings[ids] | |
| if c.loopCount: | |
| blocks = list(self.blocks) | |
| prelude = blocks[: c.numPreludeBlocks] | |
| core = blocks[c.numPreludeBlocks : c.numPreludeBlocks + c.numCoreBlocks] | |
| coda = blocks[c.numPreludeBlocks + c.numCoreBlocks :] | |
| for block in prelude: | |
| x = block(x) | |
| injected = x | |
| for loop in range(c.loopCount): | |
| if loop > 0: | |
| x = x + injected | |
| for block in core: | |
| x = block(x) | |
| for block in coda: | |
| x = block(x) | |
| else: | |
| for block in self.blocks: | |
| x = block(x) | |
| x = F.rms_norm(x, (c.featureDim,), self.final_norm, RMS_EPSILON) | |
| return x @ self.token_embeddings.T # tied unembedding | |
| class GabGPTForCausalLM(PreTrainedModel, GenerationMixin): | |
| config_class = GabGPTConfig | |
| base_model_prefix = "model" | |
| _no_split_modules = ["GabGPTBlock"] | |
| _supports_cache_class = False | |
| _is_stateful = True # no cache to reorder: beam search is not supported | |
| def __init__(self, config): | |
| super().__init__(config) | |
| self.model = GabGPTModel(config) | |
| self.post_init() | |
| def _init_weights(self, module): | |
| pass # weights always come from the checkpoint | |
| def get_input_embeddings(self): | |
| return None | |
| def forward(self, input_ids=None, labels=None, attention_mask=None, **kwargs): | |
| """input_ids [batch, length]: byte values. Only the last contextLength | |
| bytes are used. attention_mask is ignored: pad on the left only if all | |
| rows are the same length, or generate one row at a time.""" | |
| ids = input_ids[:, -self.config.contextLength :] | |
| logits = self.model(ids) | |
| loss = None | |
| if labels is not None: | |
| labels = labels[:, -ids.shape[1] :] | |
| loss = F.cross_entropy(logits[:, :-1].flatten(0, 1).float(), labels[:, 1:].flatten(), ignore_index=-100) | |
| return CausalLMOutput(loss=loss, logits=logits) | |
| def prepare_inputs_for_generation(self, input_ids, **kwargs): | |
| return {"input_ids": input_ids[:, -self.config.contextLength :]} | |