Text Generation
Transformers
Safetensors
English
gabgpt
gift-of-gab
small-language-model
trained-from-scratch
byte-level
game-boy-advance
browser
conversational
custom_code
Instructions to use gszauer/ChatGBA with libraries, inference providers, notebooks, and local apps. Follow these links to get started.
- Libraries
- Transformers
How to use gszauer/ChatGBA with Transformers:
# Use a pipeline as a high-level helper from transformers import pipeline pipe = pipeline("text-generation", model="gszauer/ChatGBA", trust_remote_code=True) messages = [ {"role": "user", "content": "Who are you?"}, ] pipe(messages)# pip install -U transformers accelerate # Load model directly from transformers import AutoModelForCausalLM model = AutoModelForCausalLM.from_pretrained("gszauer/ChatGBA", trust_remote_code=True, device_map="auto") - Notebooks
- Google Colab
- Kaggle
- Local Apps Settings
- vLLM
How to use gszauer/ChatGBA with vLLM:
Install from pip and serve model
# Install vLLM from pip: pip install vllm # Start the vLLM server: vllm serve "gszauer/ChatGBA" # Call the server using curl (OpenAI-compatible API): curl -X POST "http://localhost:8000/v1/chat/completions" \ -H "Content-Type: application/json" \ --data '{ "model": "gszauer/ChatGBA", "messages": [ { "role": "user", "content": "What is the capital of France?" } ] }'Use Docker
docker model run hf.co/gszauer/ChatGBA
- SGLang
How to use gszauer/ChatGBA with SGLang:
Install from pip and serve model
# Install SGLang from pip: pip install sglang # Start the SGLang server: python3 -m sglang.launch_server \ --model-path "gszauer/ChatGBA" \ --host 0.0.0.0 \ --port 30000 # Call the server using curl (OpenAI-compatible API): curl -X POST "http://localhost:30000/v1/chat/completions" \ -H "Content-Type: application/json" \ --data '{ "model": "gszauer/ChatGBA", "messages": [ { "role": "user", "content": "What is the capital of France?" } ] }'Use Docker images
docker run --gpus all \ --shm-size 32g \ -p 30000:30000 \ -v ~/.cache/huggingface:/root/.cache/huggingface \ --env "HF_TOKEN=<secret>" \ --ipc=host \ lmsysorg/sglang:latest \ python3 -m sglang.launch_server \ --model-path "gszauer/ChatGBA" \ --host 0.0.0.0 \ --port 30000 # Call the server using curl (OpenAI-compatible API): curl -X POST "http://localhost:30000/v1/chat/completions" \ -H "Content-Type: application/json" \ --data '{ "model": "gszauer/ChatGBA", "messages": [ { "role": "user", "content": "What is the capital of France?" } ] }' - Docker Model Runner
How to use gszauer/ChatGBA with Docker Model Runner:
docker model run hf.co/gszauer/ChatGBA
ChatGBA: chat model, Game Boy Advance cartridge and source, Transformers port, reference.js
Browse files- .gitattributes +2 -0
- ChatGBA.gba +3 -0
- README.md +147 -0
- chat.py +57 -0
- chat_template.jinja +1 -0
- chatgba-float32.gab +3 -0
- chatgba-int8.bin +3 -0
- config.json +46 -0
- configuration_gabgpt.py +31 -0
- generation_config.json +9 -0
- model.json +53 -0
- model.safetensors +3 -0
- modeling_gabgpt.py +151 -0
- reference.js +1570 -0
- source/Makefile +40 -0
- source/README.md +31 -0
- source/blob.s +6 -0
- source/chat.c +84 -0
- source/chat.h +23 -0
- source/crt0.s +53 -0
- source/dot.s +43 -0
- source/engine.c +492 -0
- source/engine.h +37 -0
- source/fix_header.py +25 -0
- source/font.h +103 -0
- source/font.py +64 -0
- source/gba.ld +53 -0
- source/gba_include/string.h +7 -0
- source/libc.c +36 -0
- source/main.c +581 -0
- source/model_config.h +17 -0
- special_tokens_map.json +1 -0
- tokenization_gabgpt.py +41 -0
- tokenizer_config.json +12 -0
.gitattributes
CHANGED
|
@@ -33,3 +33,5 @@ saved_model/**/* filter=lfs diff=lfs merge=lfs -text
|
|
| 33 |
*.zip filter=lfs diff=lfs merge=lfs -text
|
| 34 |
*.zst filter=lfs diff=lfs merge=lfs -text
|
| 35 |
*tfevents* filter=lfs diff=lfs merge=lfs -text
|
|
|
|
|
|
|
|
|
| 33 |
*.zip filter=lfs diff=lfs merge=lfs -text
|
| 34 |
*.zst filter=lfs diff=lfs merge=lfs -text
|
| 35 |
*tfevents* filter=lfs diff=lfs merge=lfs -text
|
| 36 |
+
ChatGBA.gba filter=lfs diff=lfs merge=lfs -text
|
| 37 |
+
chatgba-float32.gab filter=lfs diff=lfs merge=lfs -text
|
ChatGBA.gba
ADDED
|
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
version https://git-lfs.github.com/spec/v1
|
| 2 |
+
oid sha256:33d2f79d2743c53423cd59a623d9badf5f52dd12ab8ee225d238e5e46520d19f
|
| 3 |
+
size 4224500
|
README.md
ADDED
|
@@ -0,0 +1,147 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
---
|
| 2 |
+
license: mit
|
| 3 |
+
language:
|
| 4 |
+
- en
|
| 5 |
+
pipeline_tag: text-generation
|
| 6 |
+
library_name: transformers
|
| 7 |
+
tags:
|
| 8 |
+
- gift-of-gab
|
| 9 |
+
- small-language-model
|
| 10 |
+
- trained-from-scratch
|
| 11 |
+
- byte-level
|
| 12 |
+
- game-boy-advance
|
| 13 |
+
- browser
|
| 14 |
+
datasets:
|
| 15 |
+
- HuggingFaceTB/smollm-corpus
|
| 16 |
+
- HuggingFaceTB/smol-smoltalk
|
| 17 |
+
---
|
| 18 |
+
|
| 19 |
+
# ChatGBA
|
| 20 |
+
|
| 21 |
+
> Try it in your browser: [gabormakesgames.com/ChatGBA](https://gabormakesgames.com/ChatGBA/)
|
| 22 |
+
|
| 23 |
+
ChatGBA is a ~4 million parameter byte-level chat model trained from scratch, small enough to run on a Game Boy Advance. The cartridge (`ChatGBA.gba`) holds the whole model as int8 weights and runs it in integer arithmetic on the GBA's 16.78 MHz ARM7TDMI, with an on-screen keyboard. It is the console member of the Gab models ([Gab 10M](https://huggingface.co/gszauer/Gab10M), [Gab 100M](https://huggingface.co/gszauer/Gab100M), [Gab 1B](https://huggingface.co/gszauer/Gab1B)), with Gab 10M's block design shrunk to fit.
|
| 24 |
+
|
| 25 |
+
It is a decoder-only transformer with pre-normalization (RMSNorm), short causal convolutions ("canon layers"), grouped-query attention with QK-norm and rotary position embeddings, short attention windows in most blocks, a SwiGLU MLP, and tied input/output embeddings. There is no tokenizer to learn: every UTF-8 byte is a token. The canonical model definition is `reference.js` (included in this repo), a dependency-free JavaScript implementation that the training code is matched against.
|
| 26 |
+
|
| 27 |
+
It speaks English and holds short conversations.
|
| 28 |
+
|
| 29 |
+
| Setting | Value |
|
| 30 |
+
| --- | ---: |
|
| 31 |
+
| Parameters | 4,003,264 |
|
| 32 |
+
| Transformer blocks | 10 |
|
| 33 |
+
| Feature dimension | 192 |
|
| 34 |
+
| Attention heads | 6 query heads, 2 key/value heads (grouped-query) |
|
| 35 |
+
| Head dimension | 32 |
|
| 36 |
+
| Attention windows | 64 bytes in blocks 1–4 and 6–9, the whole context in blocks 5 and 10 |
|
| 37 |
+
| MLP hidden dimension | 512 (SwiGLU) |
|
| 38 |
+
| Canon layer kernel | 4 |
|
| 39 |
+
| Context length | 512 bytes (sliding) |
|
| 40 |
+
| Vocabulary size | 256 (UTF-8 bytes) |
|
| 41 |
+
| RoPE base | 10,000 (interleaved pairs) |
|
| 42 |
+
| RMSNorm epsilon | 1e-5 |
|
| 43 |
+
| Precision | float32; int8 on the Game Boy Advance |
|
| 44 |
+
|
| 45 |
+
## Usage
|
| 46 |
+
|
| 47 |
+
On a Game Boy Advance: load `ChatGBA.gba` in an emulator (mGBA works) or on a flash cartridge. At real Game Boy speed it writes about one letter every 2 seconds.
|
| 48 |
+
|
| 49 |
+
In the browser: [gabormakesgames.com/ChatGBA](https://gabormakesgames.com/ChatGBA/) runs the cartridge in an emulator, and a JavaScript version of the model at full speed.
|
| 50 |
+
|
| 51 |
+
With Transformers:
|
| 52 |
+
|
| 53 |
+
```python
|
| 54 |
+
from transformers import AutoModelForCausalLM, AutoTokenizer
|
| 55 |
+
|
| 56 |
+
repo = "gszauer/ChatGBA"
|
| 57 |
+
tokenizer = AutoTokenizer.from_pretrained(repo, trust_remote_code=True)
|
| 58 |
+
model = AutoModelForCausalLM.from_pretrained(repo, trust_remote_code=True)
|
| 59 |
+
|
| 60 |
+
messages = [{"role": "user", "content": "Why is the sky blue?"}]
|
| 61 |
+
inputs = tokenizer.apply_chat_template(
|
| 62 |
+
messages,
|
| 63 |
+
add_generation_prompt=True,
|
| 64 |
+
return_tensors="pt",
|
| 65 |
+
return_dict=True,
|
| 66 |
+
)
|
| 67 |
+
output = model.generate(
|
| 68 |
+
inputs["input_ids"],
|
| 69 |
+
do_sample=True,
|
| 70 |
+
temperature=0.7,
|
| 71 |
+
top_k=40,
|
| 72 |
+
max_new_tokens=400, # bytes
|
| 73 |
+
stop_strings=["<|e|>"],
|
| 74 |
+
tokenizer=tokenizer,
|
| 75 |
+
)
|
| 76 |
+
print(tokenizer.decode(output[0][inputs["input_ids"].shape[1]:]).split("<|e|>")[0])
|
| 77 |
+
```
|
| 78 |
+
|
| 79 |
+
Or chat in the terminal: `python chat.py gszauer/ChatGBA`.
|
| 80 |
+
|
| 81 |
+
Tokens are bytes, so `max_new_tokens` counts bytes. The Transformers port is a simple, faithful implementation that recomputes the last 512 bytes each generation step (no KV cache), fine for a 4M model. `reference.js` and the Game Boy Advance engine implement KV-cached generation.
|
| 82 |
+
|
| 83 |
+
## Files
|
| 84 |
+
|
| 85 |
+
| File | What it is |
|
| 86 |
+
| --- | --- |
|
| 87 |
+
| `ChatGBA.gba` | the Game Boy Advance cartridge: the program and the int8 weights (4.03 MiB) |
|
| 88 |
+
| `chatgba-int8.bin` | the int8 weights the cartridge carries: every matrix as int8 with one scale per output row, norm scales and canon filters in Q16, and lookup tables for RoPE, softmax and SiLU |
|
| 89 |
+
| `chatgba-float32.gab` | the full float32 weights in the native format: a `GAB_` file (a 12-byte preamble, a JSON header with the shape and chat markers, then the weights in `reference.js` `parameters()` order), loadable via `GabGPT.fromArrayBuffer` |
|
| 90 |
+
| `model.safetensors` | the same float32 weights for Transformers |
|
| 91 |
+
| `model.json` | the shape and the weight order, as every engine reads it |
|
| 92 |
+
| `reference.js` | the canonical implementation: tensor library, autograd, model, KV cache and trainer, in dependency-free JavaScript |
|
| 93 |
+
| `source/` | the cartridge's source: the integer inference engine (C, with an ARM assembly inner loop), the chat program and its build (`make` rebuilds `ChatGBA.gba` byte for byte) |
|
| 94 |
+
| `modeling_gabgpt.py` etc. | the Transformers port (`trust_remote_code`) |
|
| 95 |
+
| `chat.py` | a terminal chat through Transformers |
|
| 96 |
+
|
| 97 |
+
## Architecture notes
|
| 98 |
+
|
| 99 |
+
**Grouped-query attention.** Six query heads of 32 share two key/value heads, three query heads to each, so the cache that remembers the conversation is a third the size. Queries and keys get an RMSNorm each (QK-norm) before RoPE. RoPE uses interleaved pairs: adjacent channels `(2i, 2i+1)` rotate together with `freq_i = 1/10000^(2i/32)`. There is no position embedding table, so the context can slide: past 512 bytes the oldest byte falls out and nothing else changes.
|
| 100 |
+
|
| 101 |
+
**Short windows.** Eight of the ten blocks look back only 64 bytes; blocks 5 and 10 see the whole context. Local blocks are cheap and their cache is small; the two global ones carry the conversation.
|
| 102 |
+
|
| 103 |
+
**Canon layers.** Before attention and before the MLP, each block mixes every position with the three before it, a 4-tap causal convolution per feature, added back as a residual: `a = a + sum_j w_j * a[t - j]`. Bytes are tiny tokens, and this lets every position see its neighbours for 768 parameters a block.
|
| 104 |
+
|
| 105 |
+
**The MLP is SwiGLU.** `MLP(x) = (SiLU(x @ Wgate) * (x @ Wup)) @ Wdown`, 192 -> 512 -> 192, no biases.
|
| 106 |
+
|
| 107 |
+
**The unembedding is tied.** `logits = RMSNorm(h) @ E.T`, with no separate output head.
|
| 108 |
+
|
| 109 |
+
**On the Game Boy Advance.** The GBA has no floating point, no divide instruction, 32 KB of fast RAM and 256 KB of slow RAM. The weights are read straight from cartridge ROM as int8; activations are 32-bit fixed point with 16 fractional bits; exp, sigmoid and RoPE's angles come from lookup tables; and the key/value cache is int8 in rings (64 positions for the windowed blocks, 512 for the global ones), 192 KB of the 256 KB. Quantization costs about 0.001 bits per byte, and the cartridge in an emulator gives byte-for-byte the same replies as the same integer engine compiled for a desktop. Each byte takes about 37 million cycles: 2.2 seconds.
|
| 110 |
+
|
| 111 |
+
## Chat format
|
| 112 |
+
|
| 113 |
+
There is no system role. Turns are concatenated directly, with short markers:
|
| 114 |
+
|
| 115 |
+
```text
|
| 116 |
+
<|u|>QUESTION<|e|><|a|>ANSWER<|e|>
|
| 117 |
+
```
|
| 118 |
+
|
| 119 |
+
A reply ends when the model writes `<|e|>`. The context slides: in a long conversation the model sees the latest 512 bytes, which it was trained for (fine-tuning windows start anywhere in a conversation).
|
| 120 |
+
|
| 121 |
+
## Training
|
| 122 |
+
|
| 123 |
+
Pre-trained on all of Cosmopedia v2 ([smollm-corpus](https://huggingface.co/datasets/HuggingFaceTB/smollm-corpus)), one pass: 147,277,920,173 bytes, 2,247,282 steps of 128 windows of 512 bytes, on one NVIDIA RTX PRO 6000 (about 94 hours). 1 document in 10,000 was held out for validation and another for test.
|
| 124 |
+
|
| 125 |
+
Then fine-tuned into a chat model on [smol-smoltalk](https://huggingface.co/datasets/HuggingFaceTB/smol-smoltalk): 54,201 steps of 128 windows of 512 bytes, the windows starting every 256 bytes so that every conversation is also seen cut off at the top (about 2.3 hours), learning only from assistant replies.
|
| 126 |
+
|
| 127 |
+
Muon for the attention and MLP matrices, AdamW (betas 0.9/0.95, weight decay 0.1) for the rest, global-norm gradient clipping at 1.0. The learning rate warms up over 0.1% of the steps, stays constant, and decays linearly to zero over the last 10% (warmup-stable-decay): peak 1e-3 for pre-training and 2e-4 for fine-tuning, both chosen by pilot runs. FP32 weights with TF32 matrix products, seed 20261027.
|
| 128 |
+
|
| 129 |
+
Results, in bits per byte (lower is better; the chat test grades assistant bytes only):
|
| 130 |
+
|
| 131 |
+
| | Cosmopedia test | smol-smoltalk test |
|
| 132 |
+
| --- | ---: | ---: |
|
| 133 |
+
| End of pre-training | 0.901 | |
|
| 134 |
+
| ChatGBA (this model, fine-tuned) | 1.217 | 0.824 |
|
| 135 |
+
|
| 136 |
+
Fine-tuning on chat alone costs plain-text skill, as the Cosmopedia column shows.
|
| 137 |
+
|
| 138 |
+
## Limitations
|
| 139 |
+
|
| 140 |
+
A 4M-parameter model. It writes fluent-looking English and follows the chat format, but it knows very little and makes things up; expect factual errors and quirks. It has no safety tuning beyond what SmolTalk's replies teach.
|
| 141 |
+
|
| 142 |
+
Game Boy Advance is a trademark of Nintendo. ChatGBA is not affiliated with or endorsed by Nintendo.
|
| 143 |
+
|
| 144 |
+
## Links
|
| 145 |
+
|
| 146 |
+
- Try it out: [gabormakesgames.com/ChatGBA](https://gabormakesgames.com/ChatGBA/)
|
| 147 |
+
- Blog/site: [gabormakesgames.com](https://gabormakesgames.com)
|
chat.py
ADDED
|
@@ -0,0 +1,57 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""Chat with ChatGBA in the terminal, through transformers.
|
| 2 |
+
|
| 3 |
+
pip install torch transformers safetensors
|
| 4 |
+
python chat.py [MODEL] [--temperature 0.7] [--max-bytes 400] [--seed N]
|
| 5 |
+
|
| 6 |
+
MODEL is the repository's name on the Hub (gszauer/ChatGBA) or a local copy
|
| 7 |
+
of it (default: the folder this script is in). An empty line quits; /reset
|
| 8 |
+
starts over.
|
| 9 |
+
|
| 10 |
+
The model sees the conversation as plain bytes:
|
| 11 |
+
<|u|>question<|e|><|a|>reply<|e|><|u|>...
|
| 12 |
+
and its reply ends when it writes <|e|>. Its context slides: past
|
| 13 |
+
contextLength bytes, it keeps only the latest ones.
|
| 14 |
+
"""
|
| 15 |
+
import argparse
|
| 16 |
+
from pathlib import Path
|
| 17 |
+
|
| 18 |
+
import torch
|
| 19 |
+
from transformers import AutoModelForCausalLM, AutoTokenizer
|
| 20 |
+
|
| 21 |
+
parser = argparse.ArgumentParser()
|
| 22 |
+
parser.add_argument("model", nargs="?", default=str(Path(__file__).resolve().parent))
|
| 23 |
+
parser.add_argument("--temperature", type=float, default=0.7)
|
| 24 |
+
parser.add_argument("--top-k", type=int, default=40)
|
| 25 |
+
parser.add_argument("--max-bytes", type=int, default=400)
|
| 26 |
+
parser.add_argument("--seed", type=int)
|
| 27 |
+
args = parser.parse_args()
|
| 28 |
+
|
| 29 |
+
if args.seed is not None:
|
| 30 |
+
torch.manual_seed(args.seed)
|
| 31 |
+
tokenizer = AutoTokenizer.from_pretrained(args.model, trust_remote_code=True)
|
| 32 |
+
model = AutoModelForCausalLM.from_pretrained(args.model, trust_remote_code=True).eval()
|
| 33 |
+
end = model.config.chat["end"]
|
| 34 |
+
print(f"ChatGBA: {sum(p.numel() for p in model.parameters()):,} parameters, "
|
| 35 |
+
f"{model.config.contextLength:,}-byte context")
|
| 36 |
+
|
| 37 |
+
messages = []
|
| 38 |
+
while True:
|
| 39 |
+
try:
|
| 40 |
+
text = input("\nyou> ")
|
| 41 |
+
except EOFError:
|
| 42 |
+
break
|
| 43 |
+
if not text:
|
| 44 |
+
break
|
| 45 |
+
if text == "/reset":
|
| 46 |
+
messages = []
|
| 47 |
+
continue
|
| 48 |
+
messages.append({"role": "user", "content": text})
|
| 49 |
+
prompt = tokenizer.apply_chat_template(messages, add_generation_prompt=True, return_tensors="pt", return_dict=True)
|
| 50 |
+
ids = prompt["input_ids"]
|
| 51 |
+
with torch.no_grad():
|
| 52 |
+
out = model.generate(ids, do_sample=args.temperature > 0, temperature=args.temperature or None, top_k=args.top_k,
|
| 53 |
+
max_new_tokens=args.max_bytes, stop_strings=[end], tokenizer=tokenizer)
|
| 54 |
+
reply = tokenizer.decode(out[0, ids.shape[1] :])
|
| 55 |
+
reply = reply.split(end)[0]
|
| 56 |
+
print(f"model> {reply}")
|
| 57 |
+
messages.append({"role": "assistant", "content": reply})
|
chat_template.jinja
ADDED
|
@@ -0,0 +1 @@
|
|
|
|
|
|
|
| 1 |
+
{% for message in messages %}{% if message['role'] == 'user' %}<|u|>{{ message['content'] }}<|e|>{% elif message['role'] == 'assistant' %}<|a|>{{ message['content'] }}<|e|>{% endif %}{% endfor %}{% if add_generation_prompt %}<|a|>{% endif %}
|
chatgba-float32.gab
ADDED
|
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
version https://git-lfs.github.com/spec/v1
|
| 2 |
+
oid sha256:461ead45c8ef2a332ac5d6868d3abf2043de275c3f9053dd37439bb532d26c36
|
| 3 |
+
size 16013432
|
chatgba-int8.bin
ADDED
|
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
version https://git-lfs.github.com/spec/v1
|
| 2 |
+
oid sha256:dc0b7703ea8b2aea9d37dc90b83e8bd8780f4cf04882408c7d36bb0c6ff4bebf
|
| 3 |
+
size 4207968
|
config.json
ADDED
|
@@ -0,0 +1,46 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{
|
| 2 |
+
"architecture": "chatgba",
|
| 3 |
+
"vocabSize": 256,
|
| 4 |
+
"featureDim": 192,
|
| 5 |
+
"numHeads": 6,
|
| 6 |
+
"numKVHeads": 2,
|
| 7 |
+
"headDim": 32,
|
| 8 |
+
"ropeBase": 10000,
|
| 9 |
+
"numBlocks": 10,
|
| 10 |
+
"mlpHiddenDim": 512,
|
| 11 |
+
"convKernel": 4,
|
| 12 |
+
"contextLength": 512,
|
| 13 |
+
"attentionWindows": [
|
| 14 |
+
64,
|
| 15 |
+
64,
|
| 16 |
+
64,
|
| 17 |
+
64,
|
| 18 |
+
0,
|
| 19 |
+
64,
|
| 20 |
+
64,
|
| 21 |
+
64,
|
| 22 |
+
64,
|
| 23 |
+
0
|
| 24 |
+
],
|
| 25 |
+
"chat": {
|
| 26 |
+
"user": "<|u|>",
|
| 27 |
+
"assistant": "<|a|>",
|
| 28 |
+
"end": "<|e|>",
|
| 29 |
+
"endOfText": "<|endoftext|>"
|
| 30 |
+
},
|
| 31 |
+
"model_type": "gabgpt",
|
| 32 |
+
"architectures": [
|
| 33 |
+
"GabGPTForCausalLM"
|
| 34 |
+
],
|
| 35 |
+
"auto_map": {
|
| 36 |
+
"AutoConfig": "configuration_gabgpt.GabGPTConfig",
|
| 37 |
+
"AutoModelForCausalLM": "modeling_gabgpt.GabGPTForCausalLM"
|
| 38 |
+
},
|
| 39 |
+
"torch_dtype": "float32",
|
| 40 |
+
"parameterCount": 4003264,
|
| 41 |
+
"trainedFrom": {
|
| 42 |
+
"stage": "finetune",
|
| 43 |
+
"step": 54201,
|
| 44 |
+
"seed": 20261027
|
| 45 |
+
}
|
| 46 |
+
}
|
configuration_gabgpt.py
ADDED
|
@@ -0,0 +1,31 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""The ChatGBA configuration for transformers: model.json's fields, as is."""
|
| 2 |
+
from transformers import PretrainedConfig
|
| 3 |
+
|
| 4 |
+
|
| 5 |
+
class GabGPTConfig(PretrainedConfig):
|
| 6 |
+
"""ChatGBA's shape. Bytes are tokens: vocabSize is 256, and the chat markers
|
| 7 |
+
(chat) are spelled out as plain text. numKVHeads key/value heads serve
|
| 8 |
+
numHeads query heads (grouped-query attention); attentionWindows gives
|
| 9 |
+
each block's window (0 = the whole context)."""
|
| 10 |
+
|
| 11 |
+
model_type = "gabgpt"
|
| 12 |
+
|
| 13 |
+
def __init__(self, architecture="chatgba", vocabSize=256, featureDim=192, numHeads=6, numKVHeads=2,
|
| 14 |
+
headDim=32, ropeBase=10000, numBlocks=10, numPreludeBlocks=None, numCoreBlocks=None,
|
| 15 |
+
numCodaBlocks=None, loopCount=None, mlpHiddenDim=512, convKernel=4, contextLength=512,
|
| 16 |
+
attentionWindows=None, chat=None, **kwargs):
|
| 17 |
+
self.architecture = architecture
|
| 18 |
+
self.vocabSize, self.featureDim, self.numHeads = vocabSize, featureDim, numHeads
|
| 19 |
+
self.numKVHeads = numKVHeads or numHeads
|
| 20 |
+
self.headDim = headDim or featureDim // numHeads
|
| 21 |
+
self.ropeBase, self.mlpHiddenDim, self.convKernel = ropeBase, mlpHiddenDim, convKernel
|
| 22 |
+
self.numBlocks, self.loopCount = numBlocks, loopCount
|
| 23 |
+
self.numPreludeBlocks, self.numCoreBlocks, self.numCodaBlocks = numPreludeBlocks, numCoreBlocks, numCodaBlocks
|
| 24 |
+
self.contextLength = contextLength
|
| 25 |
+
blocks = numBlocks if numBlocks else (numPreludeBlocks or 0) + (numCoreBlocks or 0) + (numCodaBlocks or 0)
|
| 26 |
+
self.attentionWindows = attentionWindows or [0] * blocks
|
| 27 |
+
self.chat = chat or {"user": "<|u|>", "assistant": "<|a|>", "end": "<|e|>", "endOfText": "<|endoftext|>"}
|
| 28 |
+
kwargs.setdefault("tie_word_embeddings", False) # the tying is inside the model
|
| 29 |
+
self.vocab_size, self.hidden_size, self.num_attention_heads = vocabSize, featureDim, numHeads
|
| 30 |
+
self.num_hidden_layers, self.max_position_embeddings = blocks, contextLength
|
| 31 |
+
super().__init__(**kwargs)
|
generation_config.json
ADDED
|
@@ -0,0 +1,9 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{
|
| 2 |
+
"do_sample": true,
|
| 3 |
+
"temperature": 0.7,
|
| 4 |
+
"top_k": 40,
|
| 5 |
+
"max_new_tokens": 400,
|
| 6 |
+
"stop_strings": [
|
| 7 |
+
"<|e|>"
|
| 8 |
+
]
|
| 9 |
+
}
|
model.json
ADDED
|
@@ -0,0 +1,53 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{
|
| 2 |
+
"name": "ChatGBA",
|
| 3 |
+
"architecture": "chatgba",
|
| 4 |
+
"vocabSize": 256,
|
| 5 |
+
"featureDim": 192,
|
| 6 |
+
"numHeads": 6,
|
| 7 |
+
"numKVHeads": 2,
|
| 8 |
+
"headDim": 32,
|
| 9 |
+
"ropeBase": 10000,
|
| 10 |
+
"numBlocks": 10,
|
| 11 |
+
"mlpHiddenDim": 512,
|
| 12 |
+
"convKernel": 4,
|
| 13 |
+
"contextLength": 512,
|
| 14 |
+
"attentionWindows": [
|
| 15 |
+
64,
|
| 16 |
+
64,
|
| 17 |
+
64,
|
| 18 |
+
64,
|
| 19 |
+
0,
|
| 20 |
+
64,
|
| 21 |
+
64,
|
| 22 |
+
64,
|
| 23 |
+
64,
|
| 24 |
+
0
|
| 25 |
+
],
|
| 26 |
+
"derived": {
|
| 27 |
+
"parameterCount": 4003264,
|
| 28 |
+
"queryHeadsPerKVHead": 3,
|
| 29 |
+
"kvCacheBytesInt8": "2 global blocks x 512 positions x 128 + 8 windowed blocks x 64 positions x 128 = 196,608"
|
| 30 |
+
},
|
| 31 |
+
"chat": {
|
| 32 |
+
"user": "<|u|>",
|
| 33 |
+
"assistant": "<|a|>",
|
| 34 |
+
"end": "<|e|>",
|
| 35 |
+
"endOfText": "<|endoftext|>",
|
| 36 |
+
"encoding": "markers are plain UTF-8 text spelled out byte by byte; no reserved ids",
|
| 37 |
+
"template": "<|u|>{user}<|e|><|a|>{assistant}<|e|>, repeated for each turn pair",
|
| 38 |
+
"structure": "one or more user/assistant turn pairs, starting with the user; no system prompt",
|
| 39 |
+
"generationPrompt": "the conversation so far followed by <|a|>; the reply ends at <|e|>"
|
| 40 |
+
},
|
| 41 |
+
"weights": {
|
| 42 |
+
"format": "self-describing GAB_ file: 12-byte preamble (magic GAB_, version, header length), JSON header, then little-endian float32 weights in this order",
|
| 43 |
+
"matrixLayout": "row-major [in, out]; activations are multiplied as x @ W",
|
| 44 |
+
"order": [
|
| 45 |
+
"tokenEmbeddings [vocabSize, featureDim] (tied: also the output projection, transposed)",
|
| 46 |
+
"for each block: attentionNorm.gamma [1, featureDim], canonA.weight [convKernel, featureDim] (row j weighs the input j positions back)",
|
| 47 |
+
" for each key/value head g: learnedK [featureDim, headDim], learnedV [featureDim, headDim], keyNorm.gamma [1, headDim]",
|
| 48 |
+
" then for each of its numHeads / numKVHeads query heads: learnedQ [featureDim, headDim], learnedO [headDim, featureDim], queryNorm.gamma [1, headDim]",
|
| 49 |
+
" mlpNorm.gamma [1, featureDim], canonC.weight [convKernel, featureDim], mlp.learnedGate [featureDim, mlpHiddenDim], mlp.learnedUp [featureDim, mlpHiddenDim], mlp.learnedDown [mlpHiddenDim, featureDim]",
|
| 50 |
+
"finalNorm.gamma [1, featureDim]"
|
| 51 |
+
]
|
| 52 |
+
}
|
| 53 |
+
}
|
model.safetensors
ADDED
|
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
version https://git-lfs.github.com/spec/v1
|
| 2 |
+
oid sha256:843bc1405e0d8756577b8ca353f55f766044190dce2935c47ee44c7a02e13c13
|
| 3 |
+
size 16024856
|
modeling_gabgpt.py
ADDED
|
@@ -0,0 +1,151 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""ChatGBA for transformers: AutoModelForCausalLM with trust_remote_code.
|
| 2 |
+
|
| 3 |
+
The same model as reference.js and the Game Boy Advance engine: byte tokens;
|
| 4 |
+
blocks with RMSNorm, canon convolutions, grouped-query attention with QK-norm
|
| 5 |
+
and interleaved RoPE (most blocks see only the last 64 bytes), and a SwiGLU
|
| 6 |
+
MLP; a tied embedding table.
|
| 7 |
+
|
| 8 |
+
There is no KV cache: every call runs the last contextLength bytes from
|
| 9 |
+
scratch. The model was fine-tuned on windows that start anywhere in a
|
| 10 |
+
conversation, so a long chat simply keeps its latest contextLength bytes, a
|
| 11 |
+
sliding context. At 4 million parameters, that is fast enough.
|
| 12 |
+
"""
|
| 13 |
+
import torch
|
| 14 |
+
import torch.nn.functional as F
|
| 15 |
+
from torch import nn
|
| 16 |
+
from transformers import GenerationMixin, PreTrainedModel
|
| 17 |
+
from transformers.modeling_outputs import CausalLMOutput
|
| 18 |
+
|
| 19 |
+
from .configuration_gabgpt import GabGPTConfig
|
| 20 |
+
|
| 21 |
+
RMS_EPSILON = 1e-5
|
| 22 |
+
|
| 23 |
+
|
| 24 |
+
def rope(x, base):
|
| 25 |
+
"""Rotate adjacent pairs (2i, 2i+1) by position x base^(-2i/head_dim)."""
|
| 26 |
+
length, head_dim = x.shape[-2], x.shape[-1]
|
| 27 |
+
inverse = base ** (-torch.arange(0, head_dim, 2, dtype=torch.float32, device=x.device) / head_dim)
|
| 28 |
+
angles = torch.arange(length, dtype=torch.float32, device=x.device)[:, None] * inverse
|
| 29 |
+
cos, sin = angles.cos().to(x.dtype), angles.sin().to(x.dtype)
|
| 30 |
+
x0, x1 = x[..., 0::2], x[..., 1::2]
|
| 31 |
+
return torch.stack([x0 * cos - x1 * sin, x0 * sin + x1 * cos], dim=-1).flatten(-2)
|
| 32 |
+
|
| 33 |
+
|
| 34 |
+
def causal_conv(x, weight):
|
| 35 |
+
"""out[t] = sum over j of weight[j] * x[t - j], per feature."""
|
| 36 |
+
out = weight[0] * x
|
| 37 |
+
for j in range(1, weight.shape[0]):
|
| 38 |
+
out = out + weight[j] * F.pad(x, (0, 0, j, 0))[:, : x.shape[1]]
|
| 39 |
+
return out
|
| 40 |
+
|
| 41 |
+
|
| 42 |
+
class GabGPTBlock(nn.Module):
|
| 43 |
+
def __init__(self, c, window):
|
| 44 |
+
super().__init__()
|
| 45 |
+
d, h, kv, hd = c.featureDim, c.numHeads, c.numKVHeads, c.headDim
|
| 46 |
+
self.heads, self.kv_heads, self.head_dim, self.rope_base, self.window = h, kv, hd, c.ropeBase, window
|
| 47 |
+
self.attention_norm = nn.Parameter(torch.ones(d))
|
| 48 |
+
self.conv_a = nn.Parameter(torch.zeros(c.convKernel, d))
|
| 49 |
+
self.wq = nn.Parameter(torch.zeros(d, h * hd))
|
| 50 |
+
self.wk = nn.Parameter(torch.zeros(d, kv * hd))
|
| 51 |
+
self.wv = nn.Parameter(torch.zeros(d, kv * hd))
|
| 52 |
+
self.wo = nn.Parameter(torch.zeros(h * hd, d))
|
| 53 |
+
self.q_norm = nn.Parameter(torch.ones(h, hd))
|
| 54 |
+
self.k_norm = nn.Parameter(torch.ones(kv, hd))
|
| 55 |
+
self.mlp_norm = nn.Parameter(torch.ones(d))
|
| 56 |
+
self.conv_c = nn.Parameter(torch.zeros(c.convKernel, d))
|
| 57 |
+
self.gate = nn.Parameter(torch.zeros(d, c.mlpHiddenDim))
|
| 58 |
+
self.up = nn.Parameter(torch.zeros(d, c.mlpHiddenDim))
|
| 59 |
+
self.down = nn.Parameter(torch.zeros(c.mlpHiddenDim, d))
|
| 60 |
+
|
| 61 |
+
def forward(self, x):
|
| 62 |
+
batch, length, d = x.shape
|
| 63 |
+
a = F.rms_norm(x, (d,), self.attention_norm, RMS_EPSILON)
|
| 64 |
+
a = a + causal_conv(a, self.conv_a)
|
| 65 |
+
q = (a @ self.wq).view(batch, length, self.heads, self.head_dim).transpose(1, 2)
|
| 66 |
+
k = (a @ self.wk).view(batch, length, self.kv_heads, self.head_dim).transpose(1, 2)
|
| 67 |
+
v = (a @ self.wv).view(batch, length, self.kv_heads, self.head_dim).transpose(1, 2)
|
| 68 |
+
q = F.rms_norm(q, (self.head_dim,), None, RMS_EPSILON) * self.q_norm[:, None, :]
|
| 69 |
+
k = F.rms_norm(k, (self.head_dim,), None, RMS_EPSILON) * self.k_norm[:, None, :]
|
| 70 |
+
q, k = rope(q, self.rope_base), rope(k, self.rope_base)
|
| 71 |
+
if self.kv_heads != self.heads:
|
| 72 |
+
group = self.heads // self.kv_heads
|
| 73 |
+
k, v = k.repeat_interleave(group, dim=1), v.repeat_interleave(group, dim=1)
|
| 74 |
+
if self.window and self.window < length:
|
| 75 |
+
i = torch.arange(length, device=x.device)
|
| 76 |
+
distance = i[:, None] - i[None, :]
|
| 77 |
+
mixed = F.scaled_dot_product_attention(q, k, v, attn_mask=(distance >= 0) & (distance < self.window),
|
| 78 |
+
scale=self.head_dim ** -0.5)
|
| 79 |
+
else:
|
| 80 |
+
mixed = F.scaled_dot_product_attention(q, k, v, is_causal=True, scale=self.head_dim ** -0.5)
|
| 81 |
+
x = x + mixed.transpose(1, 2).reshape(batch, length, -1) @ self.wo
|
| 82 |
+
m = F.rms_norm(x, (d,), self.mlp_norm, RMS_EPSILON)
|
| 83 |
+
m = m + causal_conv(m, self.conv_c)
|
| 84 |
+
return x + (F.silu(m @ self.gate) * (m @ self.up)) @ self.down
|
| 85 |
+
|
| 86 |
+
|
| 87 |
+
class GabGPTModel(nn.Module):
|
| 88 |
+
def __init__(self, c):
|
| 89 |
+
super().__init__()
|
| 90 |
+
self.c = c
|
| 91 |
+
self.token_embeddings = nn.Parameter(torch.zeros(c.vocabSize, c.featureDim))
|
| 92 |
+
self.blocks = nn.ModuleList(GabGPTBlock(c, w) for w in c.attentionWindows)
|
| 93 |
+
self.final_norm = nn.Parameter(torch.ones(c.featureDim))
|
| 94 |
+
|
| 95 |
+
def forward(self, ids):
|
| 96 |
+
c = self.c
|
| 97 |
+
x = self.token_embeddings[ids]
|
| 98 |
+
if c.loopCount:
|
| 99 |
+
blocks = list(self.blocks)
|
| 100 |
+
prelude = blocks[: c.numPreludeBlocks]
|
| 101 |
+
core = blocks[c.numPreludeBlocks : c.numPreludeBlocks + c.numCoreBlocks]
|
| 102 |
+
coda = blocks[c.numPreludeBlocks + c.numCoreBlocks :]
|
| 103 |
+
for block in prelude:
|
| 104 |
+
x = block(x)
|
| 105 |
+
injected = x
|
| 106 |
+
for loop in range(c.loopCount):
|
| 107 |
+
if loop > 0:
|
| 108 |
+
x = x + injected
|
| 109 |
+
for block in core:
|
| 110 |
+
x = block(x)
|
| 111 |
+
for block in coda:
|
| 112 |
+
x = block(x)
|
| 113 |
+
else:
|
| 114 |
+
for block in self.blocks:
|
| 115 |
+
x = block(x)
|
| 116 |
+
x = F.rms_norm(x, (c.featureDim,), self.final_norm, RMS_EPSILON)
|
| 117 |
+
return x @ self.token_embeddings.T # tied unembedding
|
| 118 |
+
|
| 119 |
+
|
| 120 |
+
class GabGPTForCausalLM(PreTrainedModel, GenerationMixin):
|
| 121 |
+
config_class = GabGPTConfig
|
| 122 |
+
base_model_prefix = "model"
|
| 123 |
+
_no_split_modules = ["GabGPTBlock"]
|
| 124 |
+
_supports_cache_class = False
|
| 125 |
+
_is_stateful = True # no cache to reorder: beam search is not supported
|
| 126 |
+
|
| 127 |
+
def __init__(self, config):
|
| 128 |
+
super().__init__(config)
|
| 129 |
+
self.model = GabGPTModel(config)
|
| 130 |
+
self.post_init()
|
| 131 |
+
|
| 132 |
+
def _init_weights(self, module):
|
| 133 |
+
pass # weights always come from the checkpoint
|
| 134 |
+
|
| 135 |
+
def get_input_embeddings(self):
|
| 136 |
+
return None
|
| 137 |
+
|
| 138 |
+
def forward(self, input_ids=None, labels=None, attention_mask=None, **kwargs):
|
| 139 |
+
"""input_ids [batch, length]: byte values. Only the last contextLength
|
| 140 |
+
bytes are used. attention_mask is ignored: pad on the left only if all
|
| 141 |
+
rows are the same length, or generate one row at a time."""
|
| 142 |
+
ids = input_ids[:, -self.config.contextLength :]
|
| 143 |
+
logits = self.model(ids)
|
| 144 |
+
loss = None
|
| 145 |
+
if labels is not None:
|
| 146 |
+
labels = labels[:, -ids.shape[1] :]
|
| 147 |
+
loss = F.cross_entropy(logits[:, :-1].flatten(0, 1).float(), labels[:, 1:].flatten(), ignore_index=-100)
|
| 148 |
+
return CausalLMOutput(loss=loss, logits=logits)
|
| 149 |
+
|
| 150 |
+
def prepare_inputs_for_generation(self, input_ids, **kwargs):
|
| 151 |
+
return {"input_ids": input_ids[:, -self.config.contextLength :]}
|
reference.js
ADDED
|
@@ -0,0 +1,1570 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
let randomState = 42;
|
| 2 |
+
|
| 3 |
+
function random() {
|
| 4 |
+
randomState =
|
| 5 |
+
(1664525 * randomState + 1013904223) % 4294967296;
|
| 6 |
+
return randomState / 4294967296;
|
| 7 |
+
}
|
| 8 |
+
|
| 9 |
+
export class Tensor {
|
| 10 |
+
// When false, operations skip building the graph:
|
| 11 |
+
static gradEnabled = true;
|
| 12 |
+
|
| 13 |
+
constructor(rows, columns) {
|
| 14 |
+
this.rows = rows;
|
| 15 |
+
this.columns = columns;
|
| 16 |
+
this.data = new Float32Array(rows * columns); // zero-filled
|
| 17 |
+
this.grad = null; // allocated on demand by backward()
|
| 18 |
+
this.role = null; // how the trainer updates it: "matrix", "embedding", or null for vectors and filters
|
| 19 |
+
this._inputs = []; // tensors this one was built from
|
| 20 |
+
this._backward = () => {}; // leaves do nothing
|
| 21 |
+
}
|
| 22 |
+
|
| 23 |
+
get(row, column) { // returns a plain number now, not a Value
|
| 24 |
+
return this.data[row * this.columns + column];
|
| 25 |
+
}
|
| 26 |
+
|
| 27 |
+
set(row, column, value) {
|
| 28 |
+
this.data[row * this.columns + column] = value;
|
| 29 |
+
}
|
| 30 |
+
|
| 31 |
+
// --- Initialization: these create the model's leaf parameters. ---
|
| 32 |
+
|
| 33 |
+
initToZeroes() {
|
| 34 |
+
for (let i = 0; i < this.data.length; ++i) {
|
| 35 |
+
this.data[i] = 0.0;
|
| 36 |
+
}
|
| 37 |
+
}
|
| 38 |
+
|
| 39 |
+
initToOnes() {
|
| 40 |
+
for (let i = 0; i < this.data.length; ++i) {
|
| 41 |
+
this.data[i] = 1.0;
|
| 42 |
+
}
|
| 43 |
+
}
|
| 44 |
+
|
| 45 |
+
initToSmallRandom() {
|
| 46 |
+
const scale = 1 / Math.sqrt(this.rows);
|
| 47 |
+
for (let i = 0; i < this.data.length; ++i) {
|
| 48 |
+
this.data[i] = (random() - 0.5) * 2 * scale; // uniform in [-scale, +scale)
|
| 49 |
+
}
|
| 50 |
+
}
|
| 51 |
+
|
| 52 |
+
zeroGrad() {
|
| 53 |
+
if (this.grad !== null) this.grad.fill(0);
|
| 54 |
+
}
|
| 55 |
+
|
| 56 |
+
parameters() {
|
| 57 |
+
return [this]; // a tensor is one parameter block
|
| 58 |
+
}
|
| 59 |
+
|
| 60 |
+
// --- Element-wise operations (both tensors share the same shape). ---
|
| 61 |
+
|
| 62 |
+
add(other) {
|
| 63 |
+
const out = new Tensor(this.rows, this.columns);
|
| 64 |
+
for (let i = 0; i < this.data.length; ++i) {
|
| 65 |
+
out.data[i] = this.data[i] + other.data[i];
|
| 66 |
+
}
|
| 67 |
+
if (Tensor.gradEnabled) {
|
| 68 |
+
out._inputs = [this, other];
|
| 69 |
+
out._backward = () => {
|
| 70 |
+
for (let i = 0; i < this.data.length; ++i) {
|
| 71 |
+
this.grad[i] += out.grad[i];
|
| 72 |
+
other.grad[i] += out.grad[i];
|
| 73 |
+
}
|
| 74 |
+
};
|
| 75 |
+
}
|
| 76 |
+
return out;
|
| 77 |
+
}
|
| 78 |
+
|
| 79 |
+
sub(other) {
|
| 80 |
+
const out = new Tensor(this.rows, this.columns);
|
| 81 |
+
for (let i = 0; i < this.data.length; ++i) {
|
| 82 |
+
out.data[i] = this.data[i] - other.data[i];
|
| 83 |
+
}
|
| 84 |
+
if (Tensor.gradEnabled) {
|
| 85 |
+
out._inputs = [this, other];
|
| 86 |
+
out._backward = () => {
|
| 87 |
+
for (let i = 0; i < this.data.length; ++i) {
|
| 88 |
+
this.grad[i] += out.grad[i];
|
| 89 |
+
other.grad[i] -= out.grad[i];
|
| 90 |
+
}
|
| 91 |
+
};
|
| 92 |
+
}
|
| 93 |
+
return out;
|
| 94 |
+
}
|
| 95 |
+
|
| 96 |
+
mul(other) { // element-wise multiply (matching numpy/pytorch); matrix multiply is matmul
|
| 97 |
+
const out = new Tensor(this.rows, this.columns);
|
| 98 |
+
for (let i = 0; i < this.data.length; ++i) {
|
| 99 |
+
out.data[i] = this.data[i] * other.data[i];
|
| 100 |
+
}
|
| 101 |
+
if (Tensor.gradEnabled) {
|
| 102 |
+
out._inputs = [this, other];
|
| 103 |
+
out._backward = () => {
|
| 104 |
+
for (let i = 0; i < this.data.length; ++i) {
|
| 105 |
+
this.grad[i] += other.data[i] * out.grad[i];
|
| 106 |
+
other.grad[i] += this.data[i] * out.grad[i];
|
| 107 |
+
}
|
| 108 |
+
};
|
| 109 |
+
}
|
| 110 |
+
return out;
|
| 111 |
+
}
|
| 112 |
+
|
| 113 |
+
scale(scalar) { // multiply every value by one plain number
|
| 114 |
+
const out = new Tensor(this.rows, this.columns);
|
| 115 |
+
for (let i = 0; i < this.data.length; ++i) {
|
| 116 |
+
out.data[i] = this.data[i] * scalar;
|
| 117 |
+
}
|
| 118 |
+
if (Tensor.gradEnabled) {
|
| 119 |
+
out._inputs = [this];
|
| 120 |
+
out._backward = () => {
|
| 121 |
+
for (let i = 0; i < this.data.length; ++i) {
|
| 122 |
+
this.grad[i] += scalar * out.grad[i];
|
| 123 |
+
}
|
| 124 |
+
};
|
| 125 |
+
}
|
| 126 |
+
return out;
|
| 127 |
+
}
|
| 128 |
+
|
| 129 |
+
relu() {
|
| 130 |
+
const out = new Tensor(this.rows, this.columns);
|
| 131 |
+
for (let i = 0; i < this.data.length; ++i) {
|
| 132 |
+
out.data[i] = this.data[i] > 0 ? this.data[i] : 0;
|
| 133 |
+
}
|
| 134 |
+
if (Tensor.gradEnabled) {
|
| 135 |
+
out._inputs = [this];
|
| 136 |
+
out._backward = () => {
|
| 137 |
+
for (let i = 0; i < this.data.length; ++i) {
|
| 138 |
+
this.grad[i] += (this.data[i] > 0 ? 1 : 0) * out.grad[i];
|
| 139 |
+
}
|
| 140 |
+
};
|
| 141 |
+
}
|
| 142 |
+
return out;
|
| 143 |
+
}
|
| 144 |
+
|
| 145 |
+
silu() { // x * sigmoid(x), the gate in SwiGLU
|
| 146 |
+
const out = new Tensor(this.rows, this.columns);
|
| 147 |
+
const sigmoids = new Float32Array(this.data.length); // cached for the backward
|
| 148 |
+
for (let i = 0; i < this.data.length; ++i) {
|
| 149 |
+
const x = this.data[i];
|
| 150 |
+
const s = 1 / (1 + Math.exp(-x));
|
| 151 |
+
sigmoids[i] = s;
|
| 152 |
+
out.data[i] = x * s;
|
| 153 |
+
}
|
| 154 |
+
|
| 155 |
+
if (Tensor.gradEnabled) {
|
| 156 |
+
out._inputs = [this];
|
| 157 |
+
out._backward = () => {
|
| 158 |
+
for (let i = 0; i < this.data.length; ++i) {
|
| 159 |
+
// d/dx [x * s(x)] = s + x * s * (1 - s)
|
| 160 |
+
const s = sigmoids[i];
|
| 161 |
+
this.grad[i] += (s + this.data[i] * s * (1 - s)) * out.grad[i];
|
| 162 |
+
}
|
| 163 |
+
};
|
| 164 |
+
}
|
| 165 |
+
return out;
|
| 166 |
+
}
|
| 167 |
+
|
| 168 |
+
// --- Matrix multiplication: the one node that replaces the most scalars. ---
|
| 169 |
+
|
| 170 |
+
matmul(other) { // (this.rows x this.columns) * (other.rows x other.columns)
|
| 171 |
+
const m = this.rows;
|
| 172 |
+
const k = this.columns; // shared inner dimension
|
| 173 |
+
const n = other.columns;
|
| 174 |
+
const out = new Tensor(m, n);
|
| 175 |
+
|
| 176 |
+
for (let row = 0; row < m; ++row) {
|
| 177 |
+
for (let col = 0; col < n; ++col) {
|
| 178 |
+
let sum = 0.0; // accumulate in double, store as float
|
| 179 |
+
for (let element = 0; element < k; ++element) {
|
| 180 |
+
sum += this.data[row * k + element] * other.data[element * n + col];
|
| 181 |
+
}
|
| 182 |
+
out.data[row * n + col] = sum;
|
| 183 |
+
}
|
| 184 |
+
}
|
| 185 |
+
|
| 186 |
+
if (Tensor.gradEnabled) {
|
| 187 |
+
out._inputs = [this, other];
|
| 188 |
+
out._backward = () => {
|
| 189 |
+
// dThis = dOut * other^T
|
| 190 |
+
for (let row = 0; row < m; ++row) {
|
| 191 |
+
for (let p = 0; p < k; ++p) {
|
| 192 |
+
let sum = 0.0;
|
| 193 |
+
for (let col = 0; col < n; ++col) {
|
| 194 |
+
sum += out.grad[row * n + col] * other.data[p * n + col];
|
| 195 |
+
}
|
| 196 |
+
this.grad[row * k + p] += sum;
|
| 197 |
+
}
|
| 198 |
+
}
|
| 199 |
+
// dOther = this^T * dOut
|
| 200 |
+
for (let p = 0; p < k; ++p) {
|
| 201 |
+
for (let col = 0; col < n; ++col) {
|
| 202 |
+
let sum = 0.0;
|
| 203 |
+
for (let row = 0; row < m; ++row) {
|
| 204 |
+
sum += this.data[row * k + p] * out.grad[row * n + col];
|
| 205 |
+
}
|
| 206 |
+
other.grad[p * n + col] += sum;
|
| 207 |
+
}
|
| 208 |
+
}
|
| 209 |
+
};
|
| 210 |
+
}
|
| 211 |
+
return out;
|
| 212 |
+
}
|
| 213 |
+
|
| 214 |
+
transposed() {
|
| 215 |
+
const out = new Tensor(this.columns, this.rows);
|
| 216 |
+
for (let row = 0; row < this.rows; ++row) {
|
| 217 |
+
for (let col = 0; col < this.columns; ++col) {
|
| 218 |
+
out.data[col * this.rows + row] = this.data[row * this.columns + col];
|
| 219 |
+
}
|
| 220 |
+
}
|
| 221 |
+
if (Tensor.gradEnabled) {
|
| 222 |
+
out._inputs = [this];
|
| 223 |
+
out._backward = () => {
|
| 224 |
+
for (let row = 0; row < this.rows; ++row) {
|
| 225 |
+
for (let col = 0; col < this.columns; ++col) {
|
| 226 |
+
this.grad[row * this.columns + col] += out.grad[col * this.rows + row];
|
| 227 |
+
}
|
| 228 |
+
}
|
| 229 |
+
};
|
| 230 |
+
}
|
| 231 |
+
return out;
|
| 232 |
+
}
|
| 233 |
+
|
| 234 |
+
// --- Fused operations. Each is one node with a single analytic backward
|
| 235 |
+
// rule instead of being assembled from many smaller nodes, which is
|
| 236 |
+
// both faster and far lighter on memory. These are general array
|
| 237 |
+
// operations, so they live here. RMS normalization and the
|
| 238 |
+
// cross-entropy loss are also fused, but each belongs to one component
|
| 239 |
+
// (the norm layer, the trainer), so they are defined there using the
|
| 240 |
+
// same _inputs/_backward protocol. ---
|
| 241 |
+
|
| 242 |
+
softMaxedRows() {
|
| 243 |
+
const rows = this.rows;
|
| 244 |
+
const cols = this.columns;
|
| 245 |
+
const out = new Tensor(rows, cols);
|
| 246 |
+
|
| 247 |
+
for (let row = 0; row < rows; ++row) {
|
| 248 |
+
const base = row * cols;
|
| 249 |
+
|
| 250 |
+
let rowMax = this.data[base];
|
| 251 |
+
for (let col = 1; col < cols; ++col) {
|
| 252 |
+
const v = this.data[base + col];
|
| 253 |
+
if (v > rowMax) rowMax = v;
|
| 254 |
+
}
|
| 255 |
+
|
| 256 |
+
let rowSum = 0.0;
|
| 257 |
+
for (let col = 0; col < cols; ++col) {
|
| 258 |
+
const e = Math.exp(this.data[base + col] - rowMax);
|
| 259 |
+
out.data[base + col] = e;
|
| 260 |
+
rowSum += e;
|
| 261 |
+
}
|
| 262 |
+
for (let col = 0; col < cols; ++col) {
|
| 263 |
+
out.data[base + col] /= rowSum;
|
| 264 |
+
}
|
| 265 |
+
}
|
| 266 |
+
|
| 267 |
+
if (Tensor.gradEnabled) {
|
| 268 |
+
out._inputs = [this];
|
| 269 |
+
out._backward = () => {
|
| 270 |
+
// For each row: dx_j = p_j * (g_j - sum_k g_k p_k)
|
| 271 |
+
for (let row = 0; row < rows; ++row) {
|
| 272 |
+
const base = row * cols;
|
| 273 |
+
let dot = 0.0;
|
| 274 |
+
for (let col = 0; col < cols; ++col) {
|
| 275 |
+
dot += out.grad[base + col] * out.data[base + col];
|
| 276 |
+
}
|
| 277 |
+
for (let col = 0; col < cols; ++col) {
|
| 278 |
+
const p = out.data[base + col];
|
| 279 |
+
this.grad[base + col] += p * (out.grad[base + col] - dot);
|
| 280 |
+
}
|
| 281 |
+
}
|
| 282 |
+
};
|
| 283 |
+
}
|
| 284 |
+
return out;
|
| 285 |
+
}
|
| 286 |
+
|
| 287 |
+
// Attention mask for a block of scores: row r is the query at absolute
|
| 288 |
+
// position offset + r, column c the key at position c. A query sees keys
|
| 289 |
+
// at or before its own position, and with a window, only the last window
|
| 290 |
+
// of them (0: no window).
|
| 291 |
+
attentionMasked(offset, window) {
|
| 292 |
+
const rows = this.rows;
|
| 293 |
+
const cols = this.columns;
|
| 294 |
+
const out = new Tensor(rows, cols);
|
| 295 |
+
const visible = (row, col) => col <= offset + row && (window === 0 || offset + row - col < window);
|
| 296 |
+
|
| 297 |
+
for (let row = 0; row < rows; ++row) {
|
| 298 |
+
for (let col = 0; col < cols; ++col) {
|
| 299 |
+
const index = row * cols + col;
|
| 300 |
+
out.data[index] = visible(row, col) ? this.data[index] : -Infinity;
|
| 301 |
+
}
|
| 302 |
+
}
|
| 303 |
+
|
| 304 |
+
if (Tensor.gradEnabled) {
|
| 305 |
+
out._inputs = [this];
|
| 306 |
+
out._backward = () => {
|
| 307 |
+
// Masked cells are constants, so gradient only flows where visible.
|
| 308 |
+
for (let row = 0; row < rows; ++row) {
|
| 309 |
+
for (let col = 0; col < cols; ++col) {
|
| 310 |
+
if (visible(row, col)) {
|
| 311 |
+
const index = row * cols + col;
|
| 312 |
+
this.grad[index] += out.grad[index];
|
| 313 |
+
}
|
| 314 |
+
}
|
| 315 |
+
}
|
| 316 |
+
};
|
| 317 |
+
}
|
| 318 |
+
return out;
|
| 319 |
+
}
|
| 320 |
+
|
| 321 |
+
// Embedding lookup: copy the rows named by `indices` into a new tensor.
|
| 322 |
+
// The gradient scatters back, accumulating when an index repeats.
|
| 323 |
+
gatherRows(indices) {
|
| 324 |
+
const out = new Tensor(indices.length, this.columns);
|
| 325 |
+
for (let r = 0; r < indices.length; ++r) {
|
| 326 |
+
const src = indices[r] * this.columns;
|
| 327 |
+
const dst = r * this.columns;
|
| 328 |
+
for (let col = 0; col < this.columns; ++col) {
|
| 329 |
+
out.data[dst + col] = this.data[src + col];
|
| 330 |
+
}
|
| 331 |
+
}
|
| 332 |
+
if (Tensor.gradEnabled) {
|
| 333 |
+
out._inputs = [this];
|
| 334 |
+
out._backward = () => {
|
| 335 |
+
for (let r = 0; r < indices.length; ++r) {
|
| 336 |
+
const src = indices[r] * this.columns;
|
| 337 |
+
const dst = r * this.columns;
|
| 338 |
+
for (let col = 0; col < this.columns; ++col) {
|
| 339 |
+
this.grad[src + col] += out.grad[dst + col];
|
| 340 |
+
}
|
| 341 |
+
}
|
| 342 |
+
};
|
| 343 |
+
}
|
| 344 |
+
return out;
|
| 345 |
+
}
|
| 346 |
+
|
| 347 |
+
// Stack another tensor's rows below this one's, returning a new, taller
|
| 348 |
+
// tensor. Both must share the same column count. The KV cache uses this to
|
| 349 |
+
// grow one row per generated token. The gradient of the top rows flows back
|
| 350 |
+
// to this, the bottom rows to other. Generation runs with the graph off, so
|
| 351 |
+
// that rule never fires in practice, but it keeps appendRow a first-class op.
|
| 352 |
+
appendRow(other) {
|
| 353 |
+
// A new tensor: this tensor's rows on top, other's rows stacked below.
|
| 354 |
+
// Both must share the same column count.
|
| 355 |
+
const out = new Tensor(this.rows + other.rows, this.columns);
|
| 356 |
+
out.data.set(this.data, 0); // top block
|
| 357 |
+
out.data.set(other.data, this.data.length); // bottom block
|
| 358 |
+
|
| 359 |
+
if (Tensor.gradEnabled) {
|
| 360 |
+
out._inputs = [this, other];
|
| 361 |
+
out._backward = () => {
|
| 362 |
+
// Gradient of the top rows flows to this, the bottom rows to other.
|
| 363 |
+
for (let i = 0; i < this.data.length; ++i) {
|
| 364 |
+
this.grad[i] += out.grad[i];
|
| 365 |
+
}
|
| 366 |
+
for (let i = 0; i < other.data.length; ++i) {
|
| 367 |
+
other.grad[i] += out.grad[this.data.length + i];
|
| 368 |
+
}
|
| 369 |
+
};
|
| 370 |
+
}
|
| 371 |
+
return out;
|
| 372 |
+
}
|
| 373 |
+
|
| 374 |
+
// RoPE: rotate each row's component pairs by an angle set by the row's
|
| 375 |
+
// position. Query and key rows sit in sequence order, so the row index is
|
| 376 |
+
// the position — plus positionOffset, which the KV cache uses to rotate a
|
| 377 |
+
// new token by its true position rather than its row (offset zero, the
|
| 378 |
+
// default, reproduces the original behavior). The angles are fixed functions
|
| 379 |
+
// of position and frequency, not learned, so this op adds nothing to
|
| 380 |
+
// parameters(); the gradient only routes back to the input. The cosine and
|
| 381 |
+
// sine of each angle are cached, since the backward pass — a rotation by the
|
| 382 |
+
// negated angle — reuses them.
|
| 383 |
+
ropeRotated(thetaBase = 10000, positionOffset = 0) { // positionOffset is new
|
| 384 |
+
const rows = this.rows;
|
| 385 |
+
const cols = this.columns; // head_dim, must be even
|
| 386 |
+
const halfCols = cols / 2;
|
| 387 |
+
const out = new Tensor(rows, cols);
|
| 388 |
+
|
| 389 |
+
// Frequency depends on the pair, not the row
|
| 390 |
+
// Compute it once and cache
|
| 391 |
+
const freqs = new Float32Array(halfCols);
|
| 392 |
+
for (let pair = 0; pair < halfCols; ++pair) {
|
| 393 |
+
freqs[pair] = 1.0 / Math.pow(thetaBase, (2 * pair) / cols);
|
| 394 |
+
}
|
| 395 |
+
|
| 396 |
+
// Cached for the backward pass.
|
| 397 |
+
const cosTable = new Float32Array(rows * halfCols);
|
| 398 |
+
const sinTable = new Float32Array(rows * halfCols);
|
| 399 |
+
|
| 400 |
+
for (let row = 0; row < rows; ++row) {
|
| 401 |
+
const base = row * cols;
|
| 402 |
+
const trig = row * halfCols;
|
| 403 |
+
for (let pair = 0; pair < halfCols; ++pair) {
|
| 404 |
+
const angle = (positionOffset + row) * freqs[pair]; // true position = offset + row
|
| 405 |
+
const cos = Math.cos(angle);
|
| 406 |
+
const sin = Math.sin(angle);
|
| 407 |
+
cosTable[trig + pair] = cos;
|
| 408 |
+
sinTable[trig + pair] = sin;
|
| 409 |
+
|
| 410 |
+
const a = this.data[base + 2 * pair];
|
| 411 |
+
const b = this.data[base + 2 * pair + 1];
|
| 412 |
+
out.data[base + 2 * pair] = a * cos - b * sin;
|
| 413 |
+
out.data[base + 2 * pair + 1] = a * sin + b * cos;
|
| 414 |
+
}
|
| 415 |
+
}
|
| 416 |
+
|
| 417 |
+
if (Tensor.gradEnabled) {
|
| 418 |
+
out._inputs = [this];
|
| 419 |
+
out._backward = () => {
|
| 420 |
+
// Rotation is linear, the gradient is the inverse rotation.
|
| 421 |
+
for (let row = 0; row < rows; ++row) {
|
| 422 |
+
const base = row * cols;
|
| 423 |
+
const trig = row * halfCols;
|
| 424 |
+
for (let pair = 0; pair < halfCols; ++pair) {
|
| 425 |
+
const cos = cosTable[trig + pair];
|
| 426 |
+
const sin = sinTable[trig + pair];
|
| 427 |
+
const ga = out.grad[base + 2 * pair];
|
| 428 |
+
const gb = out.grad[base + 2 * pair + 1];
|
| 429 |
+
this.grad[base + 2 * pair] += ga * cos + gb * sin;
|
| 430 |
+
this.grad[base + 2 * pair + 1] += -ga * sin + gb * cos;
|
| 431 |
+
}
|
| 432 |
+
}
|
| 433 |
+
};
|
| 434 |
+
}
|
| 435 |
+
return out;
|
| 436 |
+
}
|
| 437 |
+
|
| 438 |
+
// Short causal convolution, one filter per column (depthwise): out[t] =
|
| 439 |
+
// sum over j of weight[j] * x[t - j], where weight is [kernel, columns] and
|
| 440 |
+
// row j weighs the input j rows back. Rows before the first are zeros, or
|
| 441 |
+
// come from history (the previous kernel - 1 rows, during generation). The
|
| 442 |
+
// gradient flows to this tensor and the filter; history is a constant.
|
| 443 |
+
causalConv(weight, history = null) {
|
| 444 |
+
const rows = this.rows;
|
| 445 |
+
const cols = this.columns;
|
| 446 |
+
const taps = weight.rows - 1;
|
| 447 |
+
const out = new Tensor(rows, cols);
|
| 448 |
+
|
| 449 |
+
const input = (source, col) => {
|
| 450 |
+
if (source >= 0) return this.data[source * cols + col];
|
| 451 |
+
if (history !== null) return history.data[(taps + source) * cols + col];
|
| 452 |
+
return 0.0;
|
| 453 |
+
};
|
| 454 |
+
|
| 455 |
+
for (let row = 0; row < rows; ++row) {
|
| 456 |
+
for (let col = 0; col < cols; ++col) {
|
| 457 |
+
let sum = 0.0;
|
| 458 |
+
for (let j = 0; j <= taps; ++j) {
|
| 459 |
+
sum += weight.data[j * cols + col] * input(row - j, col);
|
| 460 |
+
}
|
| 461 |
+
out.data[row * cols + col] = sum;
|
| 462 |
+
}
|
| 463 |
+
}
|
| 464 |
+
|
| 465 |
+
if (Tensor.gradEnabled) {
|
| 466 |
+
out._inputs = [this, weight];
|
| 467 |
+
out._backward = () => {
|
| 468 |
+
for (let row = 0; row < rows; ++row) {
|
| 469 |
+
for (let col = 0; col < cols; ++col) {
|
| 470 |
+
const g = out.grad[row * cols + col];
|
| 471 |
+
for (let j = 0; j <= taps; ++j) {
|
| 472 |
+
const source = row - j;
|
| 473 |
+
weight.grad[j * cols + col] += g * input(source, col);
|
| 474 |
+
if (source >= 0) {
|
| 475 |
+
this.grad[source * cols + col] += g * weight.data[j * cols + col];
|
| 476 |
+
}
|
| 477 |
+
}
|
| 478 |
+
}
|
| 479 |
+
}
|
| 480 |
+
};
|
| 481 |
+
}
|
| 482 |
+
return out;
|
| 483 |
+
}
|
| 484 |
+
|
| 485 |
+
// The last count rows, with earlier rows from history (or zeros) when this
|
| 486 |
+
// tensor is shorter. Carries a convolution's input from one generation step
|
| 487 |
+
// to the next, so it builds no graph.
|
| 488 |
+
lastRows(count, history = null) {
|
| 489 |
+
const out = new Tensor(count, this.columns);
|
| 490 |
+
for (let r = 0; r < count; ++r) {
|
| 491 |
+
const source = this.rows - count + r;
|
| 492 |
+
if (source >= 0) {
|
| 493 |
+
out.data.set(this.data.subarray(source * this.columns, (source + 1) * this.columns), r * this.columns);
|
| 494 |
+
} else if (history !== null) {
|
| 495 |
+
const h = history.rows + source;
|
| 496 |
+
out.data.set(history.data.subarray(h * this.columns, (h + 1) * this.columns), r * this.columns);
|
| 497 |
+
}
|
| 498 |
+
}
|
| 499 |
+
return out;
|
| 500 |
+
}
|
| 501 |
+
|
| 502 |
+
backward() {
|
| 503 |
+
// Topological sort: inputs come before the nodes built from them.
|
| 504 |
+
const topo = [];
|
| 505 |
+
const visited = new Set();
|
| 506 |
+
|
| 507 |
+
const visit = (node) => {
|
| 508 |
+
if (visited.has(node)) return;
|
| 509 |
+
visited.add(node);
|
| 510 |
+
for (let i = 0; i < node._inputs.length; ++i) {
|
| 511 |
+
visit(node._inputs[i]);
|
| 512 |
+
}
|
| 513 |
+
topo.push(node);
|
| 514 |
+
};
|
| 515 |
+
visit(this);
|
| 516 |
+
|
| 517 |
+
// Make sure every node in the graph has a gradient buffer. Freshly
|
| 518 |
+
// built tensors start at null and get a zeroed buffer here. Parameter
|
| 519 |
+
// tensors keep the buffer the trainer already zeroed.
|
| 520 |
+
for (let i = 0; i < topo.length; ++i) {
|
| 521 |
+
if (topo[i].grad === null) {
|
| 522 |
+
topo[i].grad = new Float32Array(topo[i].data.length);
|
| 523 |
+
}
|
| 524 |
+
}
|
| 525 |
+
|
| 526 |
+
// Seed: derivative of the output with respect to itself is 1.
|
| 527 |
+
this.grad.fill(1);
|
| 528 |
+
|
| 529 |
+
// Walk in reverse. Each node pushes gradient onto its inputs.
|
| 530 |
+
for (let i = topo.length - 1; i >= 0; --i) {
|
| 531 |
+
topo[i]._backward();
|
| 532 |
+
}
|
| 533 |
+
}
|
| 534 |
+
}
|
| 535 |
+
|
| 536 |
+
|
| 537 |
+
export class RMSNorm {
|
| 538 |
+
constructor(feature_dim) {
|
| 539 |
+
// Learned per-feature scale, applied after normalization. Starts at one,
|
| 540 |
+
// so the layer begins as a plain normalize until training moves it.
|
| 541 |
+
this.gamma = new Tensor(1, feature_dim);
|
| 542 |
+
this.gamma.initToOnes();
|
| 543 |
+
}
|
| 544 |
+
|
| 545 |
+
forward(x) {
|
| 546 |
+
const rows = x.rows;
|
| 547 |
+
const n = x.columns;
|
| 548 |
+
const eps = 1e-5;
|
| 549 |
+
const gamma = this.gamma;
|
| 550 |
+
const out = new Tensor(rows, n);
|
| 551 |
+
const invRmsByRow = new Float32Array(rows); // cached for the backward
|
| 552 |
+
|
| 553 |
+
for (let row = 0; row < rows; ++row) {
|
| 554 |
+
const base = row * n;
|
| 555 |
+
|
| 556 |
+
let sumSq = 0.0;
|
| 557 |
+
for (let col = 0; col < n; ++col) {
|
| 558 |
+
const v = x.data[base + col];
|
| 559 |
+
sumSq += v * v;
|
| 560 |
+
}
|
| 561 |
+
|
| 562 |
+
const invRms = 1 / Math.sqrt(sumSq / n + eps);
|
| 563 |
+
invRmsByRow[row] = invRms;
|
| 564 |
+
|
| 565 |
+
for (let col = 0; col < n; ++col) {
|
| 566 |
+
// normalize, then scale by the per-feature gamma
|
| 567 |
+
out.data[base + col] = x.data[base + col] * invRms * gamma.data[col];
|
| 568 |
+
}
|
| 569 |
+
}
|
| 570 |
+
|
| 571 |
+
if (Tensor.gradEnabled) {
|
| 572 |
+
out._inputs = [x, gamma];
|
| 573 |
+
out._backward = () => {
|
| 574 |
+
for (let row = 0; row < rows; ++row) {
|
| 575 |
+
const base = row * n;
|
| 576 |
+
const invRms = invRmsByRow[row];
|
| 577 |
+
const invRms3 = invRms * invRms * invRms;
|
| 578 |
+
|
| 579 |
+
// Upstream gradient passes through gamma before the RMS rule.
|
| 580 |
+
let dotGX = 0.0;
|
| 581 |
+
for (let col = 0; col < n; ++col) {
|
| 582 |
+
const gNorm = out.grad[base + col] * gamma.data[col];
|
| 583 |
+
dotGX += gNorm * x.data[base + col];
|
| 584 |
+
}
|
| 585 |
+
for (let col = 0; col < n; ++col) {
|
| 586 |
+
const g = out.grad[base + col];
|
| 587 |
+
const xv = x.data[base + col];
|
| 588 |
+
const gNorm = g * gamma.data[col];
|
| 589 |
+
x.grad[base + col] += gNorm * invRms - (xv * dotGX * invRms3) / n;
|
| 590 |
+
// gamma is shared across rows: sum the normalized input times upstream grad
|
| 591 |
+
gamma.grad[col] += xv * invRms * g;
|
| 592 |
+
}
|
| 593 |
+
}
|
| 594 |
+
};
|
| 595 |
+
}
|
| 596 |
+
return out;
|
| 597 |
+
}
|
| 598 |
+
|
| 599 |
+
parameters() {
|
| 600 |
+
return [this.gamma];
|
| 601 |
+
}
|
| 602 |
+
}
|
| 603 |
+
|
| 604 |
+
// Canon layer: a short causal convolution added back onto its input, applied
|
| 605 |
+
// right after a pre-norm. It lets each position mix in the few positions before
|
| 606 |
+
// it, which a byte-level model needs constantly to assemble bytes into words.
|
| 607 |
+
// The filter starts at zero, so the layer starts as a no-op.
|
| 608 |
+
export class CanonLayer {
|
| 609 |
+
constructor(feature_dim, kernel) {
|
| 610 |
+
this.weight = new Tensor(kernel, feature_dim);
|
| 611 |
+
this.weight.initToZeroes();
|
| 612 |
+
}
|
| 613 |
+
|
| 614 |
+
// history: the previous kernel - 1 inputs, kept in the cache's slot.
|
| 615 |
+
forward(x, cache = null, layerIndex = 0, slot = "convA") {
|
| 616 |
+
const history = cache === null ? null : cache[slot][layerIndex];
|
| 617 |
+
const out = x.add(x.causalConv(this.weight, history));
|
| 618 |
+
if (cache !== null) {
|
| 619 |
+
cache[slot][layerIndex] = x.lastRows(this.weight.rows - 1, history);
|
| 620 |
+
}
|
| 621 |
+
return out;
|
| 622 |
+
}
|
| 623 |
+
|
| 624 |
+
parameters() {
|
| 625 |
+
return [this.weight];
|
| 626 |
+
}
|
| 627 |
+
}
|
| 628 |
+
|
| 629 |
+
export class MultiHeadAttention {
|
| 630 |
+
constructor(feature_dim, num_heads, num_kv_heads, head_dim, rope_base, window) {
|
| 631 |
+
this.num_heads = num_heads;
|
| 632 |
+
this.num_kv_heads = num_kv_heads;
|
| 633 |
+
this.head_dim = head_dim; // even, for RoPE
|
| 634 |
+
this.rope_base = rope_base;
|
| 635 |
+
this.window = window; // attend to at most this many recent positions; 0: all
|
| 636 |
+
|
| 637 |
+
// Grouped-query attention: fewer key/value heads than query heads.
|
| 638 |
+
// Each key/value head serves a group of consecutive query heads, so the
|
| 639 |
+
// cache holds num_kv_heads keys and values per position, not num_heads.
|
| 640 |
+
// Each query head still has its own output projection back up to
|
| 641 |
+
// feature_dim; the summed outputs equal a concat plus one big projection.
|
| 642 |
+
this.groups = [];
|
| 643 |
+
for (let g = 0; g < num_kv_heads; ++g) {
|
| 644 |
+
const learnedK = new Tensor(feature_dim, head_dim);
|
| 645 |
+
learnedK.initToSmallRandom();
|
| 646 |
+
|
| 647 |
+
const learnedV = new Tensor(feature_dim, head_dim);
|
| 648 |
+
learnedV.initToSmallRandom();
|
| 649 |
+
|
| 650 |
+
// QK-norm: each key/value head normalizes its keys, and each query
|
| 651 |
+
// head its queries, with a learned scale, before RoPE.
|
| 652 |
+
const keyNorm = new RMSNorm(head_dim);
|
| 653 |
+
|
| 654 |
+
const queries = [];
|
| 655 |
+
for (let i = 0; i < num_heads / num_kv_heads; ++i) {
|
| 656 |
+
const learnedQ = new Tensor(feature_dim, head_dim);
|
| 657 |
+
learnedQ.initToSmallRandom();
|
| 658 |
+
|
| 659 |
+
const learnedO = new Tensor(head_dim, feature_dim);
|
| 660 |
+
learnedO.initToSmallRandom();
|
| 661 |
+
|
| 662 |
+
const queryNorm = new RMSNorm(head_dim);
|
| 663 |
+
queries.push({ learnedQ, learnedO, queryNorm });
|
| 664 |
+
}
|
| 665 |
+
|
| 666 |
+
for (const matrix of [learnedK, learnedV, ...queries.flatMap(q => [q.learnedQ, q.learnedO])]) {
|
| 667 |
+
matrix.role = "matrix";
|
| 668 |
+
}
|
| 669 |
+
this.groups.push({ learnedK, learnedV, keyNorm, queries });
|
| 670 |
+
}
|
| 671 |
+
}
|
| 672 |
+
|
| 673 |
+
forward(rms_norm_matrix, cache = null, layerIndex = 0) { // Shape: new_tokens, feature_dim
|
| 674 |
+
const scale = 1.0 / Math.sqrt(this.head_dim);
|
| 675 |
+
let output = null;
|
| 676 |
+
|
| 677 |
+
for (let g = 0; g < this.num_kv_heads; ++g) {
|
| 678 |
+
const group = this.groups[g];
|
| 679 |
+
|
| 680 |
+
let K = group.keyNorm.forward(rms_norm_matrix.matmul(group.learnedK)); // Shape: new_tokens, head_dim
|
| 681 |
+
let V = rms_norm_matrix.matmul(group.learnedV); // Shape: new_tokens, head_dim
|
| 682 |
+
|
| 683 |
+
// The new tokens sit after whatever this head has already cached.
|
| 684 |
+
const pastK = cache === null ? null : cache.keys[layerIndex][g];
|
| 685 |
+
const pastV = cache === null ? null : cache.values[layerIndex][g];
|
| 686 |
+
const positionOffset = pastK === null ? 0 : pastK.rows;
|
| 687 |
+
|
| 688 |
+
// Rotate by true position. Cached keys keep the rotation they were
|
| 689 |
+
// stored with; the new keys are rotated from the offset onward.
|
| 690 |
+
K = K.ropeRotated(this.rope_base, positionOffset);
|
| 691 |
+
|
| 692 |
+
// Grow the cache with the new keys and values, then attend over the
|
| 693 |
+
// history. Without a cache, K and V are just the new tokens.
|
| 694 |
+
if (cache !== null) {
|
| 695 |
+
K = pastK === null ? K : pastK.appendRow(K);
|
| 696 |
+
V = pastV === null ? V : pastV.appendRow(V);
|
| 697 |
+
cache.keys[layerIndex][g] = K;
|
| 698 |
+
cache.values[layerIndex][g] = V;
|
| 699 |
+
}
|
| 700 |
+
const keysTransposed = K.transposed();
|
| 701 |
+
|
| 702 |
+
// Every query head in the group reads the same keys and values.
|
| 703 |
+
for (const head of group.queries) {
|
| 704 |
+
let Q = head.queryNorm.forward(rms_norm_matrix.matmul(head.learnedQ)); // Shape: new_tokens, head_dim
|
| 705 |
+
Q = Q.ropeRotated(this.rope_base, positionOffset);
|
| 706 |
+
|
| 707 |
+
let scores = Q.matmul(keysTransposed); // Shape: new_tokens, total_tokens
|
| 708 |
+
scores = scores.scale(scale);
|
| 709 |
+
|
| 710 |
+
// Each query sees only its past, and only the last window
|
| 711 |
+
// positions of it.
|
| 712 |
+
scores = scores.attentionMasked(positionOffset, this.window);
|
| 713 |
+
|
| 714 |
+
const probabilities = scores.softMaxedRows();
|
| 715 |
+
const mixed = probabilities.matmul(V); // Shape: new_tokens, head_dim
|
| 716 |
+
|
| 717 |
+
// Project this head up to feature_dim and accumulate.
|
| 718 |
+
const projected = mixed.matmul(head.learnedO); // Shape: new_tokens, feature_dim
|
| 719 |
+
output = output === null ? projected : output.add(projected);
|
| 720 |
+
}
|
| 721 |
+
}
|
| 722 |
+
|
| 723 |
+
return output; // Shape: new_tokens, feature_dim
|
| 724 |
+
}
|
| 725 |
+
|
| 726 |
+
// Weight-file order: each key/value head's K, V and key norm, then each of
|
| 727 |
+
// its query heads' Q, O and query norm.
|
| 728 |
+
parameters() {
|
| 729 |
+
const params = [];
|
| 730 |
+
for (const group of this.groups) {
|
| 731 |
+
params.push(group.learnedK, group.learnedV, group.keyNorm.gamma);
|
| 732 |
+
for (const head of group.queries) {
|
| 733 |
+
params.push(head.learnedQ, head.learnedO, head.queryNorm.gamma);
|
| 734 |
+
}
|
| 735 |
+
}
|
| 736 |
+
return params;
|
| 737 |
+
}
|
| 738 |
+
}
|
| 739 |
+
|
| 740 |
+
// (x @ up) passes, then down projects back. No bias.
|
| 741 |
+
export class MLP {
|
| 742 |
+
constructor(feature_dim_size, hidden_dim_size) {
|
| 743 |
+
this.learnedGate = new Tensor(feature_dim_size, hidden_dim_size);
|
| 744 |
+
this.learnedGate.initToSmallRandom();
|
| 745 |
+
|
| 746 |
+
this.learnedUp = new Tensor(feature_dim_size, hidden_dim_size);
|
| 747 |
+
this.learnedUp.initToSmallRandom();
|
| 748 |
+
|
| 749 |
+
this.learnedDown = new Tensor(hidden_dim_size, feature_dim_size);
|
| 750 |
+
this.learnedDown.initToSmallRandom();
|
| 751 |
+
|
| 752 |
+
for (const matrix of [this.learnedGate, this.learnedUp, this.learnedDown]) {
|
| 753 |
+
matrix.role = "matrix";
|
| 754 |
+
}
|
| 755 |
+
}
|
| 756 |
+
|
| 757 |
+
forward(x) { // Shape: sequence_length, feature_dim
|
| 758 |
+
const gate = x.matmul(this.learnedGate).silu(); // Shape: sequence_length, hidden_dim
|
| 759 |
+
const up = x.matmul(this.learnedUp); // Shape: sequence_length, hidden_dim
|
| 760 |
+
return gate.mul(up).matmul(this.learnedDown); // Shape: sequence_length, feature_dim
|
| 761 |
+
}
|
| 762 |
+
|
| 763 |
+
parameters() {
|
| 764 |
+
return [this.learnedGate, this.learnedUp, this.learnedDown];
|
| 765 |
+
}
|
| 766 |
+
}
|
| 767 |
+
|
| 768 |
+
export class TransformerBlock {
|
| 769 |
+
constructor(feature_dim, num_heads, num_kv_heads, head_dim, rope_base, hidden_dim, conv_kernel, window) {
|
| 770 |
+
this.attentionNorm = new RMSNorm(feature_dim);
|
| 771 |
+
this.canonA = new CanonLayer(feature_dim, conv_kernel);
|
| 772 |
+
this.attention = new MultiHeadAttention(feature_dim, num_heads, num_kv_heads, head_dim, rope_base, window);
|
| 773 |
+
|
| 774 |
+
this.mlpNorm = new RMSNorm(feature_dim);
|
| 775 |
+
this.canonC = new CanonLayer(feature_dim, conv_kernel);
|
| 776 |
+
this.mlp = new MLP(feature_dim, hidden_dim);
|
| 777 |
+
}
|
| 778 |
+
|
| 779 |
+
forward(x, cache = null, layerIndex = 0) {
|
| 780 |
+
// Attention sub-layer, with residual. The canon layer mixes in the last
|
| 781 |
+
// few positions first. The cache holds this layer's keys, values and
|
| 782 |
+
// both convolution histories.
|
| 783 |
+
const attentionInput = this.canonA.forward(this.attentionNorm.forward(x), cache, layerIndex, "convA");
|
| 784 |
+
const attentionDelta = this.attention.forward(attentionInput, cache, layerIndex);
|
| 785 |
+
const afterAttention = x.add(attentionDelta);
|
| 786 |
+
|
| 787 |
+
// MLP sub-layer, with residual
|
| 788 |
+
const mlpInput = this.canonC.forward(this.mlpNorm.forward(afterAttention), cache, layerIndex, "convC");
|
| 789 |
+
const mlpDelta = this.mlp.forward(mlpInput);
|
| 790 |
+
const afterMLP = afterAttention.add(mlpDelta);
|
| 791 |
+
|
| 792 |
+
return afterMLP;
|
| 793 |
+
}
|
| 794 |
+
|
| 795 |
+
parameters() {
|
| 796 |
+
const params = [];
|
| 797 |
+
const components = [this.attentionNorm, this.canonA, this.attention, this.mlpNorm, this.canonC, this.mlp];
|
| 798 |
+
for (let i = 0; i < components.length; ++i) {
|
| 799 |
+
const sub = components[i].parameters();
|
| 800 |
+
for (let j = 0; j < sub.length; ++j) {
|
| 801 |
+
params.push(sub[j]);
|
| 802 |
+
}
|
| 803 |
+
}
|
| 804 |
+
return params;
|
| 805 |
+
}
|
| 806 |
+
}
|
| 807 |
+
|
| 808 |
+
|
| 809 |
+
|
| 810 |
+
export class GabGPT {
|
| 811 |
+
constructor(vocabSize, featureDim, numHeads, numKVHeads, headDim, ropeBase, numBlocks, mlpHiddenDim, convKernel, attentionWindows) {
|
| 812 |
+
this.vocabSize = vocabSize;
|
| 813 |
+
this.featureDim = featureDim;
|
| 814 |
+
this.numHeads = numHeads;
|
| 815 |
+
this.numKVHeads = numKVHeads; // key/value heads, each shared by numHeads / numKVHeads query heads
|
| 816 |
+
this.headDim = headDim;
|
| 817 |
+
this.attentionWindows = attentionWindows; // per block: how many recent positions it sees; 0: all
|
| 818 |
+
this.ropeBase = ropeBase;
|
| 819 |
+
this.numBlocks = numBlocks;
|
| 820 |
+
this.mlpHiddenDim = mlpHiddenDim; // SwiGLU's hidden width, about 2/3 of 4x to keep the budget
|
| 821 |
+
this.convKernel = convKernel; // canon layer filter length
|
| 822 |
+
|
| 823 |
+
// Token embedding table (position enters through rotation in attention).
|
| 824 |
+
this.tokenEmbeddings = new Tensor(vocabSize, featureDim);
|
| 825 |
+
this.tokenEmbeddings.initToSmallRandom();
|
| 826 |
+
this.tokenEmbeddings.role = "embedding";
|
| 827 |
+
|
| 828 |
+
// Stack of transformer blocks.
|
| 829 |
+
this.blocks = [];
|
| 830 |
+
for (let i = 0; i < numBlocks; ++i) {
|
| 831 |
+
this.blocks.push(new TransformerBlock(featureDim, numHeads, numKVHeads, headDim, ropeBase, mlpHiddenDim, convKernel, attentionWindows[i]));
|
| 832 |
+
}
|
| 833 |
+
|
| 834 |
+
// Final norm before the unembedding.
|
| 835 |
+
this.finalNorm = new RMSNorm(featureDim);
|
| 836 |
+
}
|
| 837 |
+
|
| 838 |
+
embed(ids) {
|
| 839 |
+
// Token embeddings only; position now enters through rotation in attention
|
| 840 |
+
return this.tokenEmbeddings.gatherRows(ids);
|
| 841 |
+
}
|
| 842 |
+
|
| 843 |
+
forward(tokenIdArray, cache = null) { // cache is new
|
| 844 |
+
// No maxContextLength check: there is no position table to index past.
|
| 845 |
+
// A sequence beyond the trained length just runs slower and predicts worse.
|
| 846 |
+
// With a cache, tokenIdArray holds only the new tokens; the block index
|
| 847 |
+
// threads through so each layer reads and writes its own cached slot.
|
| 848 |
+
let x = this.embed(tokenIdArray); // Shape: new_tokens, feature_dim
|
| 849 |
+
for (let i = 0; i < this.blocks.length; ++i) {
|
| 850 |
+
x = this.blocks[i].forward(x, cache, i); // Shape preserved.
|
| 851 |
+
}
|
| 852 |
+
x = this.finalNorm.forward(x); // Shape: new_tokens, feature_dim
|
| 853 |
+
|
| 854 |
+
// Tied unembedding: (sequence_length, feature_dim) * (feature_dim, vocabSize)
|
| 855 |
+
const logits = x.matmul(this.tokenEmbeddings.transposed()); // Shape: sequence_length, vocabSize
|
| 856 |
+
return logits;
|
| 857 |
+
}
|
| 858 |
+
|
| 859 |
+
// Predict the next token. Temperature reshapes the distribution, top-k and
|
| 860 |
+
// top-p rule out the unlikely tail, and an optional KV cache skips rerunning
|
| 861 |
+
// tokens that were processed on an earlier call.
|
| 862 |
+
predictNextToken(tokenIdArray, temperature = 1.0, topK = 50, topP = 0.9, cache = null) {
|
| 863 |
+
// With a cache, the stored tokens do not run again. Only the tokens
|
| 864 |
+
// past the cached length are new.
|
| 865 |
+
const newTokens = cache === null
|
| 866 |
+
? tokenIdArray
|
| 867 |
+
: tokenIdArray.slice(cache.length);
|
| 868 |
+
|
| 869 |
+
const logits = this.forward(newTokens, cache); // was forward(tokenIdArray)
|
| 870 |
+
|
| 871 |
+
// The last row holds the scores for the token after the sequence,
|
| 872 |
+
// whether that is the last row of a full prompt or the only row of a
|
| 873 |
+
// single cached step.
|
| 874 |
+
const lastRow = logits.rows - 1;
|
| 875 |
+
|
| 876 |
+
// Temperature zero means greedy decoding: the single highest logit.
|
| 877 |
+
// Softmax is monotonic, so the argmax of the logits is the argmax of
|
| 878 |
+
// the probabilities, and the softmax can be skipped here.
|
| 879 |
+
if (temperature === 0) {
|
| 880 |
+
let bestId = 0;
|
| 881 |
+
let bestScore = logits.get(lastRow, 0);
|
| 882 |
+
for (let col = 1; col < this.vocabSize; ++col) {
|
| 883 |
+
const score = logits.get(lastRow, col);
|
| 884 |
+
if (score > bestScore) {
|
| 885 |
+
bestScore = score;
|
| 886 |
+
bestId = col;
|
| 887 |
+
}
|
| 888 |
+
}
|
| 889 |
+
return bestId;
|
| 890 |
+
}
|
| 891 |
+
|
| 892 |
+
// Reshape by temperature and turn the last row into probabilities.
|
| 893 |
+
const scaled = logits.scale(1 / temperature);
|
| 894 |
+
const probabilities = scaled.softMaxedRows();
|
| 895 |
+
|
| 896 |
+
// Rank the tokens from most to least likely so the worst can be cut.
|
| 897 |
+
const ranked = [];
|
| 898 |
+
for (let col = 0; col < this.vocabSize; ++col) {
|
| 899 |
+
ranked.push({ id: col, probability: probabilities.get(lastRow, col) });
|
| 900 |
+
}
|
| 901 |
+
ranked.sort((a, b) => b.probability - a.probability);
|
| 902 |
+
|
| 903 |
+
// top-k: keep only the k most likely tokens.
|
| 904 |
+
const capped = ranked.slice(0, topK);
|
| 905 |
+
|
| 906 |
+
// top-p: from the top, keep tokens until their combined probability
|
| 907 |
+
// crosses topP, then stop. Always keeps at least the single top token.
|
| 908 |
+
const kept = [];
|
| 909 |
+
let keptTotal = 0.0;
|
| 910 |
+
for (let i = 0; i < capped.length; ++i) {
|
| 911 |
+
kept.push(capped[i]);
|
| 912 |
+
keptTotal += capped[i].probability;
|
| 913 |
+
if (keptTotal >= topP) {
|
| 914 |
+
break;
|
| 915 |
+
}
|
| 916 |
+
}
|
| 917 |
+
|
| 918 |
+
// The survivors cover only part of the number line now, up to keptTotal.
|
| 919 |
+
// Draw a random number up to that total, then walk them as before.
|
| 920 |
+
const draw = Math.random() * keptTotal;
|
| 921 |
+
let cumulative = 0.0;
|
| 922 |
+
for (let i = 0; i < kept.length; ++i) {
|
| 923 |
+
cumulative += kept[i].probability;
|
| 924 |
+
if (draw < cumulative) {
|
| 925 |
+
return kept[i].id;
|
| 926 |
+
}
|
| 927 |
+
}
|
| 928 |
+
|
| 929 |
+
return kept[kept.length - 1].id; // guard against floating-point drift
|
| 930 |
+
}
|
| 931 |
+
|
| 932 |
+
parameters() {
|
| 933 |
+
const params = [];
|
| 934 |
+
|
| 935 |
+
// Token embedding table (also used tied as the unembedding).
|
| 936 |
+
const tokenParams = this.tokenEmbeddings.parameters();
|
| 937 |
+
for (let i = 0; i < tokenParams.length; ++i) {
|
| 938 |
+
params.push(tokenParams[i]);
|
| 939 |
+
}
|
| 940 |
+
|
| 941 |
+
// Per-block parameters.
|
| 942 |
+
for (let i = 0; i < this.blocks.length; ++i) {
|
| 943 |
+
const blockParams = this.blocks[i].parameters();
|
| 944 |
+
for (let j = 0; j < blockParams.length; ++j) {
|
| 945 |
+
params.push(blockParams[j]);
|
| 946 |
+
}
|
| 947 |
+
}
|
| 948 |
+
|
| 949 |
+
// Final norm now carries a learned scale (gamma), gathered like the rest.
|
| 950 |
+
const normParams = this.finalNorm.parameters();
|
| 951 |
+
for (let i = 0; i < normParams.length; ++i) {
|
| 952 |
+
params.push(normParams[i]);
|
| 953 |
+
}
|
| 954 |
+
|
| 955 |
+
return params;
|
| 956 |
+
}
|
| 957 |
+
|
| 958 |
+
parameterCount() {
|
| 959 |
+
let total = 0;
|
| 960 |
+
for (const t of this.parameters()) {
|
| 961 |
+
total += t.data.length;
|
| 962 |
+
}
|
| 963 |
+
return total;
|
| 964 |
+
}
|
| 965 |
+
|
| 966 |
+
// Build from a model.json object, or from a weight file's header.
|
| 967 |
+
static fromConfig(config) {
|
| 968 |
+
return new GabGPT(
|
| 969 |
+
config.vocabSize, config.featureDim, config.numHeads, config.numKVHeads, config.headDim, config.ropeBase,
|
| 970 |
+
config.numBlocks, config.mlpHiddenDim, config.convKernel, config.attentionWindows,
|
| 971 |
+
);
|
| 972 |
+
}
|
| 973 |
+
|
| 974 |
+
// The architecture and shape, using model.json's field names. Written into
|
| 975 |
+
// every weight file's header, and enough for fromConfig to rebuild the model.
|
| 976 |
+
toConfig() {
|
| 977 |
+
return {
|
| 978 |
+
architecture: "chatgba",
|
| 979 |
+
vocabSize: this.vocabSize,
|
| 980 |
+
featureDim: this.featureDim,
|
| 981 |
+
numHeads: this.numHeads,
|
| 982 |
+
numKVHeads: this.numKVHeads,
|
| 983 |
+
headDim: this.headDim,
|
| 984 |
+
ropeBase: this.ropeBase,
|
| 985 |
+
numBlocks: this.numBlocks,
|
| 986 |
+
mlpHiddenDim: this.mlpHiddenDim,
|
| 987 |
+
convKernel: this.convKernel,
|
| 988 |
+
attentionWindows: this.attentionWindows,
|
| 989 |
+
parameterCount: this.parameterCount(),
|
| 990 |
+
};
|
| 991 |
+
}
|
| 992 |
+
|
| 993 |
+
// metadata is anything worth keeping with the weights, typically the
|
| 994 |
+
// model.json the model was built from plus training details. The shape
|
| 995 |
+
// fields always come from the model itself, so the header cannot lie.
|
| 996 |
+
serializeToArrayBuffer(metadata = {}) {
|
| 997 |
+
const weights = new Float32Array(this.parameterCount());
|
| 998 |
+
let offset = 0;
|
| 999 |
+
for (const t of this.parameters()) {
|
| 1000 |
+
weights.set(t.data, offset);
|
| 1001 |
+
offset += t.data.length;
|
| 1002 |
+
}
|
| 1003 |
+
return encodeWeightFile({ ...metadata, ...this.toConfig() }, weights);
|
| 1004 |
+
}
|
| 1005 |
+
|
| 1006 |
+
// Loads weights with the same layout. The file's own header is kept in
|
| 1007 |
+
// loadedHeader, so a run can always tell which weights it is running.
|
| 1008 |
+
deserializeFromArrayBuffer(buffer) {
|
| 1009 |
+
const { header, weights } = decodeWeightFile(buffer);
|
| 1010 |
+
checkWeightLayout(header, this.toConfig());
|
| 1011 |
+
let offset = 0;
|
| 1012 |
+
for (const t of this.parameters()) {
|
| 1013 |
+
t.data.set(weights.subarray(offset, offset + t.data.length));
|
| 1014 |
+
offset += t.data.length;
|
| 1015 |
+
}
|
| 1016 |
+
this.loadedHeader = header;
|
| 1017 |
+
}
|
| 1018 |
+
|
| 1019 |
+
// Rebuild the model a weight file describes, and load its weights.
|
| 1020 |
+
static fromArrayBuffer(buffer) {
|
| 1021 |
+
const { header } = decodeWeightFile(buffer);
|
| 1022 |
+
if (header.architecture !== "chatgba") {
|
| 1023 |
+
throw new Error(
|
| 1024 |
+
`this engine runs chatgba, but the file holds ${header.architecture} weights. ` +
|
| 1025 |
+
`To run them here anyway, build the model with fromConfig and call deserializeFromArrayBuffer.`
|
| 1026 |
+
);
|
| 1027 |
+
}
|
| 1028 |
+
const model = GabGPT.fromConfig(header);
|
| 1029 |
+
model.deserializeFromArrayBuffer(buffer);
|
| 1030 |
+
return model;
|
| 1031 |
+
}
|
| 1032 |
+
}
|
| 1033 |
+
|
| 1034 |
+
|
| 1035 |
+
// A key-value cache holds one key tensor and one value tensor for every block,
|
| 1036 |
+
// for every key/value head. The model reads and writes it but does not own it: the caller
|
| 1037 |
+
// creates one and passes it in, so a single GabGPT can serve many independent
|
| 1038 |
+
// generations at once, each with its own cache.
|
| 1039 |
+
export class KVCache {
|
| 1040 |
+
constructor(numBlocks, numKVHeads) {
|
| 1041 |
+
// One key tensor and one value tensor per block, per key/value head. Each starts
|
| 1042 |
+
// null and becomes a growing tensor as tokens flow through.
|
| 1043 |
+
this.keys = [];
|
| 1044 |
+
this.values = [];
|
| 1045 |
+
for (let block = 0; block < numBlocks; ++block) {
|
| 1046 |
+
const blockKeys = [];
|
| 1047 |
+
const blockValues = [];
|
| 1048 |
+
for (let head = 0; head < numKVHeads; ++head) {
|
| 1049 |
+
blockKeys.push(null);
|
| 1050 |
+
blockValues.push(null);
|
| 1051 |
+
}
|
| 1052 |
+
this.keys.push(blockKeys);
|
| 1053 |
+
this.values.push(blockValues);
|
| 1054 |
+
}
|
| 1055 |
+
|
| 1056 |
+
// Each block's two canon layers keep their last few inputs, so a new
|
| 1057 |
+
// token can be convolved with the ones before it.
|
| 1058 |
+
this.convA = new Array(numBlocks).fill(null);
|
| 1059 |
+
this.convC = new Array(numBlocks).fill(null);
|
| 1060 |
+
}
|
| 1061 |
+
|
| 1062 |
+
// Tokens cached so far. Every block and head advances together, so the
|
| 1063 |
+
// first head's key tensor speaks for all of them.
|
| 1064 |
+
get length() {
|
| 1065 |
+
const first = this.keys[0][0];
|
| 1066 |
+
return first === null ? 0 : first.rows;
|
| 1067 |
+
}
|
| 1068 |
+
}
|
| 1069 |
+
|
| 1070 |
+
|
| 1071 |
+
|
| 1072 |
+
// Muon for the matrices, AdamW for everything else, with the full set of
|
| 1073 |
+
// training features:
|
| 1074 |
+
// - Muon (MomentUm Orthogonalized by Newton-Schulz) for every attention and
|
| 1075 |
+
// MLP matrix: Nesterov momentum, then the update is orthogonalized, so it
|
| 1076 |
+
// moves every direction of the matrix by about the same amount instead of
|
| 1077 |
+
// letting a few large directions dominate. It is scaled to the size an
|
| 1078 |
+
// AdamW update would have (0.2 x sqrt(larger side)), so one learning rate
|
| 1079 |
+
// and one schedule serve both optimizers.
|
| 1080 |
+
// - AdamW, with bias correction, for the embedding table (with weight
|
| 1081 |
+
// decay), the norm scales and the canon filters (without)
|
| 1082 |
+
// - decoupled weight decay on the matrices and the embedding table
|
| 1083 |
+
// - linear warmup, then cosine decay to a floor learning rate
|
| 1084 |
+
// - global gradient-norm clipping
|
| 1085 |
+
// - mini-batches: gradients from every example are combined before a step
|
| 1086 |
+
// - loss masking: a 0/1 mask per example marks which targets are graded
|
| 1087 |
+
// - token-weighted batches: the batch loss is the mean over every graded
|
| 1088 |
+
// target in the batch, so a long example counts for more than a short one
|
| 1089 |
+
// Every hyperparameter is an option, so each model can use its own settings.
|
| 1090 |
+
export class MuonAdamWTrainer {
|
| 1091 |
+
constructor(model, {
|
| 1092 |
+
maxLearningRate = 3e-4,
|
| 1093 |
+
minLearningRate = 3e-5,
|
| 1094 |
+
warmupSteps = 100,
|
| 1095 |
+
totalSteps = 10000,
|
| 1096 |
+
beta1 = 0.9,
|
| 1097 |
+
beta2 = 0.95, // 0.95, not 0.999: adapts faster when gradient scale shifts, the usual choice for language models
|
| 1098 |
+
epsilon = 1e-8,
|
| 1099 |
+
weightDecay = 0.1,
|
| 1100 |
+
gradientClip = 1.0,
|
| 1101 |
+
optimizer = "muon", // "muon", or "adamw" to train the matrices with AdamW too
|
| 1102 |
+
muonMomentum = 0.95,
|
| 1103 |
+
muonSteps = 5, // Newton-Schulz iterations
|
| 1104 |
+
} = {}) {
|
| 1105 |
+
this.model = model;
|
| 1106 |
+
|
| 1107 |
+
this.maxLearningRate = maxLearningRate;
|
| 1108 |
+
this.minLearningRate = minLearningRate;
|
| 1109 |
+
this.warmupSteps = warmupSteps;
|
| 1110 |
+
this.totalSteps = totalSteps;
|
| 1111 |
+
|
| 1112 |
+
this.beta1 = beta1;
|
| 1113 |
+
this.beta2 = beta2;
|
| 1114 |
+
this.epsilon = epsilon;
|
| 1115 |
+
this.weightDecay = weightDecay;
|
| 1116 |
+
this.gradientClip = gradientClip;
|
| 1117 |
+
this.muonMomentum = muonMomentum;
|
| 1118 |
+
this.muonSteps = muonSteps;
|
| 1119 |
+
|
| 1120 |
+
// Grab the parameter list once. The tensors inside are stable references,
|
| 1121 |
+
// so the moment buffers below stay matched to them for the whole run.
|
| 1122 |
+
this.params = model.parameters();
|
| 1123 |
+
|
| 1124 |
+
// Each tensor's role decides its update. Matrices go to Muon. Weight
|
| 1125 |
+
// decay pulls weights toward zero, which suits matrices and the
|
| 1126 |
+
// embedding table, but not norm scales or filters: each of their values
|
| 1127 |
+
// has one job, and decaying it just fights that job.
|
| 1128 |
+
this.usesMuon = this.params.map((t) => optimizer === "muon" && t.role === "matrix");
|
| 1129 |
+
this.decays = this.params.map((t) => t.role === "matrix" || t.role === "embedding");
|
| 1130 |
+
|
| 1131 |
+
// Adam keeps two running averages per parameter, Muon one momentum
|
| 1132 |
+
// buffer, each the same shape as the parameter. A fresh Float32Array is
|
| 1133 |
+
// already all zeros.
|
| 1134 |
+
this.firstMoments = [];
|
| 1135 |
+
this.secondMoments = [];
|
| 1136 |
+
for (let i = 0; i < this.params.length; ++i) {
|
| 1137 |
+
const length = this.params[i].data.length;
|
| 1138 |
+
this.firstMoments.push(new Float32Array(length)); // m, or Muon's momentum
|
| 1139 |
+
this.secondMoments.push(this.usesMuon[i] ? null : new Float32Array(length)); // v
|
| 1140 |
+
}
|
| 1141 |
+
|
| 1142 |
+
// Updates applied so far. Drives both bias correction and the schedule.
|
| 1143 |
+
this.step = 0;
|
| 1144 |
+
|
| 1145 |
+
// Exposed for logging.
|
| 1146 |
+
this.lastLearningRate = 0;
|
| 1147 |
+
this.lastGradientNorm = 0;
|
| 1148 |
+
}
|
| 1149 |
+
|
| 1150 |
+
// Fused mean cross-entropy over the graded rows. The forward pass uses the
|
| 1151 |
+
// log-sum-exp form for each row of logits, which avoids exponentiating large
|
| 1152 |
+
// values directly: for a row with correct token t, the loss is
|
| 1153 |
+
// log(sum_j e^{x_j}) - x_t. The softmax probabilities are cached, because
|
| 1154 |
+
// the backward rule reuses them.
|
| 1155 |
+
crossEntropyLoss(logits, targetIds, targetMask = null) {
|
| 1156 |
+
const rows = logits.rows;
|
| 1157 |
+
const cols = logits.columns;
|
| 1158 |
+
const probs = new Float32Array(rows * cols); // cached for the backward
|
| 1159 |
+
let total = 0.0;
|
| 1160 |
+
let count = 0; // graded rows, the divisor for the mean
|
| 1161 |
+
|
| 1162 |
+
for (let row = 0; row < rows; ++row) {
|
| 1163 |
+
// A context-only row contributes no loss and no gradient.
|
| 1164 |
+
if (targetMask !== null && !targetMask[row]) {
|
| 1165 |
+
continue;
|
| 1166 |
+
}
|
| 1167 |
+
|
| 1168 |
+
const base = row * cols;
|
| 1169 |
+
|
| 1170 |
+
let rowMax = logits.data[base];
|
| 1171 |
+
for (let col = 1; col < cols; ++col) {
|
| 1172 |
+
const v = logits.data[base + col];
|
| 1173 |
+
if (v > rowMax) rowMax = v;
|
| 1174 |
+
}
|
| 1175 |
+
|
| 1176 |
+
let sum = 0.0;
|
| 1177 |
+
for (let col = 0; col < cols; ++col) {
|
| 1178 |
+
const e = Math.exp(logits.data[base + col] - rowMax);
|
| 1179 |
+
probs[base + col] = e;
|
| 1180 |
+
sum += e;
|
| 1181 |
+
}
|
| 1182 |
+
for (let col = 0; col < cols; ++col) {
|
| 1183 |
+
probs[base + col] /= sum;
|
| 1184 |
+
}
|
| 1185 |
+
|
| 1186 |
+
const logSumExp = rowMax + Math.log(sum);
|
| 1187 |
+
total += logSumExp - logits.data[base + targetIds[row]];
|
| 1188 |
+
count += 1;
|
| 1189 |
+
}
|
| 1190 |
+
|
| 1191 |
+
if (count === 0) {
|
| 1192 |
+
throw new Error("No unmasked targets were provided");
|
| 1193 |
+
}
|
| 1194 |
+
|
| 1195 |
+
const loss = new Tensor(1, 1);
|
| 1196 |
+
loss.data[0] = total / count;
|
| 1197 |
+
|
| 1198 |
+
if (Tensor.gradEnabled) {
|
| 1199 |
+
loss._inputs = [logits];
|
| 1200 |
+
loss._backward = () => {
|
| 1201 |
+
const seed = loss.grad[0] / count;
|
| 1202 |
+
for (let row = 0; row < rows; ++row) {
|
| 1203 |
+
if (targetMask !== null && !targetMask[row]) {
|
| 1204 |
+
continue;
|
| 1205 |
+
}
|
| 1206 |
+
const base = row * cols;
|
| 1207 |
+
for (let col = 0; col < cols; ++col) {
|
| 1208 |
+
logits.grad[base + col] += seed * probs[base + col];
|
| 1209 |
+
}
|
| 1210 |
+
logits.grad[base + targetIds[row]] -= seed;
|
| 1211 |
+
}
|
| 1212 |
+
};
|
| 1213 |
+
}
|
| 1214 |
+
return loss;
|
| 1215 |
+
}
|
| 1216 |
+
|
| 1217 |
+
learningRate() {
|
| 1218 |
+
const step = this.step;
|
| 1219 |
+
|
| 1220 |
+
// Warmup: ramp linearly from zero to the peak over the first steps.
|
| 1221 |
+
if (step < this.warmupSteps) {
|
| 1222 |
+
return this.maxLearningRate * (step / this.warmupSteps);
|
| 1223 |
+
}
|
| 1224 |
+
|
| 1225 |
+
// Past the planned end: hold at the floor.
|
| 1226 |
+
if (step >= this.totalSteps) {
|
| 1227 |
+
return this.minLearningRate;
|
| 1228 |
+
}
|
| 1229 |
+
|
| 1230 |
+
// Cosine decay: ease from the peak down to the floor.
|
| 1231 |
+
const progress = (step - this.warmupSteps) / (this.totalSteps - this.warmupSteps);
|
| 1232 |
+
const cosine = 0.5 * (1 + Math.cos(Math.PI * progress));
|
| 1233 |
+
return this.minLearningRate + (this.maxLearningRate - this.minLearningRate) * cosine;
|
| 1234 |
+
}
|
| 1235 |
+
|
| 1236 |
+
clipGradients() {
|
| 1237 |
+
// Global L2 norm across every gradient in the model.
|
| 1238 |
+
let sumSquares = 0.0;
|
| 1239 |
+
for (let i = 0; i < this.params.length; ++i) {
|
| 1240 |
+
const grad = this.params[i].grad;
|
| 1241 |
+
for (let j = 0; j < grad.length; ++j) {
|
| 1242 |
+
sumSquares += grad[j] * grad[j];
|
| 1243 |
+
}
|
| 1244 |
+
}
|
| 1245 |
+
const norm = Math.sqrt(sumSquares);
|
| 1246 |
+
|
| 1247 |
+
// Only scale down when the norm is over the threshold.
|
| 1248 |
+
if (norm > this.gradientClip) {
|
| 1249 |
+
const scale = this.gradientClip / norm;
|
| 1250 |
+
for (let i = 0; i < this.params.length; ++i) {
|
| 1251 |
+
const grad = this.params[i].grad;
|
| 1252 |
+
for (let j = 0; j < grad.length; ++j) {
|
| 1253 |
+
grad[j] *= scale;
|
| 1254 |
+
}
|
| 1255 |
+
}
|
| 1256 |
+
}
|
| 1257 |
+
|
| 1258 |
+
return norm; // before clipping, for logging
|
| 1259 |
+
}
|
| 1260 |
+
|
| 1261 |
+
train(tokenIdBatch, tokenMaskBatch = null) {
|
| 1262 |
+
const counts = gradedTargetCounts(tokenIdBatch, tokenMaskBatch);
|
| 1263 |
+
const totalCount = counts.reduce((a, b) => a + b, 0);
|
| 1264 |
+
if (totalCount === 0) {
|
| 1265 |
+
throw new Error("Mini-batch has no graded targets");
|
| 1266 |
+
}
|
| 1267 |
+
|
| 1268 |
+
// Clear gradients once, then let every training example add to them.
|
| 1269 |
+
for (let i = 0; i < this.params.length; ++i) {
|
| 1270 |
+
this.params[i].zeroGrad();
|
| 1271 |
+
}
|
| 1272 |
+
|
| 1273 |
+
// Each example's loss is a mean over its own graded targets. Scaling it
|
| 1274 |
+
// by that example's share of the batch's graded targets makes the sum
|
| 1275 |
+
// of the scaled losses the mean over every graded target in the batch.
|
| 1276 |
+
let batchLoss = 0.0;
|
| 1277 |
+
for (let batchIndex = 0; batchIndex < tokenIdBatch.length; ++batchIndex) {
|
| 1278 |
+
if (counts[batchIndex] === 0) {
|
| 1279 |
+
continue; // nothing to grade: the example would add nothing
|
| 1280 |
+
}
|
| 1281 |
+
|
| 1282 |
+
const tokenIds = tokenIdBatch[batchIndex];
|
| 1283 |
+
const tokenMask = tokenMaskBatch === null ? null : tokenMaskBatch[batchIndex];
|
| 1284 |
+
|
| 1285 |
+
const inputIds = tokenIds.slice(0, tokenIds.length - 1);
|
| 1286 |
+
const targetIds = tokenIds.slice(1);
|
| 1287 |
+
// The mask lines up with tokenIds, so shift it like the targets. The
|
| 1288 |
+
// first token is never a target, so its mask is dropped.
|
| 1289 |
+
const targetMask = tokenMask === null ? null : tokenMask.slice(1);
|
| 1290 |
+
|
| 1291 |
+
const logits = this.model.forward(inputIds);
|
| 1292 |
+
const loss = this.crossEntropyLoss(logits, targetIds, targetMask)
|
| 1293 |
+
.scale(counts[batchIndex] / totalCount);
|
| 1294 |
+
loss.backward();
|
| 1295 |
+
batchLoss += loss.data[0];
|
| 1296 |
+
}
|
| 1297 |
+
|
| 1298 |
+
// Rein in the gradient before it moves anything.
|
| 1299 |
+
this.lastGradientNorm = this.clipGradients();
|
| 1300 |
+
|
| 1301 |
+
// Advance the step, then read this step's scheduled learning rate.
|
| 1302 |
+
this.step += 1;
|
| 1303 |
+
const learningRate = this.learningRate();
|
| 1304 |
+
this.lastLearningRate = learningRate;
|
| 1305 |
+
|
| 1306 |
+
const beta1 = this.beta1;
|
| 1307 |
+
const beta2 = this.beta2;
|
| 1308 |
+
const epsilon = this.epsilon;
|
| 1309 |
+
|
| 1310 |
+
// Bias-correction denominators for this step.
|
| 1311 |
+
const correction1 = 1 - Math.pow(beta1, this.step);
|
| 1312 |
+
const correction2 = 1 - Math.pow(beta2, this.step);
|
| 1313 |
+
|
| 1314 |
+
// Muon or AdamW update, parameter by parameter.
|
| 1315 |
+
for (let i = 0; i < this.params.length; ++i) {
|
| 1316 |
+
const param = this.params[i];
|
| 1317 |
+
if (this.usesMuon[i]) {
|
| 1318 |
+
this.muonUpdate(param, this.firstMoments[i], learningRate);
|
| 1319 |
+
continue;
|
| 1320 |
+
}
|
| 1321 |
+
const m = this.firstMoments[i];
|
| 1322 |
+
const v = this.secondMoments[i];
|
| 1323 |
+
const weightDecay = this.decays[i] ? this.weightDecay : 0;
|
| 1324 |
+
|
| 1325 |
+
for (let j = 0; j < param.data.length; ++j) {
|
| 1326 |
+
const g = param.grad[j];
|
| 1327 |
+
const weight = param.data[j];
|
| 1328 |
+
|
| 1329 |
+
// Update the running averages of the gradient and its square.
|
| 1330 |
+
m[j] = beta1 * m[j] + (1 - beta1) * g;
|
| 1331 |
+
v[j] = beta2 * v[j] + (1 - beta2) * g * g;
|
| 1332 |
+
|
| 1333 |
+
// Undo the startup bias toward zero.
|
| 1334 |
+
const mHat = m[j] / correction1;
|
| 1335 |
+
const vHat = v[j] / correction2;
|
| 1336 |
+
|
| 1337 |
+
// The Adam step: smoothed direction, per-parameter scale.
|
| 1338 |
+
const adamStep = mHat / (Math.sqrt(vHat) + epsilon);
|
| 1339 |
+
|
| 1340 |
+
// Decoupled weight decay: pull the weight toward zero on its
|
| 1341 |
+
// own, not routed through the adaptive scaling above.
|
| 1342 |
+
param.data[j] = weight - learningRate * (adamStep + weightDecay * weight);
|
| 1343 |
+
}
|
| 1344 |
+
}
|
| 1345 |
+
|
| 1346 |
+
return batchLoss;
|
| 1347 |
+
}
|
| 1348 |
+
|
| 1349 |
+
muonUpdate(param, momentum, learningRate) {
|
| 1350 |
+
const mu = this.muonMomentum;
|
| 1351 |
+
const rows = param.rows;
|
| 1352 |
+
const cols = param.columns;
|
| 1353 |
+
|
| 1354 |
+
// Nesterov momentum: keep a running sum of gradients, then step along
|
| 1355 |
+
// the gradient plus a look-ahead share of that sum.
|
| 1356 |
+
const direction = new Float32Array(param.data.length);
|
| 1357 |
+
for (let j = 0; j < param.data.length; ++j) {
|
| 1358 |
+
momentum[j] = mu * momentum[j] + param.grad[j];
|
| 1359 |
+
direction[j] = param.grad[j] + mu * momentum[j];
|
| 1360 |
+
}
|
| 1361 |
+
|
| 1362 |
+
const orthogonal = newtonSchulz(direction, rows, cols, this.muonSteps);
|
| 1363 |
+
|
| 1364 |
+
// Scale to an AdamW-sized update, so both share one learning rate.
|
| 1365 |
+
const scale = 0.2 * Math.sqrt(Math.max(rows, cols));
|
| 1366 |
+
for (let j = 0; j < param.data.length; ++j) {
|
| 1367 |
+
const weight = param.data[j];
|
| 1368 |
+
param.data[j] = weight - learningRate * (scale * orthogonal[j] + this.weightDecay * weight);
|
| 1369 |
+
}
|
| 1370 |
+
}
|
| 1371 |
+
}
|
| 1372 |
+
|
| 1373 |
+
// Newton-Schulz iteration: pushes every singular value of a matrix toward one,
|
| 1374 |
+
// keeping its singular vectors. The result is close to the nearest orthogonal
|
| 1375 |
+
// matrix (U V^T from the SVD) at the cost of a few matrix products. The
|
| 1376 |
+
// quintic's coefficients (3.4445, -4.7750, 2.0315) trade exactness for speed:
|
| 1377 |
+
// singular values land roughly between 0.7 and 1.2, which is all Muon needs.
|
| 1378 |
+
// The matrix is first scaled to Frobenius norm one, so every singular value
|
| 1379 |
+
// starts at most one. A tall matrix is worked on transposed, which keeps the
|
| 1380 |
+
// Gram matrix X X^T small.
|
| 1381 |
+
function newtonSchulz(values, rows, cols, steps) {
|
| 1382 |
+
const a = 3.4445, b = -4.7750, c = 2.0315;
|
| 1383 |
+
const transpose = rows > cols;
|
| 1384 |
+
const m = transpose ? cols : rows; // X is m x n with m <= n
|
| 1385 |
+
const n = transpose ? rows : cols;
|
| 1386 |
+
|
| 1387 |
+
let x = new Float32Array(m * n);
|
| 1388 |
+
let norm = 0.0;
|
| 1389 |
+
for (let r = 0; r < rows; ++r) {
|
| 1390 |
+
for (let k = 0; k < cols; ++k) {
|
| 1391 |
+
const v = values[r * cols + k];
|
| 1392 |
+
norm += v * v;
|
| 1393 |
+
if (transpose) x[k * n + r] = v; else x[r * n + k] = v;
|
| 1394 |
+
}
|
| 1395 |
+
}
|
| 1396 |
+
norm = Math.sqrt(norm) + 1e-7;
|
| 1397 |
+
for (let i = 0; i < x.length; ++i) x[i] /= norm;
|
| 1398 |
+
|
| 1399 |
+
const multiply = (left, right, p, q, s) => { // [p, q] x [q, s]
|
| 1400 |
+
const out = new Float32Array(p * s);
|
| 1401 |
+
for (let i = 0; i < p; ++i) {
|
| 1402 |
+
for (let k = 0; k < q; ++k) {
|
| 1403 |
+
const l = left[i * q + k];
|
| 1404 |
+
for (let j = 0; j < s; ++j) out[i * s + j] += l * right[k * s + j];
|
| 1405 |
+
}
|
| 1406 |
+
}
|
| 1407 |
+
return out;
|
| 1408 |
+
};
|
| 1409 |
+
|
| 1410 |
+
for (let step = 0; step < steps; ++step) {
|
| 1411 |
+
const xT = new Float32Array(n * m);
|
| 1412 |
+
for (let i = 0; i < m; ++i) for (let j = 0; j < n; ++j) xT[j * m + i] = x[i * n + j];
|
| 1413 |
+
const A = multiply(x, xT, m, n, m); // X X^T, [m, m]
|
| 1414 |
+
const AA = multiply(A, A, m, m, m);
|
| 1415 |
+
const B = new Float32Array(m * m);
|
| 1416 |
+
for (let i = 0; i < B.length; ++i) B[i] = b * A[i] + c * AA[i];
|
| 1417 |
+
const BX = multiply(B, x, m, m, n);
|
| 1418 |
+
for (let i = 0; i < x.length; ++i) x[i] = a * x[i] + BX[i];
|
| 1419 |
+
}
|
| 1420 |
+
|
| 1421 |
+
if (!transpose) return x;
|
| 1422 |
+
const out = new Float32Array(rows * cols);
|
| 1423 |
+
for (let k = 0; k < m; ++k) for (let r = 0; r < n; ++r) out[r * cols + k] = x[k * n + r];
|
| 1424 |
+
return out;
|
| 1425 |
+
}
|
| 1426 |
+
|
| 1427 |
+
// Checks a mini-batch and counts each example's graded targets: the positions
|
| 1428 |
+
// after the first whose mask is 1, or every position after the first when
|
| 1429 |
+
// there is no mask. The trainer weights each example's loss by its count.
|
| 1430 |
+
function gradedTargetCounts(tokenIdBatch, tokenMaskBatch) {
|
| 1431 |
+
if (!Array.isArray(tokenIdBatch) || tokenIdBatch.length === 0) {
|
| 1432 |
+
throw new Error("Mini-batch must contain at least one training example");
|
| 1433 |
+
}
|
| 1434 |
+
if (tokenMaskBatch !== null && tokenMaskBatch.length !== tokenIdBatch.length) {
|
| 1435 |
+
throw new Error(
|
| 1436 |
+
`mask batch length ${tokenMaskBatch.length} does not match ` +
|
| 1437 |
+
`token batch length ${tokenIdBatch.length}`
|
| 1438 |
+
);
|
| 1439 |
+
}
|
| 1440 |
+
|
| 1441 |
+
const counts = [];
|
| 1442 |
+
for (let batchIndex = 0; batchIndex < tokenIdBatch.length; ++batchIndex) {
|
| 1443 |
+
const tokenIds = tokenIdBatch[batchIndex];
|
| 1444 |
+
const tokenMask = tokenMaskBatch === null ? null : tokenMaskBatch[batchIndex];
|
| 1445 |
+
|
| 1446 |
+
if (tokenIds.length < 2) {
|
| 1447 |
+
throw new Error(`training example ${batchIndex} needs at least 2 tokens, got ${tokenIds.length}`);
|
| 1448 |
+
}
|
| 1449 |
+
if (tokenMask !== null && tokenMask.length !== tokenIds.length) {
|
| 1450 |
+
throw new Error(
|
| 1451 |
+
`mask length ${tokenMask.length} does not match ` +
|
| 1452 |
+
`token length ${tokenIds.length} for example ${batchIndex}`
|
| 1453 |
+
);
|
| 1454 |
+
}
|
| 1455 |
+
|
| 1456 |
+
let count = tokenIds.length - 1;
|
| 1457 |
+
if (tokenMask !== null) {
|
| 1458 |
+
count = 0;
|
| 1459 |
+
for (let i = 1; i < tokenMask.length; ++i) {
|
| 1460 |
+
if (tokenMask[i]) count += 1;
|
| 1461 |
+
}
|
| 1462 |
+
}
|
| 1463 |
+
counts.push(count);
|
| 1464 |
+
}
|
| 1465 |
+
return counts;
|
| 1466 |
+
}
|
| 1467 |
+
|
| 1468 |
+
// --- Weight files ---------------------------------------------------------
|
| 1469 |
+
// A weight file carries everything needed to rebuild the model that wrote it:
|
| 1470 |
+
//
|
| 1471 |
+
// bytes 0-3 magic "GAB_"
|
| 1472 |
+
// bytes 4-7 format version, uint32 little-endian (currently 1)
|
| 1473 |
+
// bytes 8-11 header length in bytes, uint32 little-endian, a multiple of 4
|
| 1474 |
+
// header UTF-8 JSON, space-padded: architecture, shape, parameterCount,
|
| 1475 |
+
// plus any metadata the writer adds (chat markers, training info)
|
| 1476 |
+
// weights float32 little-endian, in parameters() order
|
| 1477 |
+
//
|
| 1478 |
+
// Padding the header to a multiple of 4 keeps the weights 4-byte aligned, so
|
| 1479 |
+
// they can be read as a Float32Array in place.
|
| 1480 |
+
|
| 1481 |
+
const WEIGHT_MAGIC = "GAB_";
|
| 1482 |
+
const WEIGHT_VERSION = 1;
|
| 1483 |
+
|
| 1484 |
+
function assertLittleEndian() {
|
| 1485 |
+
if (new Uint8Array(new Uint32Array([1]).buffer)[0] !== 1) {
|
| 1486 |
+
throw new Error("weight files are little-endian, and this platform is not");
|
| 1487 |
+
}
|
| 1488 |
+
}
|
| 1489 |
+
|
| 1490 |
+
export function encodeWeightFile(header, weights) { // weights: Float32Array
|
| 1491 |
+
assertLittleEndian();
|
| 1492 |
+
const json = JSON.stringify(header);
|
| 1493 |
+
let headerBytes = new TextEncoder().encode(json);
|
| 1494 |
+
const padding = (4 - (headerBytes.length % 4)) % 4;
|
| 1495 |
+
if (padding > 0) {
|
| 1496 |
+
headerBytes = new TextEncoder().encode(json + " ".repeat(padding));
|
| 1497 |
+
}
|
| 1498 |
+
|
| 1499 |
+
const buffer = new ArrayBuffer(12 + headerBytes.length + weights.length * 4);
|
| 1500 |
+
const bytes = new Uint8Array(buffer);
|
| 1501 |
+
const view = new DataView(buffer);
|
| 1502 |
+
bytes.set(new TextEncoder().encode(WEIGHT_MAGIC), 0);
|
| 1503 |
+
view.setUint32(4, WEIGHT_VERSION, true);
|
| 1504 |
+
view.setUint32(8, headerBytes.length, true);
|
| 1505 |
+
bytes.set(headerBytes, 12);
|
| 1506 |
+
new Float32Array(buffer, 12 + headerBytes.length, weights.length).set(weights);
|
| 1507 |
+
return buffer;
|
| 1508 |
+
}
|
| 1509 |
+
|
| 1510 |
+
// Accepts an ArrayBuffer, or any typed array over one (such as the Buffer
|
| 1511 |
+
// that Node's readFileSync returns). Returns { header, weights }.
|
| 1512 |
+
export function decodeWeightFile(buffer) {
|
| 1513 |
+
assertLittleEndian();
|
| 1514 |
+
if (ArrayBuffer.isView(buffer)) {
|
| 1515 |
+
// Copy into a fresh buffer: a view can start at an unaligned offset.
|
| 1516 |
+
buffer = buffer.buffer.slice(buffer.byteOffset, buffer.byteOffset + buffer.byteLength);
|
| 1517 |
+
}
|
| 1518 |
+
if (buffer.byteLength < 12 || new TextDecoder().decode(new Uint8Array(buffer, 0, 4)) !== WEIGHT_MAGIC) {
|
| 1519 |
+
throw new Error("not a GabGPT weight file: missing the GAB_ magic");
|
| 1520 |
+
}
|
| 1521 |
+
|
| 1522 |
+
const view = new DataView(buffer);
|
| 1523 |
+
const version = view.getUint32(4, true);
|
| 1524 |
+
if (version !== WEIGHT_VERSION) {
|
| 1525 |
+
throw new Error(`unsupported weight file version ${version}`);
|
| 1526 |
+
}
|
| 1527 |
+
|
| 1528 |
+
const headerLength = view.getUint32(8, true);
|
| 1529 |
+
const header = JSON.parse(new TextDecoder().decode(new Uint8Array(buffer, 12, headerLength)));
|
| 1530 |
+
|
| 1531 |
+
const weightCount = (buffer.byteLength - 12 - headerLength) / 4;
|
| 1532 |
+
if (weightCount !== header.parameterCount) {
|
| 1533 |
+
throw new Error(`weight file holds ${weightCount} values, but its header says ${header.parameterCount}`);
|
| 1534 |
+
}
|
| 1535 |
+
return { header, weights: new Float32Array(buffer, 12 + headerLength, weightCount) };
|
| 1536 |
+
}
|
| 1537 |
+
|
| 1538 |
+
// Two architectures can share weights when their tensors line up one for one:
|
| 1539 |
+
// the same family, widths and number of unique blocks. A looped model and its
|
| 1540 |
+
// plain counterpart qualify.
|
| 1541 |
+
function weightLayout(config) {
|
| 1542 |
+
const looped = config.architecture.endsWith("_looped");
|
| 1543 |
+
return {
|
| 1544 |
+
family: config.architecture.replace("_looped", ""),
|
| 1545 |
+
vocabSize: config.vocabSize,
|
| 1546 |
+
featureDim: config.featureDim,
|
| 1547 |
+
numHeads: config.numHeads ?? null,
|
| 1548 |
+
numKVHeads: config.numKVHeads ?? null, // ChatGBA only
|
| 1549 |
+
headDim: config.headDim ?? null, // ChatGBA only
|
| 1550 |
+
mlpHiddenDim: config.mlpHiddenDim ?? null,
|
| 1551 |
+
convKernel: config.convKernel ?? null,
|
| 1552 |
+
uniqueBlocks: looped
|
| 1553 |
+
? config.numPreludeBlocks + config.numCoreBlocks + config.numCodaBlocks
|
| 1554 |
+
: config.numBlocks,
|
| 1555 |
+
parameterCount: config.parameterCount,
|
| 1556 |
+
};
|
| 1557 |
+
}
|
| 1558 |
+
|
| 1559 |
+
function checkWeightLayout(fileConfig, modelConfig) {
|
| 1560 |
+
const file = weightLayout(fileConfig);
|
| 1561 |
+
const model = weightLayout(modelConfig);
|
| 1562 |
+
for (const key of Object.keys(model)) {
|
| 1563 |
+
if (file[key] !== model[key]) {
|
| 1564 |
+
throw new Error(
|
| 1565 |
+
`${fileConfig.architecture} weights do not fit a ${modelConfig.architecture} model: ` +
|
| 1566 |
+
`${key} is ${file[key]} in the file and ${model[key]} in the model`
|
| 1567 |
+
);
|
| 1568 |
+
}
|
| 1569 |
+
}
|
| 1570 |
+
}
|
source/Makefile
ADDED
|
@@ -0,0 +1,40 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
# ChatGBA, the Game Boy Advance ROM.
|
| 2 |
+
#
|
| 3 |
+
# make
|
| 4 |
+
#
|
| 5 |
+
# builds BUILD/ChatGBA.gba from these sources and the quantized weights,
|
| 6 |
+
# ../chatgba-int8.bin (model_config.h describes them). The result matches the
|
| 7 |
+
# ChatGBA.gba next to this folder byte for byte.
|
| 8 |
+
# Tools: arm-none-eabi-gcc (brew install arm-none-eabi-gcc) and python3.
|
| 9 |
+
|
| 10 |
+
WEIGHTS ?= ../chatgba-int8.bin
|
| 11 |
+
BUILD ?= build
|
| 12 |
+
GBA_CC = arm-none-eabi-gcc
|
| 13 |
+
GBA_FLAGS = -mcpu=arm7tdmi -mthumb -mthumb-interwork -O2 -ffreestanding -fno-strict-aliasing \
|
| 14 |
+
-fno-builtin -fno-tree-loop-distribute-patterns -D__GBA__ -Wall -Wextra -I. -Igba_include
|
| 15 |
+
|
| 16 |
+
all: $(BUILD)/ChatGBA.gba
|
| 17 |
+
|
| 18 |
+
# blob.s includes the weights as model.bin.
|
| 19 |
+
$(BUILD)/model.bin: $(WEIGHTS)
|
| 20 |
+
mkdir -p $(BUILD)
|
| 21 |
+
cp $< $@
|
| 22 |
+
|
| 23 |
+
# The engine runs as ARM code from the fast internal RAM: compiled as ARM, its
|
| 24 |
+
# .text renamed so the linker places it there (crt0.s copies it in from ROM).
|
| 25 |
+
# Everything calls across the gap between ROM and internal RAM with long calls.
|
| 26 |
+
$(BUILD)/engine.o: engine.c engine.h model_config.h
|
| 27 |
+
mkdir -p $(BUILD)
|
| 28 |
+
$(GBA_CC) $(GBA_FLAGS) -marm -mlong-calls -c engine.c -o $@
|
| 29 |
+
arm-none-eabi-objcopy --rename-section .text=.iwram.text $@
|
| 30 |
+
|
| 31 |
+
$(BUILD)/chatgba.elf: crt0.s blob.s dot.s main.c chat.c libc.c chat.h font.h gba.ld model_config.h $(BUILD)/engine.o $(BUILD)/model.bin
|
| 32 |
+
$(GBA_CC) $(GBA_FLAGS) -mlong-calls -nostdlib -nostartfiles -T gba.ld -Wl,--no-warn-rwx-segments -o $@ \
|
| 33 |
+
crt0.s -Wa,-I$(BUILD) blob.s dot.s main.c chat.c libc.c $(BUILD)/engine.o -lgcc
|
| 34 |
+
|
| 35 |
+
# The Nintendo logo and header checksum, so real hardware boots it.
|
| 36 |
+
$(BUILD)/ChatGBA.gba: $(BUILD)/chatgba.elf fix_header.py
|
| 37 |
+
arm-none-eabi-objcopy -O binary $< $@
|
| 38 |
+
python3 fix_header.py $@
|
| 39 |
+
|
| 40 |
+
.PHONY: all
|
source/README.md
ADDED
|
@@ -0,0 +1,31 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
# ChatGBA source
|
| 2 |
+
|
| 3 |
+
The Game Boy Advance inference engine and chat program behind `ChatGBA.gba`.
|
| 4 |
+
|
| 5 |
+
## Build
|
| 6 |
+
|
| 7 |
+
```
|
| 8 |
+
brew install arm-none-eabi-gcc # or any arm-none-eabi toolchain
|
| 9 |
+
make
|
| 10 |
+
```
|
| 11 |
+
|
| 12 |
+
This writes `build/ChatGBA.gba`, byte for byte the same as `../ChatGBA.gba`. The weights come from `../chatgba-int8.bin`; `make WEIGHTS=other.bin` builds with another blob of the same shape.
|
| 13 |
+
|
| 14 |
+
## Files
|
| 15 |
+
|
| 16 |
+
| File | What it is |
|
| 17 |
+
|---|---|
|
| 18 |
+
| `engine.c`, `engine.h` | The integer inference engine: int8 weights read straight from cartridge ROM, an int8 KV cache in a ring per block, fixed-point everything. Compiled as ARM and run from the fast internal RAM. |
|
| 19 |
+
| `dot.s` | The hand-written ARM inner loop: int8 dot products, 16 weights per load burst. |
|
| 20 |
+
| `chat.c`, `chat.h` | The conversation: `<|u|>message<|e|><|a|>reply<|e|>` turns through the engine, sampling and stopping at `<|e|>`. |
|
| 21 |
+
| `main.c` | The program: the screen, the on-screen keyboard, the buttons, timing. |
|
| 22 |
+
| `font.h` | The 6 × 10 font (Monaco at 9 px, with hand-drawn descenders). `font.py` regenerates it (needs Pillow). |
|
| 23 |
+
| `model_config.h` | The model's shape and chat markers as compile-time constants, checked against the weights at start-up. |
|
| 24 |
+
| `blob.s` | Places the weights (`model.bin`) in ROM. |
|
| 25 |
+
| `crt0.s`, `gba.ld` | Start-up code and the memory map: ROM, the 256 KB of external work RAM (the KV cache), the 32 KB of internal RAM (the engine). |
|
| 26 |
+
| `libc.c`, `gba_include/string.h` | The four C library functions the program needs; there is no C library. |
|
| 27 |
+
| `fix_header.py` | Writes the Nintendo logo and header checksum into the ROM, so real hardware boots it. |
|
| 28 |
+
|
| 29 |
+
## The weights
|
| 30 |
+
|
| 31 |
+
`chatgba-int8.bin` is the float32 model (`../chatgba-float32.gab`) quantized for this engine: every matrix as int8 with one scale per output row, norm scales and canon filters in Q16, plus lookup tables for RoPE, softmax and SiLU. The quantizer is part of the training code (not included here), since it reads the float model with it; it also writes `model_config.h`.
|
source/blob.s
ADDED
|
@@ -0,0 +1,6 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
@ The weights and tables from export.py, in cartridge ROM.
|
| 2 |
+
.section .rodata.blob, "a"
|
| 3 |
+
.balign 4
|
| 4 |
+
.global model_blob
|
| 5 |
+
model_blob:
|
| 6 |
+
.incbin "model.bin"
|
source/chat.c
ADDED
|
@@ -0,0 +1,84 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
// See chat.h.
|
| 2 |
+
#include "chat.h"
|
| 3 |
+
|
| 4 |
+
#include <string.h>
|
| 5 |
+
|
| 6 |
+
#include "engine.h"
|
| 7 |
+
|
| 8 |
+
// The chat markers the weights were fine-tuned with (export.py writes them
|
| 9 |
+
// into model_config.h): <|u|>, <|a|>, <|e|> for the production ChatGBA,
|
| 10 |
+
// <|user|>, <|assistant|>, <|end|> for the Mac runs. Below, <|end|> stands for
|
| 11 |
+
// whichever end marker it is.
|
| 12 |
+
static const char USER[] = CHAT_USER;
|
| 13 |
+
static const char ASSISTANT[] = CHAT_ASSISTANT;
|
| 14 |
+
static const char END[] = CHAT_END;
|
| 15 |
+
#define USER_LENGTH ((int)sizeof(USER) - 1)
|
| 16 |
+
#define ASSISTANT_LENGTH ((int)sizeof(ASSISTANT) - 1)
|
| 17 |
+
#define END_LENGTH ((int)sizeof(END) - 1)
|
| 18 |
+
|
| 19 |
+
static int end_seen = END_LENGTH; // bytes of the last reply's closing <|end|> the engine has processed
|
| 20 |
+
static int32_t logits[MODEL_VOCAB];
|
| 21 |
+
|
| 22 |
+
void chat_reset(void) {
|
| 23 |
+
end_seen = END_LENGTH; // nothing to close
|
| 24 |
+
engine_reset();
|
| 25 |
+
}
|
| 26 |
+
|
| 27 |
+
// The longest k such that the reply ends with the first k bytes of <|end|>.
|
| 28 |
+
static int partial_end(const uint8_t* reply, int length) {
|
| 29 |
+
for (int k = END_LENGTH - 1; k > 0; --k) {
|
| 30 |
+
if (k <= length && memcmp(reply + length - k, END, k) == 0) return k;
|
| 31 |
+
}
|
| 32 |
+
return 0;
|
| 33 |
+
}
|
| 34 |
+
|
| 35 |
+
int chat_reply(const uint8_t* message, int length, ChatSettings* settings, uint8_t* reply, int capacity) {
|
| 36 |
+
// The rest of the last reply's <|end|>, then the new turn.
|
| 37 |
+
int total = (END_LENGTH - end_seen) + USER_LENGTH + length + END_LENGTH + ASSISTANT_LENGTH;
|
| 38 |
+
int done = 0;
|
| 39 |
+
#define FEED(bytes, n) \
|
| 40 |
+
for (int i_ = 0; i_ < (n); ++i_) { \
|
| 41 |
+
engine_step(((const uint8_t*)(bytes))[i_], logits); \
|
| 42 |
+
if (settings->on_prompt) settings->on_prompt(++done, total); \
|
| 43 |
+
}
|
| 44 |
+
FEED(END + end_seen, END_LENGTH - end_seen);
|
| 45 |
+
FEED(USER, USER_LENGTH);
|
| 46 |
+
FEED(message, length);
|
| 47 |
+
FEED(END, END_LENGTH);
|
| 48 |
+
FEED(ASSISTANT, ASSISTANT_LENGTH);
|
| 49 |
+
#undef FEED
|
| 50 |
+
|
| 51 |
+
int budget = settings->max_reply < capacity ? settings->max_reply : capacity;
|
| 52 |
+
int n = 0, shown = 0, ended = 0;
|
| 53 |
+
while (n < budget) {
|
| 54 |
+
if (n > 0 && settings->stop && settings->stop()) break;
|
| 55 |
+
int byte = engine_sample(logits, settings->temperature, settings->top_k, &settings->rng);
|
| 56 |
+
reply[n++] = (uint8_t)byte;
|
| 57 |
+
if (n >= END_LENGTH && memcmp(reply + n - END_LENGTH, END, END_LENGTH) == 0) {
|
| 58 |
+
n -= END_LENGTH;
|
| 59 |
+
ended = 1;
|
| 60 |
+
break; // its last byte waits for the next message
|
| 61 |
+
}
|
| 62 |
+
int hold = partial_end(reply, n);
|
| 63 |
+
while (shown < n - hold) {
|
| 64 |
+
if (settings->on_byte) settings->on_byte(reply[shown]);
|
| 65 |
+
++shown;
|
| 66 |
+
}
|
| 67 |
+
engine_step(byte, logits);
|
| 68 |
+
}
|
| 69 |
+
|
| 70 |
+
// The engine has seen every reply byte, and the part of <|end|> sampled so
|
| 71 |
+
// far, except the very last byte of a complete one. A reply cut off in the
|
| 72 |
+
// start of <|end|> keeps that start as the start of its closing <|end|>.
|
| 73 |
+
if (ended) {
|
| 74 |
+
end_seen = END_LENGTH - 1;
|
| 75 |
+
} else {
|
| 76 |
+
end_seen = partial_end(reply, n);
|
| 77 |
+
n -= end_seen;
|
| 78 |
+
}
|
| 79 |
+
while (shown < n) {
|
| 80 |
+
if (settings->on_byte) settings->on_byte(reply[shown]);
|
| 81 |
+
++shown;
|
| 82 |
+
}
|
| 83 |
+
return n;
|
| 84 |
+
}
|
source/chat.h
ADDED
|
@@ -0,0 +1,23 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
// Chat on top of engine.c: the conversation as turn pairs,
|
| 2 |
+
// <|user|>...<|end|><|assistant|>...<|end|>
|
| 3 |
+
// fed to the engine as it happens, once. The engine's context slides, so a
|
| 4 |
+
// long conversation simply forgets its oldest bytes, the way the model was
|
| 5 |
+
// fine-tuned (on windows that start anywhere in a conversation).
|
| 6 |
+
#pragma once
|
| 7 |
+
#include <stdint.h>
|
| 8 |
+
|
| 9 |
+
typedef struct {
|
| 10 |
+
int32_t temperature; // Q16; 0 picks the most likely byte
|
| 11 |
+
int top_k;
|
| 12 |
+
int max_reply; // bytes
|
| 13 |
+
uint32_t rng; // xorshift32 state, nonzero
|
| 14 |
+
void (*on_byte)(int byte); // each reply byte, once it cannot be part of <|end|>
|
| 15 |
+
void (*on_prompt)(int done, int total); // progress through the bytes processed before replying
|
| 16 |
+
int (*stop)(void); // asked before each reply byte after the first; nonzero ends the reply
|
| 17 |
+
} ChatSettings;
|
| 18 |
+
|
| 19 |
+
void chat_reset(void);
|
| 20 |
+
|
| 21 |
+
// Reply to message. Returns the reply's length (written to reply, without
|
| 22 |
+
// <|end|>). A message longer than the context is fine; its start slides out.
|
| 23 |
+
int chat_reply(const uint8_t* message, int length, ChatSettings* settings, uint8_t* reply, int capacity);
|
source/crt0.s
ADDED
|
@@ -0,0 +1,53 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
@ Start-up for the GBA: the cartridge header, stacks, then copying the
|
| 2 |
+
@ fast-RAM code and data out of ROM, clearing .bss, and calling main.
|
| 3 |
+
|
| 4 |
+
.section .crt0, "ax"
|
| 5 |
+
.arm
|
| 6 |
+
.global _start
|
| 7 |
+
_start:
|
| 8 |
+
b reset
|
| 9 |
+
.fill 156, 1, 0 @ Nintendo logo: left blank (mGBA boots without it)
|
| 10 |
+
.ascii "ChatGBA\0\0\0\0\0" @ title, 12 bytes (shown in mGBA's title bar)
|
| 11 |
+
.ascii "AAGE" @ game code
|
| 12 |
+
.ascii "GB" @ maker code
|
| 13 |
+
.byte 0x96 @ fixed value
|
| 14 |
+
.byte 0 @ main unit code
|
| 15 |
+
.byte 0 @ device type
|
| 16 |
+
.fill 7, 1, 0 @ reserved
|
| 17 |
+
.byte 0 @ software version
|
| 18 |
+
.byte 0 @ header checksum, filled in by fix_header.py
|
| 19 |
+
.fill 2, 1, 0 @ reserved
|
| 20 |
+
|
| 21 |
+
reset:
|
| 22 |
+
mov r0, #0x12 @ IRQ mode
|
| 23 |
+
msr cpsr_c, r0
|
| 24 |
+
ldr sp, =0x03007FA0
|
| 25 |
+
mov r0, #0x1F @ system mode, where main runs
|
| 26 |
+
msr cpsr_c, r0
|
| 27 |
+
ldr sp, =0x03007F00
|
| 28 |
+
|
| 29 |
+
ldr r0, =__iwram_lma @ fast-RAM code and data, from ROM
|
| 30 |
+
ldr r1, =__iwram_start
|
| 31 |
+
ldr r2, =__iwram_end
|
| 32 |
+
1: cmp r1, r2
|
| 33 |
+
ldrlo r3, [r0], #4
|
| 34 |
+
strlo r3, [r1], #4
|
| 35 |
+
blo 1b
|
| 36 |
+
|
| 37 |
+
mov r3, #0
|
| 38 |
+
ldr r1, =__shared_start @ the runner's mailbox
|
| 39 |
+
ldr r2, =__shared_end
|
| 40 |
+
2: cmp r1, r2
|
| 41 |
+
strlo r3, [r1], #4
|
| 42 |
+
blo 2b
|
| 43 |
+
ldr r1, =__bss_start
|
| 44 |
+
ldr r2, =__bss_end
|
| 45 |
+
4: cmp r1, r2
|
| 46 |
+
strlo r3, [r1], #4
|
| 47 |
+
blo 4b
|
| 48 |
+
|
| 49 |
+
ldr r0, =main
|
| 50 |
+
mov lr, pc
|
| 51 |
+
bx r0
|
| 52 |
+
3: b 3b
|
| 53 |
+
.pool
|
source/dot.s
ADDED
|
@@ -0,0 +1,43 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
@ int32_t dot_offset(const uint8_t* bytes, const int32_t* inputs, int n)
|
| 2 |
+
@
|
| 3 |
+
@ The sum of bytes[i] x inputs[i], 16 at a time (n is a multiple of 16), the
|
| 4 |
+
@ inner loop of every matrix product. ARM code in the fast internal RAM:
|
| 5 |
+
@ - the bytes (weights, int8 + 128) come from cartridge ROM four words per
|
| 6 |
+
@ LDM, a sequential burst, much cheaper than four separate loads
|
| 7 |
+
@ - each byte is unpacked with one AND (a shifted operand), which is why they
|
| 8 |
+
@ are stored offset to 0..255 rather than signed
|
| 9 |
+
@ - the byte is MLA's second operand: the ARM7 multiplies faster when that
|
| 10 |
+
@ operand is small
|
| 11 |
+
@ Registers: r0 bytes, r1 inputs, r2 end of bytes, r3 sum, r4-r7 sixteen
|
| 12 |
+
@ bytes, r8-r11 four inputs, r12 the mask 255, lr one unpacked byte.
|
| 13 |
+
|
| 14 |
+
.section .iwram.text, "ax"
|
| 15 |
+
.arm
|
| 16 |
+
.align 2
|
| 17 |
+
.global dot_offset
|
| 18 |
+
dot_offset:
|
| 19 |
+
push {r4-r11, lr}
|
| 20 |
+
mov r3, #0
|
| 21 |
+
mov r12, #255
|
| 22 |
+
add r2, r0, r2
|
| 23 |
+
|
| 24 |
+
1: ldmia r0!, {r4-r7}
|
| 25 |
+
|
| 26 |
+
.irp word, r4, r5, r6, r7
|
| 27 |
+
ldmia r1!, {r8-r11}
|
| 28 |
+
and lr, r12, \word
|
| 29 |
+
mla r3, r8, lr, r3
|
| 30 |
+
and lr, r12, \word, lsr #8
|
| 31 |
+
mla r3, r9, lr, r3
|
| 32 |
+
and lr, r12, \word, lsr #16
|
| 33 |
+
mla r3, r10, lr, r3
|
| 34 |
+
mov lr, \word, lsr #24
|
| 35 |
+
mla r3, r11, lr, r3
|
| 36 |
+
.endr
|
| 37 |
+
|
| 38 |
+
cmp r0, r2
|
| 39 |
+
blo 1b
|
| 40 |
+
|
| 41 |
+
mov r0, r3
|
| 42 |
+
pop {r4-r11, lr}
|
| 43 |
+
bx lr
|
source/engine.c
ADDED
|
@@ -0,0 +1,492 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
// ChatGBA inference in integer arithmetic. See engine.h.
|
| 2 |
+
//
|
| 3 |
+
// One byte at a time: each step runs the new position through every block,
|
| 4 |
+
// reading the earlier positions' keys and values from the cache. The cache
|
| 5 |
+
// keeps keys and values as int8 with one Q16 scale per vector, in a ring per
|
| 6 |
+
// block: a block with a window keeps its last window positions, the others the
|
| 7 |
+
// last MODEL_CONTEXT. Positions never run out: the context slides, each byte
|
| 8 |
+
// seeing at most the MODEL_CONTEXT bytes up to and including itself.
|
| 9 |
+
#include "engine.h"
|
| 10 |
+
|
| 11 |
+
#include <string.h>
|
| 12 |
+
|
| 13 |
+
#ifdef __GBA__
|
| 14 |
+
// The cache lives in the 256 KB of external work RAM. (The Makefile compiles
|
| 15 |
+
// this file as ARM code and moves it into the fast internal RAM, where 64-bit
|
| 16 |
+
// multiplies are single instructions; the inner product is dot.s.)
|
| 17 |
+
#define BIG_STATE __attribute__((section(".bss.ewram")))
|
| 18 |
+
#else
|
| 19 |
+
#define BIG_STATE
|
| 20 |
+
#endif
|
| 21 |
+
|
| 22 |
+
#define D MODEL_D
|
| 23 |
+
#define H MODEL_HEADS
|
| 24 |
+
#define KV MODEL_KV_HEADS
|
| 25 |
+
#define HD MODEL_HEAD_DIM
|
| 26 |
+
#define L MODEL_BLOCKS
|
| 27 |
+
#define F MODEL_HIDDEN
|
| 28 |
+
#define K MODEL_KERNEL
|
| 29 |
+
#define CTX MODEL_CONTEXT
|
| 30 |
+
#define VOCAB MODEL_VOCAB
|
| 31 |
+
#define GROUP (H / KV) // query heads per key/value head
|
| 32 |
+
|
| 33 |
+
#define EPS_Q32 42950 // RMSNorm's 1e-5, in units of 2^-32
|
| 34 |
+
#define INV_SQRT_HD 11585 // 1 / sqrt(32), Q16
|
| 35 |
+
#define LOG2E_Q16 94548 // log2(e), Q16
|
| 36 |
+
#define INPUT_LIMIT 4095 // matrix inputs are rounded to at most this
|
| 37 |
+
#define HISTORY_SHIFT 7 // canon histories are kept as int16 in Q9
|
| 38 |
+
|
| 39 |
+
uint32_t profile[PROFILE_PARTS];
|
| 40 |
+
uint32_t (*profile_clock)(void);
|
| 41 |
+
#ifdef PROFILE
|
| 42 |
+
#define TIME(part, statement) \
|
| 43 |
+
do { \
|
| 44 |
+
uint32_t started_ = profile_clock(); \
|
| 45 |
+
statement; \
|
| 46 |
+
profile[part] += profile_clock() - started_; \
|
| 47 |
+
} while (0)
|
| 48 |
+
#else
|
| 49 |
+
#define TIME(part, statement) statement
|
| 50 |
+
#endif
|
| 51 |
+
|
| 52 |
+
// --- The blob ------------------------------------------------------------------
|
| 53 |
+
|
| 54 |
+
typedef struct {
|
| 55 |
+
const uint8_t* rows; // [out][in], int8 + 128
|
| 56 |
+
const int32_t* scales; // [out], units of 2^-24
|
| 57 |
+
} Matrix;
|
| 58 |
+
|
| 59 |
+
typedef struct {
|
| 60 |
+
const int32_t* attention_norm; // [D]
|
| 61 |
+
const int32_t* conv_a; // [K][D]
|
| 62 |
+
Matrix wq, wk, wv;
|
| 63 |
+
const int32_t* q_norm; // [H][HD]
|
| 64 |
+
const int32_t* k_norm; // [KV][HD]
|
| 65 |
+
Matrix wo;
|
| 66 |
+
const int32_t* mlp_norm; // [D]
|
| 67 |
+
const int32_t* conv_c; // [K][D]
|
| 68 |
+
Matrix gate, up, down;
|
| 69 |
+
} Block;
|
| 70 |
+
|
| 71 |
+
static Matrix embeddings; // [VOCAB][D], also the output projection
|
| 72 |
+
static Block blocks[L];
|
| 73 |
+
static const int32_t* final_norm;
|
| 74 |
+
static const int32_t* rope_cos; // [CTX][HD / 2]
|
| 75 |
+
static const int32_t* rope_sin;
|
| 76 |
+
static const int32_t* exp2_table; // 2^(-i/256), i = 0..256
|
| 77 |
+
static const int32_t* sigmoid_table; // -16 to 16 in steps of 1/16
|
| 78 |
+
|
| 79 |
+
static const int windows[L] = MODEL_WINDOWS;
|
| 80 |
+
|
| 81 |
+
static const uint8_t* cursor;
|
| 82 |
+
|
| 83 |
+
static const void* take(int bytes) {
|
| 84 |
+
const void* at = cursor;
|
| 85 |
+
cursor += (bytes + 3) & ~3;
|
| 86 |
+
return at;
|
| 87 |
+
}
|
| 88 |
+
|
| 89 |
+
static Matrix take_matrix(int in, int out) {
|
| 90 |
+
Matrix m;
|
| 91 |
+
m.rows = take(in * out);
|
| 92 |
+
m.scales = take(out * 4);
|
| 93 |
+
return m;
|
| 94 |
+
}
|
| 95 |
+
|
| 96 |
+
// --- State ---------------------------------------------------------------------
|
| 97 |
+
|
| 98 |
+
static uint8_t cache_k[MODEL_KV_SLOTS][KV][HD] BIG_STATE; // int8 + 128, like the weights
|
| 99 |
+
static int8_t cache_v[MODEL_KV_SLOTS][KV][HD] BIG_STATE;
|
| 100 |
+
static int32_t cache_k_scale[MODEL_KV_SLOTS][KV] BIG_STATE; // Q16 scale x 1/sqrt(HD), units of 2^-24
|
| 101 |
+
static int32_t cache_v_scale[MODEL_KV_SLOTS][KV] BIG_STATE;
|
| 102 |
+
static int16_t history[L][2][K - 1][D] BIG_STATE; // each canon layer's last inputs, newest first
|
| 103 |
+
static int first_slot[L]; // where each block's part of the cache starts
|
| 104 |
+
static int position;
|
| 105 |
+
|
| 106 |
+
// RoPE past the table's CTX positions. The angle at position p is p x theta_i,
|
| 107 |
+
// split as (p - p % CTX) x theta_i, the anchor, plus (p % CTX) x theta_i from
|
| 108 |
+
// the table. The anchor turns by CTX x theta_i every CTX positions. Attention
|
| 109 |
+
// depends only on differences between positions, so the anchor's own slow
|
| 110 |
+
// drift does not matter, only each turn's accuracy.
|
| 111 |
+
static int64_t anchor_cos[HD / 2], anchor_sin[HD / 2]; // Q30
|
| 112 |
+
static int32_t step_cos[HD / 2], step_sin[HD / 2]; // this position's rotation, Q16
|
| 113 |
+
|
| 114 |
+
// Working vectors for one position.
|
| 115 |
+
static int32_t x[D], a[D], q[H * HD], k[KV * HD], v[KV * HD], mixed[H * HD], delta[D];
|
| 116 |
+
static int32_t gate[F], up[F];
|
| 117 |
+
static int32_t scores[CTX];
|
| 118 |
+
static int32_t quantized[F > D ? F : D];
|
| 119 |
+
static int64_t sums[HD];
|
| 120 |
+
|
| 121 |
+
int engine_init(const uint8_t* blob) {
|
| 122 |
+
const int32_t* header = (const int32_t*)blob;
|
| 123 |
+
const int32_t expected[] = {0x31414247, 2, VOCAB, D, H, KV, HD, L, F, K, CTX};
|
| 124 |
+
for (int i = 0; i < (int)(sizeof expected / sizeof expected[0]); ++i) {
|
| 125 |
+
if (header[i] != expected[i]) return -1 - i;
|
| 126 |
+
}
|
| 127 |
+
for (int b = 0; b < L; ++b) {
|
| 128 |
+
if (header[12 + b] != windows[b]) return -100 - b;
|
| 129 |
+
}
|
| 130 |
+
cursor = blob;
|
| 131 |
+
take((12 + L) * 4);
|
| 132 |
+
|
| 133 |
+
embeddings = take_matrix(D, VOCAB);
|
| 134 |
+
for (int b = 0; b < L; ++b) {
|
| 135 |
+
Block* blk = &blocks[b];
|
| 136 |
+
blk->attention_norm = take(D * 4);
|
| 137 |
+
blk->conv_a = take(K * D * 4);
|
| 138 |
+
blk->wq = take_matrix(D, H * HD);
|
| 139 |
+
blk->wk = take_matrix(D, KV * HD);
|
| 140 |
+
blk->wv = take_matrix(D, KV * HD);
|
| 141 |
+
blk->q_norm = take(H * HD * 4);
|
| 142 |
+
blk->k_norm = take(KV * HD * 4);
|
| 143 |
+
blk->wo = take_matrix(H * HD, D);
|
| 144 |
+
blk->mlp_norm = take(D * 4);
|
| 145 |
+
blk->conv_c = take(K * D * 4);
|
| 146 |
+
blk->gate = take_matrix(D, F);
|
| 147 |
+
blk->up = take_matrix(D, F);
|
| 148 |
+
blk->down = take_matrix(F, D);
|
| 149 |
+
}
|
| 150 |
+
final_norm = take(D * 4);
|
| 151 |
+
rope_cos = take(CTX * (HD / 2) * 4);
|
| 152 |
+
rope_sin = take(CTX * (HD / 2) * 4);
|
| 153 |
+
exp2_table = take(257 * 4);
|
| 154 |
+
sigmoid_table = take(513 * 4);
|
| 155 |
+
|
| 156 |
+
int slot = 0;
|
| 157 |
+
for (int b = 0; b < L; ++b) {
|
| 158 |
+
first_slot[b] = slot;
|
| 159 |
+
slot += windows[b] ? windows[b] : CTX;
|
| 160 |
+
}
|
| 161 |
+
engine_reset();
|
| 162 |
+
return 0;
|
| 163 |
+
}
|
| 164 |
+
|
| 165 |
+
void engine_reset(void) {
|
| 166 |
+
position = 0;
|
| 167 |
+
memset(history, 0, sizeof history);
|
| 168 |
+
for (int i = 0; i < HD / 2; ++i) {
|
| 169 |
+
anchor_cos[i] = (int64_t)1 << 30;
|
| 170 |
+
anchor_sin[i] = 0;
|
| 171 |
+
}
|
| 172 |
+
}
|
| 173 |
+
|
| 174 |
+
int engine_position(void) { return position; }
|
| 175 |
+
|
| 176 |
+
// --- Arithmetic ----------------------------------------------------------------
|
| 177 |
+
|
| 178 |
+
static int32_t saturate(int64_t value) {
|
| 179 |
+
if (value > INT32_MAX) return INT32_MAX;
|
| 180 |
+
if (value < INT32_MIN) return INT32_MIN;
|
| 181 |
+
return (int32_t)value;
|
| 182 |
+
}
|
| 183 |
+
|
| 184 |
+
static int32_t mul_q16(int32_t a, int32_t b) {
|
| 185 |
+
return saturate(((int64_t)a * b + 32768) >> 16);
|
| 186 |
+
}
|
| 187 |
+
|
| 188 |
+
static uint32_t sqrt64(uint64_t value) {
|
| 189 |
+
uint64_t root = 0, bit = (uint64_t)1 << 62;
|
| 190 |
+
while (bit > value) bit >>= 2;
|
| 191 |
+
while (bit) {
|
| 192 |
+
if (value >= root + bit) {
|
| 193 |
+
value -= root + bit;
|
| 194 |
+
root = (root >> 1) + bit;
|
| 195 |
+
} else {
|
| 196 |
+
root >>= 1;
|
| 197 |
+
}
|
| 198 |
+
bit >>= 2;
|
| 199 |
+
}
|
| 200 |
+
return (uint32_t)root;
|
| 201 |
+
}
|
| 202 |
+
|
| 203 |
+
// y = x / sqrt(mean(x^2) + 1e-5), times gamma when given, as MLX's rms_norm.
|
| 204 |
+
static void rms_norm(const int32_t* in, const int32_t* gamma, int n, int32_t* out) {
|
| 205 |
+
int64_t squares = 0;
|
| 206 |
+
for (int i = 0; i < n; ++i) squares += (int64_t)in[i] * in[i];
|
| 207 |
+
uint32_t rms = sqrt64((uint64_t)squares / n + EPS_Q32); // Q16
|
| 208 |
+
if (rms == 0) rms = 1;
|
| 209 |
+
int64_t inverse = ((int64_t)1 << 46) / rms; // 1/rms, Q30
|
| 210 |
+
for (int i = 0; i < n; ++i) {
|
| 211 |
+
int64_t y = ((int64_t)in[i] * inverse + (1 << 29)) >> 30;
|
| 212 |
+
if (gamma) y = (y * gamma[i] + 32768) >> 16;
|
| 213 |
+
out[i] = saturate(y);
|
| 214 |
+
}
|
| 215 |
+
}
|
| 216 |
+
|
| 217 |
+
// Round a Q16 vector to values of at most INPUT_LIMIT, sharing one
|
| 218 |
+
// power-of-two shift. Returns the shift; *sum gets the rounded values' sum.
|
| 219 |
+
static int quantize_input(const int32_t* in, int n, int32_t* out, int32_t* sum) {
|
| 220 |
+
uint32_t peak = 0;
|
| 221 |
+
for (int i = 0; i < n; ++i) {
|
| 222 |
+
uint32_t magnitude = in[i] < 0 ? (uint32_t)(-(int64_t)in[i]) : (uint32_t)in[i];
|
| 223 |
+
if (magnitude > peak) peak = magnitude;
|
| 224 |
+
}
|
| 225 |
+
int shift = 0;
|
| 226 |
+
while ((peak >> shift) > INPUT_LIMIT) ++shift;
|
| 227 |
+
int32_t half = shift ? 1 << (shift - 1) : 0, total = 0;
|
| 228 |
+
for (int i = 0; i < n; ++i) total += out[i] = (in[i] + half) >> shift;
|
| 229 |
+
*sum = total;
|
| 230 |
+
return shift;
|
| 231 |
+
}
|
| 232 |
+
|
| 233 |
+
// The inner product of offset bytes (int8 + 128) and inputs, without the
|
| 234 |
+
// offset taken back out: sum of byte x input. n is a multiple of 16. On the GBA
|
| 235 |
+
// it is dot.s.
|
| 236 |
+
#ifdef __GBA__
|
| 237 |
+
int32_t dot_offset(const uint8_t* bytes, const int32_t* inputs, int n);
|
| 238 |
+
#else
|
| 239 |
+
static int32_t dot_offset(const uint8_t* bytes, const int32_t* inputs, int n) {
|
| 240 |
+
int32_t sum = 0;
|
| 241 |
+
for (int i = 0; i < n; ++i) sum += bytes[i] * inputs[i];
|
| 242 |
+
return sum;
|
| 243 |
+
}
|
| 244 |
+
#endif
|
| 245 |
+
|
| 246 |
+
// The true inner product: each byte stands for byte - 128.
|
| 247 |
+
static int32_t dot(const uint8_t* bytes, const int32_t* inputs, int32_t input_sum, int n) {
|
| 248 |
+
return dot_offset(bytes, inputs, n) - 128 * input_sum;
|
| 249 |
+
}
|
| 250 |
+
|
| 251 |
+
// A dot product's int32 sum back to Q16: times the output's scale (2^-24)
|
| 252 |
+
// and the input's shift.
|
| 253 |
+
static int32_t rescale(int32_t sum, int32_t scale, int shift) {
|
| 254 |
+
int64_t r = (int64_t)sum * scale;
|
| 255 |
+
int down = 24 - shift;
|
| 256 |
+
if (down > 0) return saturate((r + ((int64_t)1 << (down - 1))) >> down);
|
| 257 |
+
return saturate(r << -down);
|
| 258 |
+
}
|
| 259 |
+
|
| 260 |
+
// out = in @ W, for a matrix stored as int8 rows of W^T.
|
| 261 |
+
static void matrix_times(const Matrix* m, const int32_t* in, int n_in, int n_out, int32_t* out) {
|
| 262 |
+
int32_t sum;
|
| 263 |
+
int shift = quantize_input(in, n_in, quantized, &sum);
|
| 264 |
+
for (int o = 0; o < n_out; ++o) {
|
| 265 |
+
out[o] = rescale(dot(m->rows + o * n_in, quantized, sum, n_in), m->scales[o], shift);
|
| 266 |
+
}
|
| 267 |
+
}
|
| 268 |
+
|
| 269 |
+
// e^d for d <= 0 (Q16), from the 2^(-i/256) table.
|
| 270 |
+
static int32_t exp_q16(int32_t d) {
|
| 271 |
+
int64_t y = ((int64_t)(-d) * LOG2E_Q16) >> 16; // -d log2(e), Q16
|
| 272 |
+
int whole = (int)(y >> 16);
|
| 273 |
+
if (whole >= 31) return 0;
|
| 274 |
+
int fraction = (int)(y & 0xFFFF);
|
| 275 |
+
int i = fraction >> 8, rest = fraction & 0xFF;
|
| 276 |
+
int32_t t = exp2_table[i] + (((exp2_table[i + 1] - exp2_table[i]) * rest) >> 8);
|
| 277 |
+
return t >> whole;
|
| 278 |
+
}
|
| 279 |
+
|
| 280 |
+
static int32_t sigmoid_q16(int32_t value) {
|
| 281 |
+
if (value <= -16 * 65536) return sigmoid_table[0];
|
| 282 |
+
if (value >= 16 * 65536) return sigmoid_table[512];
|
| 283 |
+
int32_t offset = value + 16 * 65536; // 0 to 32, Q16
|
| 284 |
+
int i = offset >> 12, rest = offset & 0xFFF;
|
| 285 |
+
return sigmoid_table[i] + (((sigmoid_table[i + 1] - sigmoid_table[i]) * rest) >> 12);
|
| 286 |
+
}
|
| 287 |
+
|
| 288 |
+
// Turn the anchor by CTX x theta_i: the table's last angle, (CTX - 1) x
|
| 289 |
+
// theta_i, then one more theta_i. Its length is put back to one, so rounding
|
| 290 |
+
// cannot grow or shrink it over a long conversation.
|
| 291 |
+
static void turn_anchor(void) {
|
| 292 |
+
const int32_t* c_last = rope_cos + (CTX - 1) * (HD / 2);
|
| 293 |
+
const int32_t* s_last = rope_sin + (CTX - 1) * (HD / 2);
|
| 294 |
+
const int32_t* c_one = rope_cos + HD / 2;
|
| 295 |
+
const int32_t* s_one = rope_sin + HD / 2;
|
| 296 |
+
for (int i = 0; i < HD / 2; ++i) {
|
| 297 |
+
int64_t tc = (int64_t)c_last[i] * c_one[i] - (int64_t)s_last[i] * s_one[i]; // Q32
|
| 298 |
+
int64_t ts = (int64_t)s_last[i] * c_one[i] + (int64_t)c_last[i] * s_one[i];
|
| 299 |
+
int64_t ac = anchor_cos[i], as = anchor_sin[i]; // Q30
|
| 300 |
+
int64_t c = ((ac * (tc >> 16)) - (as * (ts >> 16)) + ((int64_t)1 << 15)) >> 16; // Q30
|
| 301 |
+
int64_t s = ((as * (tc >> 16)) + (ac * (ts >> 16)) + ((int64_t)1 << 15)) >> 16;
|
| 302 |
+
uint32_t length = sqrt64((uint64_t)(c * c + s * s)); // Q30
|
| 303 |
+
anchor_cos[i] = (c << 30) / length;
|
| 304 |
+
anchor_sin[i] = (s << 30) / length;
|
| 305 |
+
}
|
| 306 |
+
}
|
| 307 |
+
|
| 308 |
+
// This position's rotation: the anchor, then the table's angle.
|
| 309 |
+
static void set_rotation(void) {
|
| 310 |
+
if (position > 0 && position % CTX == 0) turn_anchor();
|
| 311 |
+
const int32_t* c = rope_cos + (position % CTX) * (HD / 2);
|
| 312 |
+
const int32_t* s = rope_sin + (position % CTX) * (HD / 2);
|
| 313 |
+
for (int i = 0; i < HD / 2; ++i) {
|
| 314 |
+
step_cos[i] = (int32_t)((anchor_cos[i] * c[i] - anchor_sin[i] * s[i] + ((int64_t)1 << 29)) >> 30);
|
| 315 |
+
step_sin[i] = (int32_t)((anchor_sin[i] * c[i] + anchor_cos[i] * s[i] + ((int64_t)1 << 29)) >> 30);
|
| 316 |
+
}
|
| 317 |
+
}
|
| 318 |
+
|
| 319 |
+
// Rotate each adjacent pair (2i, 2i+1) of a head by its angle at this position.
|
| 320 |
+
static void rope(int32_t* head) {
|
| 321 |
+
const int32_t* c = step_cos;
|
| 322 |
+
const int32_t* s = step_sin;
|
| 323 |
+
for (int i = 0; i < HD / 2; ++i) {
|
| 324 |
+
int64_t x0 = head[2 * i], x1 = head[2 * i + 1];
|
| 325 |
+
head[2 * i] = saturate((x0 * c[i] - x1 * s[i] + 32768) >> 16);
|
| 326 |
+
head[2 * i + 1] = saturate((x0 * s[i] + x1 * c[i] + 32768) >> 16);
|
| 327 |
+
}
|
| 328 |
+
}
|
| 329 |
+
|
| 330 |
+
// Store a head's vector as int8 with a Q16 scale, max |value| / 127. With
|
| 331 |
+
// offset, the bytes are int8 + 128.
|
| 332 |
+
static void store_int8(const int32_t* in, int8_t* out, int32_t* scale, int offset) {
|
| 333 |
+
uint32_t peak = 0;
|
| 334 |
+
for (int i = 0; i < HD; ++i) {
|
| 335 |
+
uint32_t magnitude = in[i] < 0 ? (uint32_t)(-(int64_t)in[i]) : (uint32_t)in[i];
|
| 336 |
+
if (magnitude > peak) peak = magnitude;
|
| 337 |
+
}
|
| 338 |
+
int32_t s = (int32_t)((peak + 126) / 127);
|
| 339 |
+
if (s == 0) s = 1;
|
| 340 |
+
for (int i = 0; i < HD; ++i) {
|
| 341 |
+
int32_t value = in[i];
|
| 342 |
+
int32_t r = value >= 0 ? (value + s / 2) / s : -((-value + s / 2) / s);
|
| 343 |
+
out[i] = (int8_t)((r > 127 ? 127 : r < -127 ? -127 : r) + offset);
|
| 344 |
+
}
|
| 345 |
+
*scale = s;
|
| 346 |
+
}
|
| 347 |
+
|
| 348 |
+
// A canon layer: vec += sum over j of w[j] * (the input j positions back),
|
| 349 |
+
// then the input joins the history.
|
| 350 |
+
static void canon(int32_t* vec, const int32_t* w, int16_t (*hist)[D]) {
|
| 351 |
+
for (int i = 0; i < D; ++i) {
|
| 352 |
+
int64_t sum = (int64_t)w[i] * vec[i];
|
| 353 |
+
for (int j = 1; j < K; ++j) sum += (int64_t)w[j * D + i] * ((int32_t)hist[j - 1][i] << HISTORY_SHIFT);
|
| 354 |
+
int32_t input = vec[i];
|
| 355 |
+
for (int j = K - 2; j > 0; --j) hist[j][i] = hist[j - 1][i];
|
| 356 |
+
int32_t h = (input + (1 << (HISTORY_SHIFT - 1))) >> HISTORY_SHIFT;
|
| 357 |
+
hist[0][i] = (int16_t)(h > 32767 ? 32767 : h < -32768 ? -32768 : h);
|
| 358 |
+
vec[i] = saturate(input + ((sum + 32768) >> 16));
|
| 359 |
+
}
|
| 360 |
+
}
|
| 361 |
+
|
| 362 |
+
// --- One position --------------------------------------------------------------
|
| 363 |
+
|
| 364 |
+
static void attention(int b, const Block* blk) {
|
| 365 |
+
int window = windows[b];
|
| 366 |
+
int slots = window ? window : CTX;
|
| 367 |
+
int slot = first_slot[b] + position % slots;
|
| 368 |
+
int first = position >= slots ? position - slots + 1 : 0;
|
| 369 |
+
|
| 370 |
+
for (int g = 0; g < KV; ++g) {
|
| 371 |
+
int32_t* kg = k + g * HD;
|
| 372 |
+
rms_norm(kg, blk->k_norm + g * HD, HD, kg);
|
| 373 |
+
rope(kg);
|
| 374 |
+
int32_t scale;
|
| 375 |
+
store_int8(kg, (int8_t*)cache_k[slot][g], &scale, 128);
|
| 376 |
+
cache_k_scale[slot][g] = (int32_t)(((int64_t)scale * INV_SQRT_HD + 128) >> 8);
|
| 377 |
+
store_int8(v + g * HD, cache_v[slot][g], &cache_v_scale[slot][g], 0);
|
| 378 |
+
}
|
| 379 |
+
|
| 380 |
+
for (int h = 0; h < H; ++h) {
|
| 381 |
+
int g = h / GROUP;
|
| 382 |
+
int32_t* qh = q + h * HD;
|
| 383 |
+
rms_norm(qh, blk->q_norm + h * HD, HD, qh);
|
| 384 |
+
rope(qh);
|
| 385 |
+
int32_t sum;
|
| 386 |
+
int shift = quantize_input(qh, HD, quantized, &sum);
|
| 387 |
+
|
| 388 |
+
int32_t best = INT32_MIN;
|
| 389 |
+
for (int j = first; j <= position; ++j) {
|
| 390 |
+
int s = first_slot[b] + j % slots;
|
| 391 |
+
int64_t r = (int64_t)dot(cache_k[s][g], quantized, sum, HD) * cache_k_scale[s][g];
|
| 392 |
+
int32_t score = saturate(r >> (24 - shift));
|
| 393 |
+
scores[j - first] = score;
|
| 394 |
+
if (score > best) best = score;
|
| 395 |
+
}
|
| 396 |
+
|
| 397 |
+
int32_t total = 0;
|
| 398 |
+
for (int d = 0; d < HD; ++d) sums[d] = 0;
|
| 399 |
+
for (int j = first; j <= position; ++j) {
|
| 400 |
+
int s = first_slot[b] + j % slots;
|
| 401 |
+
int32_t weight = exp_q16(scores[j - first] - best);
|
| 402 |
+
total += weight;
|
| 403 |
+
int32_t u = (int32_t)(((int64_t)weight * cache_v_scale[s][g]) >> 8);
|
| 404 |
+
const int8_t* vs = cache_v[s][g];
|
| 405 |
+
for (int d = 0; d < HD; ++d) sums[d] += (int64_t)u * vs[d];
|
| 406 |
+
}
|
| 407 |
+
for (int d = 0; d < HD; ++d) mixed[h * HD + d] = saturate((sums[d] << 8) / total);
|
| 408 |
+
}
|
| 409 |
+
}
|
| 410 |
+
|
| 411 |
+
void engine_step(int byte, int32_t* logits) {
|
| 412 |
+
set_rotation();
|
| 413 |
+
|
| 414 |
+
// The embedding row, from 2^-24 scales to Q16.
|
| 415 |
+
const uint8_t* row = embeddings.rows + byte * D;
|
| 416 |
+
int32_t scale = embeddings.scales[byte];
|
| 417 |
+
for (int i = 0; i < D; ++i) x[i] = (int32_t)(((int64_t)(row[i] - 128) * scale + 128) >> 8);
|
| 418 |
+
|
| 419 |
+
for (int b = 0; b < L; ++b) {
|
| 420 |
+
const Block* blk = &blocks[b];
|
| 421 |
+
|
| 422 |
+
TIME(PROFILE_NORMS, rms_norm(x, blk->attention_norm, D, a));
|
| 423 |
+
TIME(PROFILE_CANON, canon(a, blk->conv_a, history[b][0]));
|
| 424 |
+
TIME(PROFILE_MATRICES, matrix_times(&blk->wq, a, D, H * HD, q));
|
| 425 |
+
TIME(PROFILE_MATRICES, matrix_times(&blk->wk, a, D, KV * HD, k));
|
| 426 |
+
TIME(PROFILE_MATRICES, matrix_times(&blk->wv, a, D, KV * HD, v));
|
| 427 |
+
TIME(PROFILE_ATTENTION, attention(b, blk));
|
| 428 |
+
TIME(PROFILE_MATRICES, matrix_times(&blk->wo, mixed, H * HD, D, delta));
|
| 429 |
+
for (int i = 0; i < D; ++i) x[i] = saturate((int64_t)x[i] + delta[i]);
|
| 430 |
+
|
| 431 |
+
TIME(PROFILE_NORMS, rms_norm(x, blk->mlp_norm, D, a));
|
| 432 |
+
TIME(PROFILE_CANON, canon(a, blk->conv_c, history[b][1]));
|
| 433 |
+
TIME(PROFILE_MATRICES, matrix_times(&blk->gate, a, D, F, gate));
|
| 434 |
+
TIME(PROFILE_MATRICES, matrix_times(&blk->up, a, D, F, up));
|
| 435 |
+
TIME(PROFILE_MLP, for (int i = 0; i < F; ++i) gate[i] = mul_q16(mul_q16(gate[i], sigmoid_q16(gate[i])), up[i]));
|
| 436 |
+
TIME(PROFILE_MATRICES, matrix_times(&blk->down, gate, F, D, delta));
|
| 437 |
+
for (int i = 0; i < D; ++i) x[i] = saturate((int64_t)x[i] + delta[i]);
|
| 438 |
+
}
|
| 439 |
+
|
| 440 |
+
TIME(PROFILE_NORMS, rms_norm(x, final_norm, D, a));
|
| 441 |
+
TIME(PROFILE_MATRICES, matrix_times(&embeddings, a, D, VOCAB, logits));
|
| 442 |
+
++position;
|
| 443 |
+
}
|
| 444 |
+
|
| 445 |
+
// --- Sampling ------------------------------------------------------------------
|
| 446 |
+
|
| 447 |
+
static uint32_t xorshift(uint32_t* state) {
|
| 448 |
+
uint32_t s = *state;
|
| 449 |
+
s ^= s << 13;
|
| 450 |
+
s ^= s >> 17;
|
| 451 |
+
s ^= s << 5;
|
| 452 |
+
return *state = s;
|
| 453 |
+
}
|
| 454 |
+
|
| 455 |
+
int engine_sample(const int32_t* logits, int32_t temperature, int top_k, uint32_t* rng) {
|
| 456 |
+
int best = 0;
|
| 457 |
+
for (int i = 1; i < VOCAB; ++i) {
|
| 458 |
+
if (logits[i] > logits[best]) best = i;
|
| 459 |
+
}
|
| 460 |
+
if (temperature <= 0) return best;
|
| 461 |
+
|
| 462 |
+
// Weights e^((logit - best) / temperature), Q16.
|
| 463 |
+
static int32_t weights[VOCAB];
|
| 464 |
+
int64_t inverse = ((int64_t)1 << 32) / temperature; // 1/temperature, Q16
|
| 465 |
+
for (int i = 0; i < VOCAB; ++i) {
|
| 466 |
+
int64_t z = ((int64_t)(logits[i] - logits[best]) * inverse) >> 16;
|
| 467 |
+
weights[i] = exp_q16(z < INT32_MIN ? INT32_MIN : (int32_t)z);
|
| 468 |
+
}
|
| 469 |
+
// Keep the top_k largest: find the k-th largest weight.
|
| 470 |
+
if (top_k > 0 && top_k < VOCAB) {
|
| 471 |
+
int32_t floor = INT32_MAX;
|
| 472 |
+
for (int n = 0; n < top_k; ++n) {
|
| 473 |
+
int32_t next = -1;
|
| 474 |
+
for (int i = 0; i < VOCAB; ++i) {
|
| 475 |
+
if (weights[i] < floor && weights[i] > next) next = weights[i];
|
| 476 |
+
}
|
| 477 |
+
if (next < 0) break;
|
| 478 |
+
floor = next;
|
| 479 |
+
}
|
| 480 |
+
for (int i = 0; i < VOCAB; ++i) {
|
| 481 |
+
if (weights[i] < floor) weights[i] = 0;
|
| 482 |
+
}
|
| 483 |
+
}
|
| 484 |
+
uint32_t total = 0;
|
| 485 |
+
for (int i = 0; i < VOCAB; ++i) total += (uint32_t)weights[i];
|
| 486 |
+
uint32_t target = (xorshift(rng) >> 8) % total;
|
| 487 |
+
for (int i = 0; i < VOCAB; ++i) {
|
| 488 |
+
if (target < (uint32_t)weights[i]) return i;
|
| 489 |
+
target -= (uint32_t)weights[i];
|
| 490 |
+
}
|
| 491 |
+
return best;
|
| 492 |
+
}
|
source/engine.h
ADDED
|
@@ -0,0 +1,37 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
// ChatGBA inference in integer arithmetic, for the Game Boy Advance (and the
|
| 2 |
+
// Mac, where the same code measures what quantization costs).
|
| 3 |
+
//
|
| 4 |
+
// Numbers are fixed-point: an int32 "Q16" value v stands for v / 65536.
|
| 5 |
+
// Weights are int8 with one scale per output; activations entering a matrix
|
| 6 |
+
// are rounded to at most 13 bits with a shared power-of-two shift, so a
|
| 7 |
+
// product of the two fits easily in an int32 sum.
|
| 8 |
+
#pragma once
|
| 9 |
+
#include <stdint.h>
|
| 10 |
+
#include "model_config.h"
|
| 11 |
+
|
| 12 |
+
// Reads the blob that export.py wrote (it stays where it is; on the GBA, in
|
| 13 |
+
// cartridge ROM). Returns 0, or a negative number if the blob does not match
|
| 14 |
+
// model_config.h.
|
| 15 |
+
int engine_init(const uint8_t* blob);
|
| 16 |
+
|
| 17 |
+
// Forget the context: the next byte is position 0.
|
| 18 |
+
void engine_reset(void);
|
| 19 |
+
|
| 20 |
+
// Bytes processed so far: the position of the next one.
|
| 21 |
+
int engine_position(void);
|
| 22 |
+
|
| 23 |
+
// Process one byte at the next position and write the scores for the byte that
|
| 24 |
+
// follows it into logits (Q16, MODEL_VOCAB of them). There is no last position:
|
| 25 |
+
// the context slides, so each byte sees at most the MODEL_CONTEXT bytes up to
|
| 26 |
+
// and including itself (fewer in a block with a shorter window).
|
| 27 |
+
void engine_step(int byte, int32_t* logits);
|
| 28 |
+
|
| 29 |
+
// Pick the next byte from logits. temperature is Q16; 0 picks the most likely
|
| 30 |
+
// byte. Sampling keeps the top_k most likely bytes. rng is an xorshift32 state.
|
| 31 |
+
int engine_sample(const int32_t* logits, int32_t temperature, int top_k, uint32_t* rng);
|
| 32 |
+
|
| 33 |
+
// With PROFILE defined, engine_step adds the time it spends in each part to
|
| 34 |
+
// profile[] (in whatever units profile_clock counts).
|
| 35 |
+
enum { PROFILE_MATRICES, PROFILE_ATTENTION, PROFILE_NORMS, PROFILE_CANON, PROFILE_MLP, PROFILE_PARTS };
|
| 36 |
+
extern uint32_t profile[PROFILE_PARTS];
|
| 37 |
+
extern uint32_t (*profile_clock)(void);
|
source/fix_header.py
ADDED
|
@@ -0,0 +1,25 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""Make a GBA ROM's header bootable on real hardware: write the Nintendo logo
|
| 2 |
+
(0x04-0x9F) and the fixed byte 0x96 (0xB2), fill in the header checksum (byte
|
| 3 |
+
0xBD) and pad the ROM to a multiple of 4 bytes. The console's BIOS refuses to
|
| 4 |
+
start a cartridge whose logo or checksum is wrong; emulators skip the check.
|
| 5 |
+
The logo bytes are the ones devkitPro's gbafix writes.
|
| 6 |
+
|
| 7 |
+
python3 fix_header.py ROM.gba"""
|
| 8 |
+
import hashlib
|
| 9 |
+
import sys
|
| 10 |
+
from pathlib import Path
|
| 11 |
+
|
| 12 |
+
LOGO = bytes.fromhex(
|
| 13 |
+
"24ffae51699aa2213d84820a84e409ad11248b98c0817f21a352be199309ce20"
|
| 14 |
+
"10464a4af82731ec58c7e83382e3cebf85f4df94ce4b09c194568ac01372a7fc"
|
| 15 |
+
"9f844d73a3ca9a615897a327fc039876231dc7610304ae56bf38840040a70efd"
|
| 16 |
+
"ff52fe036f9530f197fbc08560d68025a963be03014e38e2f9a234ffbb3e0344"
|
| 17 |
+
"780090cb88113a9465c07c6387f03cafd625e48b380aac7221d4f807")
|
| 18 |
+
assert hashlib.sha1(LOGO).hexdigest() == "17daa0fec02fc33c0f6abb549a8b80b6613b48ee"
|
| 19 |
+
|
| 20 |
+
rom = bytearray(Path(sys.argv[1]).read_bytes())
|
| 21 |
+
rom[0x04:0xA0] = LOGO
|
| 22 |
+
rom[0xB2] = 0x96
|
| 23 |
+
rom[0xBD] = -(sum(rom[0xA0:0xBD]) + 0x19) & 0xFF
|
| 24 |
+
rom += b"\0" * (-len(rom) % 4)
|
| 25 |
+
Path(sys.argv[1]).write_bytes(rom)
|
source/font.h
ADDED
|
@@ -0,0 +1,103 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
// Generated by font.py from Monaco.ttf at 9 px, with hand-drawn descenders; do not edit.
|
| 2 |
+
// Glyphs for ASCII 32-126, 6x10: one byte per row, bit x is column x.
|
| 3 |
+
#pragma once
|
| 4 |
+
#include <stdint.h>
|
| 5 |
+
#define FONT_WIDTH 6
|
| 6 |
+
#define FONT_HEIGHT 10
|
| 7 |
+
static const uint8_t font[95][10] = {
|
| 8 |
+
{0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00}, // ' '
|
| 9 |
+
{0x00, 0x04, 0x04, 0x04, 0x04, 0x04, 0x00, 0x04, 0x00, 0x00}, // '!'
|
| 10 |
+
{0x00, 0x0a, 0x0a, 0x0a, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00}, // '"'
|
| 11 |
+
{0x00, 0x0a, 0x1f, 0x0a, 0x1f, 0x0a, 0x00, 0x00, 0x00, 0x00}, // '#'
|
| 12 |
+
{0x04, 0x0e, 0x15, 0x05, 0x0e, 0x14, 0x15, 0x0e, 0x04, 0x00}, // '$'
|
| 13 |
+
{0x00, 0x12, 0x15, 0x0a, 0x04, 0x0a, 0x15, 0x09, 0x00, 0x00}, // '%'
|
| 14 |
+
{0x00, 0x06, 0x09, 0x05, 0x02, 0x15, 0x09, 0x16, 0x00, 0x00}, // '&'
|
| 15 |
+
{0x04, 0x04, 0x04, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00}, // "'"
|
| 16 |
+
{0x08, 0x04, 0x04, 0x02, 0x02, 0x02, 0x04, 0x04, 0x08, 0x00}, // '('
|
| 17 |
+
{0x04, 0x08, 0x08, 0x10, 0x10, 0x10, 0x08, 0x08, 0x04, 0x00}, // ')'
|
| 18 |
+
{0x00, 0x04, 0x15, 0x0e, 0x15, 0x04, 0x00, 0x00, 0x00, 0x00}, // '*'
|
| 19 |
+
{0x00, 0x00, 0x04, 0x04, 0x1f, 0x04, 0x04, 0x00, 0x00, 0x00}, // '+'
|
| 20 |
+
{0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x06, 0x06, 0x04, 0x00}, // ','
|
| 21 |
+
{0x00, 0x00, 0x00, 0x00, 0x1f, 0x00, 0x00, 0x00, 0x00, 0x00}, // '-'
|
| 22 |
+
{0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x0c, 0x0c, 0x00, 0x00}, // '.'
|
| 23 |
+
{0x20, 0x10, 0x10, 0x08, 0x08, 0x04, 0x04, 0x02, 0x02, 0x00}, // '/'
|
| 24 |
+
{0x00, 0x0e, 0x11, 0x19, 0x15, 0x13, 0x11, 0x0e, 0x00, 0x00}, // '0'
|
| 25 |
+
{0x00, 0x04, 0x06, 0x04, 0x04, 0x04, 0x04, 0x04, 0x00, 0x00}, // '1'
|
| 26 |
+
{0x00, 0x0e, 0x11, 0x10, 0x08, 0x04, 0x02, 0x1f, 0x00, 0x00}, // '2'
|
| 27 |
+
{0x00, 0x0e, 0x11, 0x10, 0x0c, 0x10, 0x11, 0x0e, 0x00, 0x00}, // '3'
|
| 28 |
+
{0x00, 0x08, 0x0c, 0x0a, 0x09, 0x1f, 0x08, 0x08, 0x00, 0x00}, // '4'
|
| 29 |
+
{0x00, 0x1f, 0x01, 0x0f, 0x10, 0x10, 0x11, 0x0e, 0x00, 0x00}, // '5'
|
| 30 |
+
{0x00, 0x0e, 0x01, 0x0f, 0x11, 0x11, 0x11, 0x0e, 0x00, 0x00}, // '6'
|
| 31 |
+
{0x00, 0x1f, 0x10, 0x10, 0x08, 0x04, 0x04, 0x04, 0x00, 0x00}, // '7'
|
| 32 |
+
{0x00, 0x0e, 0x11, 0x11, 0x0e, 0x11, 0x11, 0x0e, 0x00, 0x00}, // '8'
|
| 33 |
+
{0x00, 0x0e, 0x11, 0x11, 0x11, 0x1e, 0x10, 0x0e, 0x00, 0x00}, // '9'
|
| 34 |
+
{0x00, 0x00, 0x0c, 0x0c, 0x00, 0x00, 0x0c, 0x0c, 0x00, 0x00}, // ':'
|
| 35 |
+
{0x00, 0x00, 0x06, 0x06, 0x00, 0x00, 0x06, 0x06, 0x04, 0x00}, // ';'
|
| 36 |
+
{0x00, 0x10, 0x08, 0x04, 0x02, 0x04, 0x08, 0x10, 0x00, 0x00}, // '<'
|
| 37 |
+
{0x00, 0x00, 0x00, 0x1f, 0x00, 0x1f, 0x00, 0x00, 0x00, 0x00}, // '='
|
| 38 |
+
{0x00, 0x02, 0x04, 0x08, 0x10, 0x08, 0x04, 0x02, 0x00, 0x00}, // '>'
|
| 39 |
+
{0x00, 0x0e, 0x11, 0x10, 0x08, 0x04, 0x00, 0x04, 0x00, 0x00}, // '?'
|
| 40 |
+
{0x0e, 0x11, 0x11, 0x15, 0x17, 0x0d, 0x01, 0x11, 0x0e, 0x00}, // '@'
|
| 41 |
+
{0x00, 0x0e, 0x11, 0x11, 0x1f, 0x11, 0x11, 0x11, 0x00, 0x00}, // 'A'
|
| 42 |
+
{0x00, 0x0f, 0x11, 0x11, 0x0f, 0x11, 0x11, 0x0f, 0x00, 0x00}, // 'B'
|
| 43 |
+
{0x00, 0x0e, 0x11, 0x01, 0x01, 0x01, 0x11, 0x0e, 0x00, 0x00}, // 'C'
|
| 44 |
+
{0x00, 0x0f, 0x11, 0x11, 0x11, 0x11, 0x11, 0x0f, 0x00, 0x00}, // 'D'
|
| 45 |
+
{0x00, 0x1f, 0x01, 0x01, 0x0f, 0x01, 0x01, 0x1f, 0x00, 0x00}, // 'E'
|
| 46 |
+
{0x00, 0x1f, 0x01, 0x01, 0x0f, 0x01, 0x01, 0x01, 0x00, 0x00}, // 'F'
|
| 47 |
+
{0x00, 0x0e, 0x11, 0x01, 0x19, 0x11, 0x11, 0x0e, 0x00, 0x00}, // 'G'
|
| 48 |
+
{0x00, 0x11, 0x11, 0x11, 0x1f, 0x11, 0x11, 0x11, 0x00, 0x00}, // 'H'
|
| 49 |
+
{0x00, 0x04, 0x04, 0x04, 0x04, 0x04, 0x04, 0x04, 0x00, 0x00}, // 'I'
|
| 50 |
+
{0x00, 0x10, 0x10, 0x10, 0x10, 0x11, 0x11, 0x0e, 0x00, 0x00}, // 'J'
|
| 51 |
+
{0x00, 0x11, 0x09, 0x05, 0x03, 0x05, 0x09, 0x11, 0x00, 0x00}, // 'K'
|
| 52 |
+
{0x00, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x1f, 0x00, 0x00}, // 'L'
|
| 53 |
+
{0x00, 0x11, 0x1b, 0x15, 0x11, 0x11, 0x11, 0x11, 0x00, 0x00}, // 'M'
|
| 54 |
+
{0x00, 0x11, 0x13, 0x15, 0x19, 0x11, 0x11, 0x11, 0x00, 0x00}, // 'N'
|
| 55 |
+
{0x00, 0x0e, 0x11, 0x11, 0x11, 0x11, 0x11, 0x0e, 0x00, 0x00}, // 'O'
|
| 56 |
+
{0x00, 0x0f, 0x11, 0x11, 0x0f, 0x01, 0x01, 0x01, 0x00, 0x00}, // 'P'
|
| 57 |
+
{0x00, 0x0e, 0x11, 0x11, 0x11, 0x11, 0x11, 0x0e, 0x10, 0x00}, // 'Q'
|
| 58 |
+
{0x00, 0x0f, 0x11, 0x11, 0x0f, 0x11, 0x11, 0x11, 0x00, 0x00}, // 'R'
|
| 59 |
+
{0x00, 0x0e, 0x11, 0x01, 0x0e, 0x10, 0x11, 0x0e, 0x00, 0x00}, // 'S'
|
| 60 |
+
{0x00, 0x1f, 0x04, 0x04, 0x04, 0x04, 0x04, 0x04, 0x00, 0x00}, // 'T'
|
| 61 |
+
{0x00, 0x11, 0x11, 0x11, 0x11, 0x11, 0x11, 0x0e, 0x00, 0x00}, // 'U'
|
| 62 |
+
{0x00, 0x11, 0x11, 0x11, 0x0a, 0x0a, 0x04, 0x04, 0x00, 0x00}, // 'V'
|
| 63 |
+
{0x00, 0x11, 0x11, 0x11, 0x11, 0x15, 0x1b, 0x11, 0x00, 0x00}, // 'W'
|
| 64 |
+
{0x00, 0x11, 0x0a, 0x04, 0x04, 0x04, 0x0a, 0x11, 0x00, 0x00}, // 'X'
|
| 65 |
+
{0x00, 0x11, 0x11, 0x11, 0x0a, 0x04, 0x04, 0x04, 0x00, 0x00}, // 'Y'
|
| 66 |
+
{0x00, 0x1f, 0x10, 0x08, 0x04, 0x02, 0x01, 0x1f, 0x00, 0x00}, // 'Z'
|
| 67 |
+
{0x0c, 0x04, 0x04, 0x04, 0x04, 0x04, 0x04, 0x04, 0x0c, 0x00}, // '['
|
| 68 |
+
{0x01, 0x02, 0x02, 0x04, 0x04, 0x08, 0x08, 0x10, 0x10, 0x00}, // '\\'
|
| 69 |
+
{0x0c, 0x08, 0x08, 0x08, 0x08, 0x08, 0x08, 0x08, 0x0c, 0x00}, // ']'
|
| 70 |
+
{0x08, 0x14, 0x22, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00}, // '^'
|
| 71 |
+
{0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x3f, 0x00}, // '_'
|
| 72 |
+
{0x02, 0x04, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00}, // '`'
|
| 73 |
+
{0x00, 0x00, 0x00, 0x1e, 0x11, 0x11, 0x19, 0x16, 0x00, 0x00}, // 'a'
|
| 74 |
+
{0x00, 0x01, 0x01, 0x0f, 0x11, 0x11, 0x11, 0x0f, 0x00, 0x00}, // 'b'
|
| 75 |
+
{0x00, 0x00, 0x00, 0x0e, 0x11, 0x01, 0x01, 0x1e, 0x00, 0x00}, // 'c'
|
| 76 |
+
{0x00, 0x10, 0x10, 0x1e, 0x11, 0x11, 0x11, 0x1e, 0x00, 0x00}, // 'd'
|
| 77 |
+
{0x00, 0x00, 0x00, 0x0e, 0x11, 0x1f, 0x01, 0x1e, 0x00, 0x00}, // 'e'
|
| 78 |
+
{0x00, 0x18, 0x04, 0x0e, 0x04, 0x04, 0x04, 0x04, 0x00, 0x00}, // 'f'
|
| 79 |
+
{0x00, 0x00, 0x00, 0x1e, 0x11, 0x11, 0x11, 0x1e, 0x10, 0x0e}, // 'g'
|
| 80 |
+
{0x00, 0x01, 0x01, 0x0f, 0x11, 0x11, 0x11, 0x11, 0x00, 0x00}, // 'h'
|
| 81 |
+
{0x00, 0x04, 0x00, 0x04, 0x04, 0x04, 0x04, 0x04, 0x00, 0x00}, // 'i'
|
| 82 |
+
{0x00, 0x04, 0x00, 0x04, 0x04, 0x04, 0x04, 0x04, 0x04, 0x03}, // 'j'
|
| 83 |
+
{0x00, 0x01, 0x01, 0x09, 0x05, 0x07, 0x09, 0x11, 0x00, 0x00}, // 'k'
|
| 84 |
+
{0x00, 0x04, 0x04, 0x04, 0x04, 0x04, 0x04, 0x0c, 0x00, 0x00}, // 'l'
|
| 85 |
+
{0x00, 0x00, 0x00, 0x0f, 0x15, 0x15, 0x15, 0x15, 0x00, 0x00}, // 'm'
|
| 86 |
+
{0x00, 0x00, 0x00, 0x0d, 0x13, 0x11, 0x11, 0x11, 0x00, 0x00}, // 'n'
|
| 87 |
+
{0x00, 0x00, 0x00, 0x0e, 0x11, 0x11, 0x11, 0x0e, 0x00, 0x00}, // 'o'
|
| 88 |
+
{0x00, 0x00, 0x00, 0x0f, 0x11, 0x11, 0x11, 0x0f, 0x01, 0x01}, // 'p'
|
| 89 |
+
{0x00, 0x00, 0x00, 0x1e, 0x11, 0x11, 0x11, 0x1e, 0x10, 0x10}, // 'q'
|
| 90 |
+
{0x00, 0x00, 0x00, 0x0d, 0x13, 0x01, 0x01, 0x01, 0x00, 0x00}, // 'r'
|
| 91 |
+
{0x00, 0x00, 0x00, 0x1e, 0x01, 0x0e, 0x10, 0x0f, 0x00, 0x00}, // 's'
|
| 92 |
+
{0x00, 0x04, 0x04, 0x1e, 0x04, 0x04, 0x04, 0x18, 0x00, 0x00}, // 't'
|
| 93 |
+
{0x00, 0x00, 0x00, 0x11, 0x11, 0x11, 0x19, 0x16, 0x00, 0x00}, // 'u'
|
| 94 |
+
{0x00, 0x00, 0x00, 0x11, 0x11, 0x0a, 0x0a, 0x04, 0x00, 0x00}, // 'v'
|
| 95 |
+
{0x00, 0x00, 0x00, 0x15, 0x15, 0x15, 0x15, 0x0a, 0x00, 0x00}, // 'w'
|
| 96 |
+
{0x00, 0x00, 0x00, 0x11, 0x0a, 0x04, 0x0a, 0x11, 0x00, 0x00}, // 'x'
|
| 97 |
+
{0x00, 0x00, 0x00, 0x11, 0x11, 0x11, 0x11, 0x1e, 0x10, 0x0e}, // 'y'
|
| 98 |
+
{0x00, 0x00, 0x00, 0x1f, 0x08, 0x04, 0x02, 0x1f, 0x00, 0x00}, // 'z'
|
| 99 |
+
{0x18, 0x04, 0x04, 0x04, 0x03, 0x04, 0x04, 0x04, 0x18, 0x00}, // '{'
|
| 100 |
+
{0x04, 0x04, 0x04, 0x04, 0x04, 0x04, 0x04, 0x04, 0x04, 0x00}, // '|'
|
| 101 |
+
{0x03, 0x04, 0x04, 0x04, 0x18, 0x04, 0x04, 0x04, 0x03, 0x00}, // '}'
|
| 102 |
+
{0x00, 0x16, 0x0d, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00}, // '~'
|
| 103 |
+
};
|
source/font.py
ADDED
|
@@ -0,0 +1,64 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""Rasterize printable ASCII from a monospace font into 6-pixel-wide glyphs for the GBA
|
| 2 |
+
screen, and write font.h. Needs Pillow; the generated font.h is kept, so this
|
| 3 |
+
only runs again to change the font.
|
| 4 |
+
|
| 5 |
+
python3 font.py [FONT_FILE] [SIZE] [CELL_HEIGHT]
|
| 6 |
+
"""
|
| 7 |
+
import sys
|
| 8 |
+
from pathlib import Path
|
| 9 |
+
|
| 10 |
+
from PIL import Image, ImageDraw, ImageFont
|
| 11 |
+
|
| 12 |
+
FONT = sys.argv[1] if len(sys.argv) > 1 else "/System/Library/Fonts/Monaco.ttf"
|
| 13 |
+
SIZE = int(sys.argv[2]) if len(sys.argv) > 2 else 9
|
| 14 |
+
WIDTH = 6
|
| 15 |
+
HEIGHT = int(sys.argv[3]) if len(sys.argv) > 3 else 10
|
| 16 |
+
|
| 17 |
+
font = ImageFont.truetype(FONT, SIZE)
|
| 18 |
+
ascent, _ = font.getmetrics()
|
| 19 |
+
rows = []
|
| 20 |
+
for code in range(32, 127):
|
| 21 |
+
image = Image.new("1", (WIDTH, HEIGHT), 0)
|
| 22 |
+
draw = ImageDraw.Draw(image)
|
| 23 |
+
draw.fontmode = "1" # no antialiasing: on or off
|
| 24 |
+
draw.text((0, HEIGHT - 1 - ascent), chr(code), font=font, fill=1)
|
| 25 |
+
glyph = []
|
| 26 |
+
for y in range(HEIGHT):
|
| 27 |
+
bits = 0
|
| 28 |
+
for x in range(WIDTH):
|
| 29 |
+
if image.getpixel((x, y)):
|
| 30 |
+
bits |= 1 << x
|
| 31 |
+
glyph.append(bits)
|
| 32 |
+
rows.append(glyph)
|
| 33 |
+
|
| 34 |
+
# Monaco at this size gives the descenders one row, which makes g and q the
|
| 35 |
+
# same glyph: move every glyph up a row (row 0 is blank but for the tops of
|
| 36 |
+
# / \ |) and draw g j p q y by hand with two-row descenders, and _ (the
|
| 37 |
+
# typing cursor) just below the baseline again.
|
| 38 |
+
rows = [glyph[1:] + [0] for glyph in rows]
|
| 39 |
+
HAND = {
|
| 40 |
+
"g": ["......", "......", "......", ".####.", "#...#.", "#...#.", "#...#.", ".####.", "....#.", ".###.."],
|
| 41 |
+
"j": ["......", "..#...", "......", "..#...", "..#...", "..#...", "..#...", "..#...", "..#...", "##...."],
|
| 42 |
+
"p": ["......", "......", "......", "####..", "#...#.", "#...#.", "#...#.", "####..", "#.....", "#....."],
|
| 43 |
+
"q": ["......", "......", "......", ".####.", "#...#.", "#...#.", "#...#.", ".####.", "....#.", "....#."],
|
| 44 |
+
"y": ["......", "......", "......", "#...#.", "#...#.", "#...#.", "#...#.", ".####.", "....#.", ".###.."],
|
| 45 |
+
"_": ["......", "......", "......", "......", "......", "......", "......", "......", "######", "......"],
|
| 46 |
+
}
|
| 47 |
+
assert HEIGHT == 10 and WIDTH == 6
|
| 48 |
+
for c, picture in HAND.items():
|
| 49 |
+
rows[ord(c) - 32] = [sum(1 << x for x, p in enumerate(line) if p == "#") for line in picture]
|
| 50 |
+
|
| 51 |
+
lines = [f" {{{', '.join(f'0x{b:02x}' for b in g)}}}, // {chr(32 + i)!r}" for i, g in enumerate(rows)]
|
| 52 |
+
Path(__file__).with_name("font.h").write_text(
|
| 53 |
+
f"// Generated by font.py from {Path(FONT).name} at {SIZE} px, with hand-drawn descenders; do not edit.\n"
|
| 54 |
+
f"// Glyphs for ASCII 32-126, {WIDTH}x{HEIGHT}: one byte per row, bit x is column x.\n"
|
| 55 |
+
"#pragma once\n#include <stdint.h>\n"
|
| 56 |
+
f"#define FONT_WIDTH {WIDTH}\n#define FONT_HEIGHT {HEIGHT}\n"
|
| 57 |
+
f"static const uint8_t font[95][{HEIGHT}] = {{\n" + "\n".join(lines) + "\n};\n"
|
| 58 |
+
)
|
| 59 |
+
|
| 60 |
+
# A preview in the terminal.
|
| 61 |
+
for start in (33, 65, 97):
|
| 62 |
+
for y in range(HEIGHT):
|
| 63 |
+
print("".join(("#" if rows[c - 32][y] >> x & 1 else ".") for c in range(start, start + 12) for x in range(WIDTH)))
|
| 64 |
+
print()
|
source/gba.ld
ADDED
|
@@ -0,0 +1,53 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
/* The GBA's memory: cartridge ROM (weights and code), 32 KB of fast internal
|
| 2 |
+
work RAM (the runner's mailbox first, at 0x03000000, then the hot code and
|
| 3 |
+
working vectors; the top 4 KB is left to the stacks), and 256 KB of external
|
| 4 |
+
work RAM (the KV cache and canon histories). */
|
| 5 |
+
OUTPUT_FORMAT("elf32-littlearm")
|
| 6 |
+
OUTPUT_ARCH(arm)
|
| 7 |
+
ENTRY(_start)
|
| 8 |
+
|
| 9 |
+
MEMORY {
|
| 10 |
+
rom : ORIGIN = 0x08000000, LENGTH = 32M
|
| 11 |
+
iwram : ORIGIN = 0x03000000, LENGTH = 0x7000
|
| 12 |
+
ewram : ORIGIN = 0x02000000, LENGTH = 256K
|
| 13 |
+
}
|
| 14 |
+
|
| 15 |
+
SECTIONS {
|
| 16 |
+
.text : {
|
| 17 |
+
KEEP(*(.crt0))
|
| 18 |
+
*(.text .text.*)
|
| 19 |
+
*(.rodata .rodata.*)
|
| 20 |
+
. = ALIGN(4);
|
| 21 |
+
} > rom
|
| 22 |
+
|
| 23 |
+
.shared (NOLOAD) : {
|
| 24 |
+
__shared_start = .;
|
| 25 |
+
KEEP(*(.shared))
|
| 26 |
+
. = ALIGN(4);
|
| 27 |
+
__shared_end = .;
|
| 28 |
+
} > iwram
|
| 29 |
+
|
| 30 |
+
.iwram : {
|
| 31 |
+
__iwram_start = .;
|
| 32 |
+
*(.iwram.text)
|
| 33 |
+
*(.data .data.*)
|
| 34 |
+
. = ALIGN(4);
|
| 35 |
+
__iwram_end = .;
|
| 36 |
+
} > iwram AT > rom
|
| 37 |
+
__iwram_lma = LOADADDR(.iwram);
|
| 38 |
+
|
| 39 |
+
/* Before .bss, so that it claims .bss.ewram first. */
|
| 40 |
+
.ewram (NOLOAD) : {
|
| 41 |
+
*(.bss.ewram)
|
| 42 |
+
. = ALIGN(4);
|
| 43 |
+
} > ewram
|
| 44 |
+
|
| 45 |
+
.bss (NOLOAD) : {
|
| 46 |
+
__bss_start = .;
|
| 47 |
+
*(.bss .bss.* COMMON)
|
| 48 |
+
. = ALIGN(4);
|
| 49 |
+
__bss_end = .;
|
| 50 |
+
} > iwram
|
| 51 |
+
|
| 52 |
+
/DISCARD/ : { *(.ARM.exidx*) *(.ARM.attributes) *(.comment) }
|
| 53 |
+
}
|
source/gba_include/string.h
ADDED
|
@@ -0,0 +1,7 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
// The GBA build has no C library: these are libc.c's.
|
| 2 |
+
#pragma once
|
| 3 |
+
#include <stddef.h>
|
| 4 |
+
void* memcpy(void* to, const void* from, size_t n);
|
| 5 |
+
void* memmove(void* to, const void* from, size_t n);
|
| 6 |
+
void* memset(void* to, int value, size_t n);
|
| 7 |
+
int memcmp(const void* a, const void* b, size_t n);
|
source/libc.c
ADDED
|
@@ -0,0 +1,36 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
// The few C library functions the compiler and engine call; the GBA build has
|
| 2 |
+
// no C library.
|
| 3 |
+
#include <stddef.h>
|
| 4 |
+
#include <stdint.h>
|
| 5 |
+
|
| 6 |
+
void* memcpy(void* to, const void* from, size_t n) {
|
| 7 |
+
uint8_t* d = to;
|
| 8 |
+
const uint8_t* s = from;
|
| 9 |
+
while (n--) *d++ = *s++;
|
| 10 |
+
return to;
|
| 11 |
+
}
|
| 12 |
+
|
| 13 |
+
void* memmove(void* to, const void* from, size_t n) {
|
| 14 |
+
uint8_t* d = to;
|
| 15 |
+
const uint8_t* s = from;
|
| 16 |
+
if (d < s) {
|
| 17 |
+
while (n--) *d++ = *s++;
|
| 18 |
+
} else {
|
| 19 |
+
while (n--) d[n] = s[n];
|
| 20 |
+
}
|
| 21 |
+
return to;
|
| 22 |
+
}
|
| 23 |
+
|
| 24 |
+
void* memset(void* to, int value, size_t n) {
|
| 25 |
+
uint8_t* d = to;
|
| 26 |
+
while (n--) *d++ = (uint8_t)value;
|
| 27 |
+
return to;
|
| 28 |
+
}
|
| 29 |
+
|
| 30 |
+
int memcmp(const void* a, const void* b, size_t n) {
|
| 31 |
+
const uint8_t *x = a, *y = b;
|
| 32 |
+
for (; n--; ++x, ++y) {
|
| 33 |
+
if (*x != *y) return *x - *y;
|
| 34 |
+
}
|
| 35 |
+
return 0;
|
| 36 |
+
}
|
source/main.c
ADDED
|
@@ -0,0 +1,581 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
// ChatGBA on the Game Boy Advance: chat with an on-screen keyboard.
|
| 2 |
+
//
|
| 3 |
+
// The screen: the conversation at the top, then the message being typed, then
|
| 4 |
+
// the keyboard. While the model reads and replies, the keyboard gives way to
|
| 5 |
+
// the conversation and a status line. Controls:
|
| 6 |
+
// D-pad move on the keyboard A type the key
|
| 7 |
+
// B delete (hold: stop a reply) START send
|
| 8 |
+
// R shift, then caps lock, then off
|
| 9 |
+
// L a sample question SELECT new conversation
|
| 10 |
+
//
|
| 11 |
+
// The headless runner (runner.c) can also ask questions: it writes them into
|
| 12 |
+
// the mailbox at the start of internal RAM, 0x03000000, and reads the reply
|
| 13 |
+
// back from it as the bytes appear.
|
| 14 |
+
#include <stdint.h>
|
| 15 |
+
|
| 16 |
+
#include "chat.h"
|
| 17 |
+
#include "engine.h"
|
| 18 |
+
#include "font.h"
|
| 19 |
+
|
| 20 |
+
#define REG_DISPCNT (*(volatile uint16_t*)0x04000000)
|
| 21 |
+
#define REG_VCOUNT (*(volatile uint16_t*)0x04000006)
|
| 22 |
+
#define REG_KEYINPUT (*(volatile uint16_t*)0x04000130)
|
| 23 |
+
#define REG_WAITCNT (*(volatile uint16_t*)0x04000204)
|
| 24 |
+
#define REG_TM2CNT_L (*(volatile uint16_t*)0x04000108)
|
| 25 |
+
#define REG_TM2CNT_H (*(volatile uint16_t*)0x0400010A)
|
| 26 |
+
#define REG_TM3CNT_L (*(volatile uint16_t*)0x0400010C)
|
| 27 |
+
#define REG_TM3CNT_H (*(volatile uint16_t*)0x0400010E)
|
| 28 |
+
#define VRAM ((volatile uint16_t*)0x06000000)
|
| 29 |
+
#define EWRAM __attribute__((section(".bss.ewram"))) // the 256 KB external RAM, past the KV cache
|
| 30 |
+
|
| 31 |
+
#define KEY_A 0x0001
|
| 32 |
+
#define KEY_B 0x0002
|
| 33 |
+
#define KEY_SELECT 0x0004
|
| 34 |
+
#define KEY_START 0x0008
|
| 35 |
+
#define KEY_RIGHT 0x0010
|
| 36 |
+
#define KEY_LEFT 0x0020
|
| 37 |
+
#define KEY_UP 0x0040
|
| 38 |
+
#define KEY_DOWN 0x0080
|
| 39 |
+
#define KEY_R 0x0100
|
| 40 |
+
#define KEY_L 0x0200
|
| 41 |
+
#define KEY_REPEATS (KEY_RIGHT | KEY_LEFT | KEY_UP | KEY_DOWN | KEY_B) // held down, these repeat
|
| 42 |
+
|
| 43 |
+
#define SCREEN_W 240
|
| 44 |
+
#define SCREEN_H 160
|
| 45 |
+
#define COLUMNS (SCREEN_W / FONT_WIDTH) // 40
|
| 46 |
+
#define RGB(r, g, b) ((r) | (g) << 5 | (b) << 10)
|
| 47 |
+
#define BACKGROUND RGB(2, 3, 6)
|
| 48 |
+
#define TITLE RGB(31, 24, 8)
|
| 49 |
+
#define USER RGB(14, 26, 31)
|
| 50 |
+
#define MODEL RGB(28, 28, 28)
|
| 51 |
+
#define DIM RGB(14, 14, 16)
|
| 52 |
+
#define RULE RGB(6, 7, 11)
|
| 53 |
+
#define KEY_FACE RGB(6, 7, 12)
|
| 54 |
+
|
| 55 |
+
extern const uint8_t model_blob[];
|
| 56 |
+
|
| 57 |
+
// The runner's mailbox. Its layout is part of the runner's protocol: runner.c
|
| 58 |
+
// reads and writes these offsets.
|
| 59 |
+
#define MAILBOX_MAGIC 0x42475241 // "ARGB": ready
|
| 60 |
+
typedef struct {
|
| 61 |
+
uint32_t magic; // 0
|
| 62 |
+
volatile uint32_t request; // 4: the runner adds one per command
|
| 63 |
+
volatile uint32_t done; // 8: set to request when the command is finished
|
| 64 |
+
uint32_t command; // 12: 1 reply to message, 2 forget the conversation
|
| 65 |
+
uint32_t length; // 16: message bytes
|
| 66 |
+
int32_t temperature; // 20: Q16
|
| 67 |
+
uint32_t seed; // 24
|
| 68 |
+
uint32_t max_reply; // 28
|
| 69 |
+
uint8_t message[512]; // 32
|
| 70 |
+
volatile uint32_t out_length; // 544: reply bytes so far
|
| 71 |
+
uint8_t out[1024]; // 548
|
| 72 |
+
volatile uint32_t prompt_done; // 1572: bytes processed before replying
|
| 73 |
+
volatile uint32_t prompt_total; // 1576
|
| 74 |
+
int32_t status; // 1580: engine_init's result
|
| 75 |
+
uint32_t prompt_ticks; // 1584: time to process them, in 1/16384 s
|
| 76 |
+
uint32_t reply_ticks; // 1588: time to generate the reply
|
| 77 |
+
int32_t reply_length; // 1592: chat_reply's result
|
| 78 |
+
uint32_t profile[PROFILE_PARTS]; // 1596: cycles per part, with PROFILE
|
| 79 |
+
} Mailbox;
|
| 80 |
+
Mailbox mailbox __attribute__((section(".shared")));
|
| 81 |
+
|
| 82 |
+
static const char* const questions[] = {
|
| 83 |
+
"What is a rainbow?",
|
| 84 |
+
"Why do we need to sleep?",
|
| 85 |
+
"Can you tell me a short story about a cat?",
|
| 86 |
+
"What do plants need to grow?",
|
| 87 |
+
"How does a bicycle stay up?",
|
| 88 |
+
};
|
| 89 |
+
|
| 90 |
+
// --- Drawing -------------------------------------------------------------------
|
| 91 |
+
|
| 92 |
+
static void fill(int x, int y, int w, int h, uint16_t color) {
|
| 93 |
+
for (int j = y; j < y + h; ++j) {
|
| 94 |
+
for (int i = x; i < x + w; ++i) VRAM[j * SCREEN_W + i] = color;
|
| 95 |
+
}
|
| 96 |
+
}
|
| 97 |
+
|
| 98 |
+
static void draw_char(int x, int y, int c, uint16_t ink, uint16_t paper) {
|
| 99 |
+
const uint8_t* glyph = font[(c < 32 || c > 126 ? '?' : c) - 32];
|
| 100 |
+
volatile uint16_t* at = VRAM + y * SCREEN_W + x;
|
| 101 |
+
for (int j = 0; j < FONT_HEIGHT; ++j) {
|
| 102 |
+
for (int i = 0; i < FONT_WIDTH; ++i) at[j * SCREEN_W + i] = (glyph[j] >> i & 1) ? ink : paper;
|
| 103 |
+
}
|
| 104 |
+
}
|
| 105 |
+
|
| 106 |
+
static void draw_text(int x, int y, const char* text, uint16_t ink, uint16_t paper) {
|
| 107 |
+
for (; *text; ++text, x += FONT_WIDTH) draw_char(x, y, (uint8_t)*text, ink, paper);
|
| 108 |
+
}
|
| 109 |
+
|
| 110 |
+
// Building short strings: each returns the new end, always terminated.
|
| 111 |
+
static char* add(char* at, const char* text) {
|
| 112 |
+
while (*text) *at++ = *text++;
|
| 113 |
+
*at = 0;
|
| 114 |
+
return at;
|
| 115 |
+
}
|
| 116 |
+
|
| 117 |
+
static char* add_number(char* at, uint32_t n) {
|
| 118 |
+
char digits[12];
|
| 119 |
+
int i = 0;
|
| 120 |
+
do digits[i++] = (char)('0' + n % 10); while (n /= 10);
|
| 121 |
+
while (i) *at++ = digits[--i];
|
| 122 |
+
*at = 0;
|
| 123 |
+
return at;
|
| 124 |
+
}
|
| 125 |
+
|
| 126 |
+
// A duration in 1/16384 s: "52.3s" or "4m24s".
|
| 127 |
+
static char* add_time(char* at, uint32_t ticks) {
|
| 128 |
+
uint32_t tenths = (ticks * 10ull + 8192) / 16384;
|
| 129 |
+
if (tenths < 600) {
|
| 130 |
+
at = add_number(at, tenths / 10);
|
| 131 |
+
*at++ = '.';
|
| 132 |
+
at = add_number(at, tenths % 10);
|
| 133 |
+
} else {
|
| 134 |
+
uint32_t s = tenths / 10;
|
| 135 |
+
at = add_number(at, s / 60);
|
| 136 |
+
*at++ = 'm';
|
| 137 |
+
if (s % 60 < 10) *at++ = '0';
|
| 138 |
+
at = add_number(at, s % 60);
|
| 139 |
+
}
|
| 140 |
+
return add(at, "s");
|
| 141 |
+
}
|
| 142 |
+
|
| 143 |
+
// --- The conversation: wrapped lines, the last few on screen ------------------
|
| 144 |
+
|
| 145 |
+
#define LOG_LINES 128
|
| 146 |
+
typedef struct {
|
| 147 |
+
char text[COLUMNS];
|
| 148 |
+
uint8_t length;
|
| 149 |
+
uint16_t color;
|
| 150 |
+
} Line;
|
| 151 |
+
static Line lines[LOG_LINES] EWRAM; // a ring: line i is lines[i % LOG_LINES]
|
| 152 |
+
static int line_count; // lines so far; the last is the one being written
|
| 153 |
+
static int dirty_from; // the first line changed since it was drawn
|
| 154 |
+
static int log_rows; // rows of text the conversation has on screen
|
| 155 |
+
static int shown_top = -1; // the line at the top of the screen when last drawn
|
| 156 |
+
|
| 157 |
+
static Line* line(int i) { return &lines[i % LOG_LINES]; }
|
| 158 |
+
|
| 159 |
+
static void mark(int i) {
|
| 160 |
+
if (i < dirty_from) dirty_from = i;
|
| 161 |
+
}
|
| 162 |
+
|
| 163 |
+
static Line* new_line(uint16_t color) {
|
| 164 |
+
Line* l = line(line_count);
|
| 165 |
+
l->length = 0;
|
| 166 |
+
l->color = color;
|
| 167 |
+
mark(line_count++);
|
| 168 |
+
return l;
|
| 169 |
+
}
|
| 170 |
+
|
| 171 |
+
// Start writing in color, on a fresh line unless the current one is empty.
|
| 172 |
+
static void start(uint16_t color) {
|
| 173 |
+
Line* l = line(line_count - 1);
|
| 174 |
+
if (l->length) l = new_line(color);
|
| 175 |
+
l->color = color;
|
| 176 |
+
mark(line_count - 1);
|
| 177 |
+
}
|
| 178 |
+
|
| 179 |
+
// One byte of UTF-8 text: ASCII as is, one '?' per other character. A word that
|
| 180 |
+
// would run past the edge moves to the next line.
|
| 181 |
+
static void log_byte(int byte) {
|
| 182 |
+
if (byte == '\n') {
|
| 183 |
+
new_line(line(line_count - 1)->color);
|
| 184 |
+
return;
|
| 185 |
+
}
|
| 186 |
+
if (byte == '\t') byte = ' ';
|
| 187 |
+
if ((byte >= 0x80 && byte < 0xC0) || byte < 32 || byte == 127) return; // continuations, controls
|
| 188 |
+
if (byte >= 0xC0) byte = '?';
|
| 189 |
+
|
| 190 |
+
Line* l = line(line_count - 1);
|
| 191 |
+
if (l->length == COLUMNS) {
|
| 192 |
+
if (byte == ' ') {
|
| 193 |
+
new_line(l->color);
|
| 194 |
+
return;
|
| 195 |
+
}
|
| 196 |
+
int space = COLUMNS - 1;
|
| 197 |
+
while (space > 0 && l->text[space] != ' ') --space;
|
| 198 |
+
Line* next = new_line(l->color);
|
| 199 |
+
if (space > 0) {
|
| 200 |
+
for (int i = space + 1; i < COLUMNS; ++i) next->text[next->length++] = l->text[i];
|
| 201 |
+
l->length = (uint8_t)space;
|
| 202 |
+
mark(line_count - 2);
|
| 203 |
+
}
|
| 204 |
+
l = next;
|
| 205 |
+
}
|
| 206 |
+
l->text[l->length++] = (char)byte;
|
| 207 |
+
mark(line_count - 1);
|
| 208 |
+
}
|
| 209 |
+
|
| 210 |
+
static void log_text(const char* text, uint16_t color) {
|
| 211 |
+
start(color);
|
| 212 |
+
while (*text) log_byte((uint8_t)*text++);
|
| 213 |
+
}
|
| 214 |
+
|
| 215 |
+
// Draw the lines that changed, or all of them if the conversation scrolled.
|
| 216 |
+
static void draw_log(void) {
|
| 217 |
+
int top = line_count > log_rows ? line_count - log_rows : 0;
|
| 218 |
+
int from = top != shown_top ? top : (dirty_from > top ? dirty_from : top);
|
| 219 |
+
for (int i = from; i < top + log_rows; ++i) {
|
| 220 |
+
int y = (i - top) * FONT_HEIGHT;
|
| 221 |
+
if (i >= line_count) {
|
| 222 |
+
fill(0, y, SCREEN_W, FONT_HEIGHT, BACKGROUND);
|
| 223 |
+
continue;
|
| 224 |
+
}
|
| 225 |
+
Line* l = line(i);
|
| 226 |
+
for (int c = 0; c < COLUMNS; ++c) draw_char(c * FONT_WIDTH, y, c < l->length ? l->text[c] : ' ', l->color, BACKGROUND);
|
| 227 |
+
}
|
| 228 |
+
shown_top = top;
|
| 229 |
+
dirty_from = line_count;
|
| 230 |
+
}
|
| 231 |
+
|
| 232 |
+
// --- The keyboard --------------------------------------------------------------
|
| 233 |
+
|
| 234 |
+
// Below the conversation's 8 rows: a rule, 2 rows for the message being typed,
|
| 235 |
+
// then 4 rows of 12 keys and a row of wide keys.
|
| 236 |
+
#define TYPING_ROWS 8
|
| 237 |
+
#define INPUT_Y 82
|
| 238 |
+
#define KEYS_Y 104
|
| 239 |
+
#define KEY_W 20
|
| 240 |
+
#define KEY_H 11
|
| 241 |
+
#define INPUT_MAX 200
|
| 242 |
+
|
| 243 |
+
static const char keys_lower[4][13] = {"1234567890-'", "qwertyuiop?!", "asdfghjkl.,:", "zxcvbnm;()\"/"};
|
| 244 |
+
static const char keys_upper[4][13] = {"@#$%&*+=_<>~", "QWERTYUIOP?!", "ASDFGHJKL.,:", "ZXCVBNM;()\"/"};
|
| 245 |
+
// The bottom row: first column of each wide key, and one past the last.
|
| 246 |
+
static const int wide_starts[5] = {0, 2, 8, 10, 12};
|
| 247 |
+
enum { WIDE_SHIFT, WIDE_SPACE, WIDE_DELETE, WIDE_SEND };
|
| 248 |
+
|
| 249 |
+
static char input[INPUT_MAX + 1] EWRAM;
|
| 250 |
+
static int input_length;
|
| 251 |
+
static int shift; // 0 off, 1 the next key, 2 caps lock
|
| 252 |
+
static int cursor_x, cursor_y;
|
| 253 |
+
|
| 254 |
+
static int wide_key(int x) {
|
| 255 |
+
int k = 0;
|
| 256 |
+
while (x >= wide_starts[k + 1]) ++k;
|
| 257 |
+
return k;
|
| 258 |
+
}
|
| 259 |
+
|
| 260 |
+
static void draw_key(int x, int y, int w, const char* label, int selected, uint16_t ink) {
|
| 261 |
+
int px = x * KEY_W, py = KEYS_Y + y * KEY_H;
|
| 262 |
+
uint16_t face = selected ? TITLE : KEY_FACE;
|
| 263 |
+
fill(px + 1, py, w * KEY_W - 2, KEY_H - 1, face);
|
| 264 |
+
int length = 0;
|
| 265 |
+
while (label[length]) ++length;
|
| 266 |
+
draw_text(px + (w * KEY_W - length * FONT_WIDTH) / 2, py, label, selected ? BACKGROUND : ink, face);
|
| 267 |
+
}
|
| 268 |
+
|
| 269 |
+
static void draw_one_key(int x, int y) {
|
| 270 |
+
if (y < 4) {
|
| 271 |
+
char label[2] = {(shift ? keys_upper : keys_lower)[y][x], 0};
|
| 272 |
+
draw_key(x, y, 1, label, cursor_y == y && cursor_x == x, MODEL);
|
| 273 |
+
return;
|
| 274 |
+
}
|
| 275 |
+
static const char* const wide_labels[4] = {"shift", "space", "del", "send"};
|
| 276 |
+
int k = wide_key(x);
|
| 277 |
+
const char* label = k == WIDE_SHIFT && shift == 2 ? "CAPS" : wide_labels[k];
|
| 278 |
+
uint16_t ink = k == WIDE_SHIFT && shift ? USER : k == WIDE_SEND ? TITLE : DIM;
|
| 279 |
+
draw_key(wide_starts[k], 4, wide_starts[k + 1] - wide_starts[k], label, cursor_y == 4 && k == wide_key(cursor_x), ink);
|
| 280 |
+
}
|
| 281 |
+
|
| 282 |
+
// Drawing is slow enough to miss a quick key press, so after the first time
|
| 283 |
+
// only the keys that changed are drawn again: where the cursor was and is, or
|
| 284 |
+
// every key when shift changes.
|
| 285 |
+
static int drawn_x = -1, drawn_y, drawn_shift;
|
| 286 |
+
|
| 287 |
+
static void draw_keyboard(void) {
|
| 288 |
+
if (drawn_x < 0 || shift != drawn_shift) {
|
| 289 |
+
for (int y = 0; y < 4; ++y) {
|
| 290 |
+
for (int x = 0; x < 12; ++x) draw_one_key(x, y);
|
| 291 |
+
}
|
| 292 |
+
for (int k = 0; k < 4; ++k) draw_one_key(wide_starts[k], 4);
|
| 293 |
+
} else if (drawn_x != cursor_x || drawn_y != cursor_y) {
|
| 294 |
+
draw_one_key(drawn_x, drawn_y);
|
| 295 |
+
draw_one_key(cursor_x, cursor_y);
|
| 296 |
+
}
|
| 297 |
+
drawn_x = cursor_x;
|
| 298 |
+
drawn_y = cursor_y;
|
| 299 |
+
drawn_shift = shift;
|
| 300 |
+
}
|
| 301 |
+
|
| 302 |
+
// "> " and the message, its last 2 lines' worth if it is longer, and a cursor.
|
| 303 |
+
// Only the characters that changed are drawn again.
|
| 304 |
+
static uint16_t drawn_input[2 * COLUMNS]; // character | ink << 8; 0 when unknown
|
| 305 |
+
|
| 306 |
+
static void draw_input(void) {
|
| 307 |
+
int total = 2 + input_length + 1;
|
| 308 |
+
int skip = total > 2 * COLUMNS ? total - 2 * COLUMNS : 0;
|
| 309 |
+
for (int i = 0; i < 2 * COLUMNS; ++i) {
|
| 310 |
+
int at = skip + i;
|
| 311 |
+
int c = at >= total ? ' ' : at < 2 ? "> "[at] : at - 2 < input_length ? input[at - 2] : '_';
|
| 312 |
+
int ink = at == total - 1 ? 1 : at < 2 ? 2 : 3;
|
| 313 |
+
uint16_t key = (uint16_t)(c | ink << 8);
|
| 314 |
+
if (drawn_input[i] == key) continue;
|
| 315 |
+
drawn_input[i] = key;
|
| 316 |
+
draw_char(i % COLUMNS * FONT_WIDTH, INPUT_Y + i / COLUMNS * FONT_HEIGHT, c,
|
| 317 |
+
ink == 1 ? TITLE : ink == 2 ? DIM : USER, BACKGROUND);
|
| 318 |
+
}
|
| 319 |
+
}
|
| 320 |
+
|
| 321 |
+
// Back to typing: the conversation's last 8 rows, the message, the keyboard.
|
| 322 |
+
static void typing_screen(void) {
|
| 323 |
+
log_rows = TYPING_ROWS;
|
| 324 |
+
shown_top = -1;
|
| 325 |
+
draw_log();
|
| 326 |
+
fill(0, TYPING_ROWS * FONT_HEIGHT, SCREEN_W, SCREEN_H - TYPING_ROWS * FONT_HEIGHT, BACKGROUND);
|
| 327 |
+
fill(0, TYPING_ROWS * FONT_HEIGHT + 1, SCREEN_W, 1, RULE);
|
| 328 |
+
for (int i = 0; i < 2 * COLUMNS; ++i) drawn_input[i] = 0;
|
| 329 |
+
drawn_x = -1;
|
| 330 |
+
draw_input();
|
| 331 |
+
draw_keyboard();
|
| 332 |
+
}
|
| 333 |
+
|
| 334 |
+
// --- Time ----------------------------------------------------------------------
|
| 335 |
+
|
| 336 |
+
// Timers 2 and 3 chained: one tick per 1024 cycles, 16,384 per second.
|
| 337 |
+
static uint32_t ticks(void) {
|
| 338 |
+
uint32_t high, low;
|
| 339 |
+
do {
|
| 340 |
+
high = REG_TM3CNT_L;
|
| 341 |
+
low = REG_TM2CNT_L;
|
| 342 |
+
} while (high != REG_TM3CNT_L);
|
| 343 |
+
return high << 16 | low;
|
| 344 |
+
}
|
| 345 |
+
|
| 346 |
+
// Timers 0 and 1 chained: every cycle, for profiling.
|
| 347 |
+
static uint32_t cycles(void) {
|
| 348 |
+
uint32_t high, low;
|
| 349 |
+
do {
|
| 350 |
+
high = *(volatile uint16_t*)0x04000104;
|
| 351 |
+
low = *(volatile uint16_t*)0x04000100;
|
| 352 |
+
} while (high != *(volatile uint16_t*)0x04000104);
|
| 353 |
+
return high << 16 | low;
|
| 354 |
+
}
|
| 355 |
+
|
| 356 |
+
static void wait_for_vblank(void) {
|
| 357 |
+
while (REG_VCOUNT >= SCREEN_H) {}
|
| 358 |
+
while (REG_VCOUNT < SCREEN_H) {}
|
| 359 |
+
}
|
| 360 |
+
|
| 361 |
+
// --- Chat ----------------------------------------------------------------------
|
| 362 |
+
|
| 363 |
+
// While the model works, the bottom row says what it is doing.
|
| 364 |
+
#define STATUS_Y (SCREEN_H - FONT_HEIGHT)
|
| 365 |
+
|
| 366 |
+
static void status(const char* text) {
|
| 367 |
+
char row[COLUMNS + 1];
|
| 368 |
+
int i = 0;
|
| 369 |
+
for (; text[i] && i < COLUMNS; ++i) row[i] = text[i];
|
| 370 |
+
for (; i < COLUMNS; ++i) row[i] = ' ';
|
| 371 |
+
row[COLUMNS] = 0;
|
| 372 |
+
draw_text(0, STATUS_Y, row, DIM, BACKGROUND);
|
| 373 |
+
}
|
| 374 |
+
|
| 375 |
+
static uint32_t reply_start;
|
| 376 |
+
static int stopped;
|
| 377 |
+
|
| 378 |
+
static void on_prompt(int done, int total) {
|
| 379 |
+
mailbox.prompt_done = done;
|
| 380 |
+
mailbox.prompt_total = total;
|
| 381 |
+
if (done == total) {
|
| 382 |
+
reply_start = ticks();
|
| 383 |
+
status("writing... hold B to stop");
|
| 384 |
+
} else {
|
| 385 |
+
char text[COLUMNS + 1];
|
| 386 |
+
char* at = add(text, "reading ");
|
| 387 |
+
at = add_number(at, done);
|
| 388 |
+
at = add(at, " of ");
|
| 389 |
+
at = add_number(at, total);
|
| 390 |
+
add(at, " bytes...");
|
| 391 |
+
status(text);
|
| 392 |
+
}
|
| 393 |
+
}
|
| 394 |
+
|
| 395 |
+
static void on_byte(int byte) {
|
| 396 |
+
if (mailbox.out_length < sizeof mailbox.out) mailbox.out[mailbox.out_length] = (uint8_t)byte;
|
| 397 |
+
mailbox.out_length = mailbox.out_length + 1;
|
| 398 |
+
if (byte == ' ' && line(line_count - 1)->length == 0) return; // replies start with a space
|
| 399 |
+
log_byte(byte);
|
| 400 |
+
draw_log();
|
| 401 |
+
}
|
| 402 |
+
|
| 403 |
+
static int stop(void) {
|
| 404 |
+
if (!(~REG_KEYINPUT & KEY_B)) return 0;
|
| 405 |
+
stopped = 1;
|
| 406 |
+
return 1;
|
| 407 |
+
}
|
| 408 |
+
|
| 409 |
+
static void ask(const uint8_t* message, int length, int32_t temperature, uint32_t seed, int max_reply) {
|
| 410 |
+
static uint8_t reply[1024] EWRAM;
|
| 411 |
+
mailbox.out_length = 0; // first: drawing takes frames, and the runner is watching
|
| 412 |
+
mailbox.prompt_done = mailbox.prompt_total = 0;
|
| 413 |
+
start(USER);
|
| 414 |
+
log_text("you> ", USER);
|
| 415 |
+
for (int i = 0; i < length; ++i) log_byte(message[i]);
|
| 416 |
+
start(MODEL);
|
| 417 |
+
|
| 418 |
+
// The whole screen for the conversation, the bottom row for the status.
|
| 419 |
+
log_rows = SCREEN_H / FONT_HEIGHT - 1;
|
| 420 |
+
shown_top = -1;
|
| 421 |
+
draw_log();
|
| 422 |
+
status("reading...");
|
| 423 |
+
|
| 424 |
+
ChatSettings settings = {
|
| 425 |
+
.temperature = temperature, .top_k = 40, .max_reply = max_reply,
|
| 426 |
+
.rng = seed ? seed : 1, .on_byte = on_byte, .on_prompt = on_prompt, .stop = stop,
|
| 427 |
+
};
|
| 428 |
+
stopped = 0;
|
| 429 |
+
uint32_t begin = ticks();
|
| 430 |
+
reply_start = begin;
|
| 431 |
+
for (int i = 0; i < PROFILE_PARTS; ++i) profile[i] = 0;
|
| 432 |
+
int n = chat_reply(message, length, &settings, reply, sizeof reply);
|
| 433 |
+
uint32_t end = ticks();
|
| 434 |
+
for (int i = 0; i < PROFILE_PARTS; ++i) mailbox.profile[i] = profile[i];
|
| 435 |
+
mailbox.reply_length = n;
|
| 436 |
+
mailbox.prompt_ticks = reply_start - begin;
|
| 437 |
+
mailbox.reply_ticks = end - reply_start;
|
| 438 |
+
|
| 439 |
+
char text[64];
|
| 440 |
+
char* at = add(text, "[read ");
|
| 441 |
+
at = add_number(at, mailbox.prompt_total);
|
| 442 |
+
at = add(at, "B in ");
|
| 443 |
+
at = add_time(at, mailbox.prompt_ticks);
|
| 444 |
+
at = add(at, ", wrote ");
|
| 445 |
+
at = add_number(at, n);
|
| 446 |
+
at = add(at, "B in ");
|
| 447 |
+
at = add_time(at, mailbox.reply_ticks);
|
| 448 |
+
add(at, stopped ? ", stopped]" : "]");
|
| 449 |
+
log_text(text, DIM);
|
| 450 |
+
start(MODEL);
|
| 451 |
+
}
|
| 452 |
+
|
| 453 |
+
static void new_conversation(void) {
|
| 454 |
+
chat_reset();
|
| 455 |
+
log_text("-- new conversation --", DIM);
|
| 456 |
+
start(MODEL);
|
| 457 |
+
}
|
| 458 |
+
|
| 459 |
+
// --- Typing --------------------------------------------------------------------
|
| 460 |
+
|
| 461 |
+
static void type(int c) {
|
| 462 |
+
if (input_length < INPUT_MAX) input[input_length++] = (char)c;
|
| 463 |
+
if (shift == 1) shift = 0;
|
| 464 |
+
}
|
| 465 |
+
|
| 466 |
+
static void delete_one(void) {
|
| 467 |
+
if (input_length) --input_length;
|
| 468 |
+
if (!input_length && !shift) shift = 1; // a new message starts with a capital
|
| 469 |
+
}
|
| 470 |
+
|
| 471 |
+
static void send(void) {
|
| 472 |
+
if (!input_length) return;
|
| 473 |
+
ask((const uint8_t*)input, input_length, (int32_t)(0.7 * 65536), ticks() | 1, 300);
|
| 474 |
+
input_length = 0;
|
| 475 |
+
shift = 1;
|
| 476 |
+
}
|
| 477 |
+
|
| 478 |
+
static void move(int keys) {
|
| 479 |
+
if (cursor_y == 4) { // the wide keys: one step per key
|
| 480 |
+
int k = wide_key(cursor_x);
|
| 481 |
+
if (keys & KEY_LEFT) cursor_x = wide_starts[(k + 3) % 4];
|
| 482 |
+
if (keys & KEY_RIGHT) cursor_x = wide_starts[(k + 1) % 4];
|
| 483 |
+
} else {
|
| 484 |
+
if (keys & KEY_LEFT) cursor_x = (cursor_x + 11) % 12;
|
| 485 |
+
if (keys & KEY_RIGHT) cursor_x = (cursor_x + 1) % 12;
|
| 486 |
+
}
|
| 487 |
+
if (keys & KEY_UP) cursor_y = (cursor_y + 4) % 5;
|
| 488 |
+
if (keys & KEY_DOWN) cursor_y = (cursor_y + 1) % 5;
|
| 489 |
+
}
|
| 490 |
+
|
| 491 |
+
static void press(void) {
|
| 492 |
+
if (cursor_y < 4) {
|
| 493 |
+
type((shift ? keys_upper : keys_lower)[cursor_y][cursor_x]);
|
| 494 |
+
return;
|
| 495 |
+
}
|
| 496 |
+
switch (wide_key(cursor_x)) {
|
| 497 |
+
case WIDE_SHIFT: shift = (shift + 1) % 3; break;
|
| 498 |
+
case WIDE_SPACE: type(' '); break;
|
| 499 |
+
case WIDE_DELETE: delete_one(); break;
|
| 500 |
+
case WIDE_SEND: send(); break;
|
| 501 |
+
}
|
| 502 |
+
}
|
| 503 |
+
|
| 504 |
+
int main(void) {
|
| 505 |
+
REG_WAITCNT = 0x4317; // faster ROM wait states (3/1) and the prefetch buffer
|
| 506 |
+
REG_DISPCNT = 0x0403; // mode 3: a 240 x 160 bitmap, 15-bit color
|
| 507 |
+
REG_TM2CNT_H = 0;
|
| 508 |
+
REG_TM3CNT_H = 0;
|
| 509 |
+
REG_TM2CNT_L = 0;
|
| 510 |
+
REG_TM3CNT_L = 0;
|
| 511 |
+
REG_TM3CNT_H = 0x0084; // counts timer 2's overflows
|
| 512 |
+
REG_TM2CNT_H = 0x0083; // one tick per 1024 cycles
|
| 513 |
+
*(volatile uint16_t*)0x04000106 = 0x0084; // timer 1 counts timer 0's overflows
|
| 514 |
+
*(volatile uint16_t*)0x04000102 = 0x0080; // timer 0: every cycle
|
| 515 |
+
profile_clock = cycles;
|
| 516 |
+
|
| 517 |
+
line_count = 0;
|
| 518 |
+
dirty_from = 0;
|
| 519 |
+
new_line(TITLE);
|
| 520 |
+
log_text("ChatGBA: 4M parameters 512 context", TITLE);
|
| 521 |
+
int status_code = engine_init(model_blob);
|
| 522 |
+
mailbox.status = status_code;
|
| 523 |
+
if (status_code) {
|
| 524 |
+
log_text("the weights do not match the engine", TITLE);
|
| 525 |
+
typing_screen();
|
| 526 |
+
for (;;) {}
|
| 527 |
+
}
|
| 528 |
+
log_text("A type B delete START send", DIM);
|
| 529 |
+
log_text("R shift L sample question", DIM);
|
| 530 |
+
log_text("SELECT new conversation", DIM);
|
| 531 |
+
start(MODEL);
|
| 532 |
+
input_length = 0;
|
| 533 |
+
shift = 1;
|
| 534 |
+
cursor_x = cursor_y = 0;
|
| 535 |
+
typing_screen();
|
| 536 |
+
mailbox.magic = MAILBOX_MAGIC;
|
| 537 |
+
|
| 538 |
+
unsigned next_question = 0;
|
| 539 |
+
uint16_t last_keys = 0;
|
| 540 |
+
int held = 0;
|
| 541 |
+
for (;;) {
|
| 542 |
+
wait_for_vblank();
|
| 543 |
+
if (mailbox.request != mailbox.done) {
|
| 544 |
+
if (mailbox.command == 2) new_conversation();
|
| 545 |
+
else ask(mailbox.message, (int)mailbox.length, mailbox.temperature, mailbox.seed, (int)mailbox.max_reply);
|
| 546 |
+
mailbox.done = mailbox.request;
|
| 547 |
+
typing_screen();
|
| 548 |
+
continue;
|
| 549 |
+
}
|
| 550 |
+
|
| 551 |
+
uint16_t keys = (uint16_t)~REG_KEYINPUT & 0x3FF; // pressed keys read as 0
|
| 552 |
+
uint16_t pressed = keys & ~last_keys;
|
| 553 |
+
// Held keys repeat after 18 frames, then every 4.
|
| 554 |
+
if ((keys & KEY_REPEATS) && (keys & KEY_REPEATS) == (last_keys & KEY_REPEATS)) {
|
| 555 |
+
if (++held >= 18 && (held - 18) % 4 == 0) pressed |= keys & KEY_REPEATS;
|
| 556 |
+
} else {
|
| 557 |
+
held = 0;
|
| 558 |
+
}
|
| 559 |
+
last_keys = keys;
|
| 560 |
+
if (!pressed) continue;
|
| 561 |
+
|
| 562 |
+
if (pressed & (KEY_LEFT | KEY_RIGHT | KEY_UP | KEY_DOWN)) move(pressed);
|
| 563 |
+
if (pressed & KEY_A) press();
|
| 564 |
+
if (pressed & KEY_B) delete_one();
|
| 565 |
+
if (pressed & KEY_R) shift = (shift + 1) % 3;
|
| 566 |
+
if (pressed & KEY_L) {
|
| 567 |
+
const char* q = questions[next_question++ % (sizeof questions / sizeof questions[0])];
|
| 568 |
+
for (input_length = 0; q[input_length]; ++input_length) input[input_length] = q[input_length];
|
| 569 |
+
shift = 0;
|
| 570 |
+
}
|
| 571 |
+
if (pressed & KEY_SELECT) new_conversation();
|
| 572 |
+
if (pressed & KEY_START) send();
|
| 573 |
+
if (log_rows != TYPING_ROWS) { // a reply just finished
|
| 574 |
+
typing_screen();
|
| 575 |
+
} else {
|
| 576 |
+
draw_log();
|
| 577 |
+
draw_input();
|
| 578 |
+
draw_keyboard();
|
| 579 |
+
}
|
| 580 |
+
}
|
| 581 |
+
}
|
source/model_config.h
ADDED
|
@@ -0,0 +1,17 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
// Generated by export.py from final.gab; do not edit.
|
| 2 |
+
#pragma once
|
| 3 |
+
#define MODEL_VOCAB 256
|
| 4 |
+
#define MODEL_D 192
|
| 5 |
+
#define MODEL_HEADS 6
|
| 6 |
+
#define MODEL_KV_HEADS 2
|
| 7 |
+
#define MODEL_HEAD_DIM 32
|
| 8 |
+
#define MODEL_BLOCKS 10
|
| 9 |
+
#define MODEL_HIDDEN 512
|
| 10 |
+
#define MODEL_KERNEL 4
|
| 11 |
+
#define MODEL_CONTEXT 512
|
| 12 |
+
#define MODEL_WINDOWS {64, 64, 64, 64, 0, 64, 64, 64, 64, 0}
|
| 13 |
+
#define MODEL_KV_SLOTS 1536 // cached positions, summed over the blocks
|
| 14 |
+
#define MODEL_BLOB_BYTES 4207968
|
| 15 |
+
#define CHAT_USER "<|u|>"
|
| 16 |
+
#define CHAT_ASSISTANT "<|a|>"
|
| 17 |
+
#define CHAT_END "<|e|>"
|
special_tokens_map.json
ADDED
|
@@ -0,0 +1 @@
|
|
|
|
|
|
|
| 1 |
+
{}
|
tokenization_gabgpt.py
ADDED
|
@@ -0,0 +1,41 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""The ChatGBA tokenizer: every UTF-8 byte is one token, ids 0 to 255.
|
| 2 |
+
|
| 3 |
+
There are no special tokens. The chat markers (<|u|>, <|a|>, <|e|>) are plain
|
| 4 |
+
text, spelled out byte by byte, exactly as the model was trained. A reply ends
|
| 5 |
+
when the model writes <|e|>: pass stop_strings="<|e|>" (and this tokenizer)
|
| 6 |
+
to generate.
|
| 7 |
+
"""
|
| 8 |
+
from transformers import PreTrainedTokenizer
|
| 9 |
+
|
| 10 |
+
|
| 11 |
+
class GabByteTokenizer(PreTrainedTokenizer):
|
| 12 |
+
model_input_names = ["input_ids"]
|
| 13 |
+
|
| 14 |
+
def __init__(self, **kwargs):
|
| 15 |
+
kwargs.setdefault("bos_token", None)
|
| 16 |
+
kwargs.setdefault("eos_token", None)
|
| 17 |
+
kwargs.setdefault("pad_token", None)
|
| 18 |
+
kwargs.setdefault("unk_token", None)
|
| 19 |
+
super().__init__(**kwargs)
|
| 20 |
+
|
| 21 |
+
@property
|
| 22 |
+
def vocab_size(self):
|
| 23 |
+
return 256
|
| 24 |
+
|
| 25 |
+
def get_vocab(self):
|
| 26 |
+
return {self._convert_id_to_token(i): i for i in range(256)}
|
| 27 |
+
|
| 28 |
+
def _tokenize(self, text, **kwargs):
|
| 29 |
+
return [self._convert_id_to_token(b) for b in text.encode("utf-8")]
|
| 30 |
+
|
| 31 |
+
def _convert_token_to_id(self, token):
|
| 32 |
+
return int(token[3:-1], 16) # "<0x41>" -> 65
|
| 33 |
+
|
| 34 |
+
def _convert_id_to_token(self, index):
|
| 35 |
+
return f"<0x{index:02X}>"
|
| 36 |
+
|
| 37 |
+
def convert_tokens_to_string(self, tokens):
|
| 38 |
+
return bytes(self._convert_token_to_id(t) for t in tokens).decode("utf-8", errors="replace")
|
| 39 |
+
|
| 40 |
+
def save_vocabulary(self, save_directory, filename_prefix=None):
|
| 41 |
+
return () # nothing to save: the vocabulary is the 256 byte values
|
tokenizer_config.json
ADDED
|
@@ -0,0 +1,12 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{
|
| 2 |
+
"tokenizer_class": "GabByteTokenizer",
|
| 3 |
+
"auto_map": {
|
| 4 |
+
"AutoTokenizer": [
|
| 5 |
+
"tokenization_gabgpt.GabByteTokenizer",
|
| 6 |
+
null
|
| 7 |
+
]
|
| 8 |
+
},
|
| 9 |
+
"chat_template": "{% for message in messages %}{% if message['role'] == 'user' %}<|u|>{{ message['content'] }}<|e|>{% elif message['role'] == 'assistant' %}<|a|>{{ message['content'] }}<|e|>{% endif %}{% endfor %}{% if add_generation_prompt %}<|a|>{% endif %}",
|
| 10 |
+
"model_max_length": 512,
|
| 11 |
+
"clean_up_tokenization_spaces": false
|
| 12 |
+
}
|