TryChatGBA / inference.html
gszauer's picture
The ChatGBA web page: emulator and in-browser model
9e71366 verified
Raw History Blame Contribute Delete
20 kB
<!doctype html>
<html lang="en">
<head>
<meta charset="utf-8">
<meta name="viewport" content="width=device-width, initial-scale=1">
<title>ChatGBA Inference</title>
<meta name="description" content="Chat with ChatGBA's float32 weights in the browser: a JavaScript reference engine for the 4-million-parameter Game Boy Advance model.">
<link rel="preconnect" href="https://fonts.googleapis.com">
<link rel="preconnect" href="https://fonts.gstatic.com" crossorigin>
<link href="https://fonts.googleapis.com/css2?family=Press+Start+2P&family=Inter:wght@400;600;800&family=JetBrains+Mono:wght@400;600&display=swap" rel="stylesheet">
<style>
:root {
--page: #14121f;
--page-2: #1c1930;
--ink: #ecebf5;
--ink-dim: #a9a6c4;
--accent: #8f7bff;
--accent-2: #ffcf4d;
--line: #2f2b4a;
--code: #0f0d19;
}
* { box-sizing: border-box; font-variant-ligatures: none; font-feature-settings: "liga" 0, "calt" 0; }
html, body { height: 100%; }
body {
margin: 0; display: flex; flex-direction: column;
background: radial-gradient(1200px 600px at 50% -200px, #2a2350 0%, var(--page) 60%) fixed, var(--page);
color: var(--ink); font: 16px/1.6 Inter, system-ui, sans-serif;
}
.wrap { width: 100%; max-width: 820px; margin: 0 auto; padding: 0 16px; }
a { color: var(--accent-2); }
.settings { display: flex; flex-wrap: wrap; gap: 8px 18px; align-items: center; padding: 12px 0 0; border-top: 1px solid var(--line); font-size: 14px; color: var(--ink-dim); }
.settings label { display: flex; align-items: center; gap: 6px; }
.settings input[type=number] { width: 72px; background: var(--code); color: var(--ink); border: 1px solid var(--line); border-radius: 6px; padding: 4px 6px; font: 14px "JetBrains Mono", monospace; }
.log { flex: 1; overflow-y: auto; padding: 16px 0; }
.turn { margin: 0 0 14px; display: flex; gap: 10px; }
.who { flex: none; width: 76px; font: 12px/1.9 "JetBrains Mono", monospace; color: var(--ink-dim); text-align: right; }
.turn.user .who { color: var(--accent); }
.turn.model .who { color: var(--accent-2); }
.text { white-space: pre-wrap; overflow-wrap: anywhere; min-width: 0; }
.turn.model .text.busy::after { content: "▍"; color: var(--accent-2); animation: blink 1s steps(2) infinite; }
@keyframes blink { 50% { opacity: 0; } }
.meta { font: 12px "JetBrains Mono", monospace; color: var(--ink-dim); margin-top: 2px; }
.empty { color: var(--ink-dim); text-align: center; padding: 40px 0; }
.empty.error { color: #ff8a8a; }
form { display: flex; gap: 8px; padding: 10px 0 20px; }
textarea {
flex: 1; resize: none; min-height: 44px; max-height: 160px; background: var(--page-2); color: var(--ink);
border: 1px solid var(--line); border-radius: 10px; padding: 10px 12px; font: 16px/1.4 Inter, system-ui, sans-serif;
}
textarea:focus { outline: none; border-color: var(--accent); }
button {
font: 10px "Press Start 2P", monospace; background: var(--accent); color: #fff; border: 0; border-radius: 8px;
padding: 0 14px; cursor: pointer; min-height: 44px;
}
button:disabled { opacity: .45; cursor: default; }
@media (max-width: 560px) { .who { width: 44px; } }
</style>
</head>
<body>
<div class="log wrap" id="log"><div class="empty" id="empty">Loading the weights (15 MB)…</div></div>
<div class="wrap">
<div class="settings">
<label>Temperature <input type="number" id="temperature" value="0.7" min="0" max="2" step="0.05"></label>
<label>Top-k <input type="number" id="top-k" value="40" min="0" max="256" step="1"></label>
<label>Max reply <input type="number" id="max-reply" value="300" min="1" max="2000" step="10"></label>
</div>
<form id="form">
<textarea id="input" rows="1" placeholder="Say something…" disabled></textarea>
<button id="send" type="submit" disabled>SEND</button>
</form>
</div>
<script>
// --- The engine -------------------------------------------------------------------
// A float32 reference engine for ChatGBA, inference only. The math
// is GabGPTInfer.js's; the cache is the GBA engine's: one byte at a time, each
// block keeping a ring of its last window positions (or the last 512 for a
// block that sees the whole context), so the context slides and positions
// never run out. RoPE uses the true position; attention only sees differences.
// --- engine start
const CONTEXT = 512;
const EPS = 1e-5;
function decodeWeightFile(buffer) {
const bytes = new Uint8Array(buffer);
if (buffer.byteLength < 12 || String.fromCharCode(...bytes.subarray(0, 4)) !== "GAB_") {
throw new Error("not a GabGPT weight file: missing the GAB_ magic");
}
const view = new DataView(buffer);
if (view.getUint32(4, true) !== 1) throw new Error("unsupported weight file version " + view.getUint32(4, true));
const headerLength = view.getUint32(8, true);
const header = JSON.parse(new TextDecoder().decode(bytes.subarray(12, 12 + headerLength)));
const count = (buffer.byteLength - 12 - headerLength) / 4;
if (count !== header.parameterCount) throw new Error(`the file holds ${count} weights, but its header says ${header.parameterCount}`);
return { header, weights: new Float32Array(buffer, 12 + headerLength, count) };
}
class ChatGBAEngine {
constructor(buffer) {
const { header, weights } = decodeWeightFile(buffer);
if (header.architecture !== "chatgba") throw new Error(`this engine runs chatgba, not ${header.architecture}`);
this.header = header;
const D = this.D = header.featureDim, H = this.H = header.headDim, F = this.F = header.mlpHiddenDim;
this.V = header.vocabSize;
this.kvHeads = header.numKVHeads;
this.perGroup = header.numHeads / header.numKVHeads;
this.kernel = header.convKernel;
// Views into the file, in its order (model.json's "weights.order").
let offset = 0;
const take = n => { const t = weights.subarray(offset, offset + n); offset += n; return t; };
this.embedding = take(this.V * D); // [vocab, D], also the output projection
this.blocks = [];
for (let b = 0; b < header.numBlocks; ++b) {
const block = { attentionNorm: take(D), canonA: take(this.kernel * D), groups: [] };
for (let g = 0; g < this.kvHeads; ++g) {
const group = { K: take(D * H), V: take(D * H), keyNorm: take(H), queries: [] };
for (let i = 0; i < this.perGroup; ++i) group.queries.push({ Q: take(D * H), O: take(H * D), queryNorm: take(H) });
block.groups.push(group);
}
block.mlpNorm = take(D); block.canonC = take(this.kernel * D);
block.gate = take(D * F); block.up = take(D * F); block.down = take(F * D);
block.window = header.attentionWindows[b];
block.capacity = block.window || CONTEXT; // positions the ring keeps
this.blocks.push(block);
}
this.finalNorm = take(D);
if (offset !== weights.length) throw new Error("the weight file's layout does not match chatgba");
// RoPE frequencies for adjacent pairs (0,1), (2,3), ...: base^(-2i/H).
this.freqs = new Float64Array(H / 2);
for (let i = 0; i < H / 2; ++i) this.freqs[i] = 1 / Math.pow(header.ropeBase, (2 * i) / H);
// Working vectors for one position.
this.x = new Float32Array(D); this.a = new Float32Array(D); this.mixed = new Float32Array(D);
this.q = new Float32Array(H); this.k = new Float32Array(H); this.v = new Float32Array(H); this.head = new Float32Array(H);
this.gateOut = new Float32Array(F); this.upOut = new Float32Array(F);
this.scores = new Float64Array(CONTEXT);
this.logits = new Float32Array(this.V);
this.reset();
}
// A fresh conversation: empty caches, position 0.
reset() {
this.position = 0;
for (const block of this.blocks) {
block.keys = block.groups.map(() => new Float32Array(block.capacity * this.H));
block.values = block.groups.map(() => new Float32Array(block.capacity * this.H));
block.historyA = new Float32Array((this.kernel - 1) * this.D); // row j-1: the input j positions back
block.historyC = new Float32Array((this.kernel - 1) * this.D);
}
}
// out = x · W, W row-major [x.length, out.length].
static matvec(x, W, out) {
const n = out.length;
const sum = new Float64Array(n);
for (let i = 0; i < x.length; ++i) {
const xi = x[i];
if (xi === 0) continue;
const base = i * n;
for (let j = 0; j < n; ++j) sum[j] += xi * W[base + j];
}
out.set(sum);
}
static rmsNorm(x, gamma, out) {
let sumSq = 0;
for (let i = 0; i < x.length; ++i) sumSq += x[i] * x[i];
const inverse = 1 / Math.sqrt(sumSq / x.length + EPS);
for (let i = 0; i < x.length; ++i) out[i] = x[i] * inverse * gamma[i];
}
// A canon layer in place: a[t] += sum over j of w[j] * input[t - j], then
// the input joins the history.
canon(a, weight, history) {
const D = this.D, taps = this.kernel - 1;
const input = Float32Array.from(a);
for (let c = 0; c < D; ++c) {
let sum = weight[c] * input[c];
for (let j = 1; j <= taps; ++j) sum += weight[j * D + c] * history[(j - 1) * D + c];
a[c] = input[c] + sum;
}
history.copyWithin(D, 0, (taps - 1) * D);
history.set(input, 0);
}
rope(vector, position) {
for (let i = 0; i < this.H / 2; ++i) {
const angle = position * this.freqs[i];
const cos = Math.cos(angle), sin = Math.sin(angle);
const p = vector[2 * i], r = vector[2 * i + 1];
vector[2 * i] = p * cos - r * sin;
vector[2 * i + 1] = p * sin + r * cos;
}
}
// Run one byte through the model; returns the next byte's logits.
step(byte) {
const { D, H } = this;
const x = this.x, a = this.a;
const pos = this.position;
x.set(this.embedding.subarray(byte * D, (byte + 1) * D));
const scale = 1 / Math.sqrt(H);
for (const block of this.blocks) {
// Attention half.
ChatGBAEngine.rmsNorm(x, block.attentionNorm, a);
this.canon(a, block.canonA, block.historyA);
const slot = pos % block.capacity;
const seen = Math.min(pos + 1, block.capacity); // cached positions this query sees, the newest last
this.mixed.fill(0);
for (let g = 0; g < block.groups.length; ++g) {
const group = block.groups[g], keys = block.keys[g], values = block.values[g];
ChatGBAEngine.matvec(a, group.K, this.k);
ChatGBAEngine.rmsNorm(this.k, group.keyNorm, this.k);
this.rope(this.k, pos);
ChatGBAEngine.matvec(a, group.V, this.v);
keys.set(this.k, slot * H);
values.set(this.v, slot * H);
for (const query of group.queries) {
ChatGBAEngine.matvec(a, query.Q, this.q);
ChatGBAEngine.rmsNorm(this.q, query.queryNorm, this.q);
this.rope(this.q, pos);
let best = -Infinity;
for (let n = 0; n < seen; ++n) {
const s = ((pos - n) % block.capacity) * H; // n positions back
let dot = 0;
for (let i = 0; i < H; ++i) dot += this.q[i] * keys[s + i];
this.scores[n] = dot * scale;
if (this.scores[n] > best) best = this.scores[n];
}
let total = 0;
for (let n = 0; n < seen; ++n) { this.scores[n] = Math.exp(this.scores[n] - best); total += this.scores[n]; }
this.head.fill(0);
for (let n = 0; n < seen; ++n) {
const s = ((pos - n) % block.capacity) * H, w = this.scores[n] / total;
for (let i = 0; i < H; ++i) this.head[i] += w * values[s + i];
}
// mixed += head · O
for (let i = 0; i < H; ++i) {
const hi = this.head[i], base = i * D;
for (let c = 0; c < D; ++c) this.mixed[c] += hi * query.O[base + c];
}
}
}
for (let c = 0; c < D; ++c) x[c] += this.mixed[c]; // residual add
// MLP half.
ChatGBAEngine.rmsNorm(x, block.mlpNorm, a);
this.canon(a, block.canonC, block.historyC);
ChatGBAEngine.matvec(a, block.gate, this.gateOut);
ChatGBAEngine.matvec(a, block.up, this.upOut);
for (let j = 0; j < this.F; ++j) {
const g = this.gateOut[j];
this.gateOut[j] = g / (1 + Math.exp(-g)) * this.upOut[j]; // silu(gate) × up
}
ChatGBAEngine.matvec(this.gateOut, block.down, this.mixed);
for (let c = 0; c < D; ++c) x[c] += this.mixed[c]; // residual add
}
// Final norm, then the tied unembedding: logits = x · Embeddingᵀ.
ChatGBAEngine.rmsNorm(x, this.finalNorm, a);
for (let t = 0; t < this.V; ++t) {
let dot = 0;
const base = t * D;
for (let c = 0; c < D; ++c) dot += a[c] * this.embedding[base + c];
this.logits[t] = dot;
}
this.position = pos + 1;
return this.logits;
}
}
// The cartridge's sampling: temperature, then keep the top k (0: all). Temperature 0 is greedy.
function sampleByte(logits, temperature, topK, random = Math.random) {
let best = 0;
for (let i = 1; i < logits.length; ++i) if (logits[i] > logits[best]) best = i;
if (!(temperature > 0)) return best;
const weights = Array.from(logits, (l, i) => ({ i, w: Math.exp((l - logits[best]) / temperature) }));
weights.sort((p, r) => r.w - p.w);
const kept = topK > 0 ? weights.slice(0, topK) : weights;
let total = 0;
for (const k of kept) total += k.w;
let target = random() * total;
for (const k of kept) { if (target < k.w) return k.i; target -= k.w; }
return best;
}
// A conversation, as the cartridge's chat.c runs it: one endless stream of
// <|u|>message<|e|><|a|>reply<|e|> turns through the sliding cache. The last
// byte of a reply's closing <|e|> is sampled but not run until the next
// message, and a reply cut off partway into <|e|> keeps that start.
class Chat {
constructor(engine) {
const markers = engine.header.chat || { user: "<|u|>", assistant: "<|a|>", end: "<|e|>" };
const encode = s => new TextEncoder().encode(s);
this.engine = engine;
this.USER = encode(markers.user); this.ASSISTANT = encode(markers.assistant); this.END = encode(markers.end);
this.reset();
}
reset() {
this.engine.reset();
this.endSeen = this.END.length; // bytes of the last reply's <|e|> the engine has run
this.logits = null;
}
// The longest k such that the reply ends with the first k bytes of <|e|>.
partialEnd(reply) {
for (let k = this.END.length - 1; k > 0; --k) {
if (k <= reply.length && this.END.subarray(0, k).every((b, i) => reply[reply.length - k + i] === b)) return k;
}
return 0;
}
// Yields { reading, total } while it reads the turn, then { bytes } as reply
// bytes become final (not a possible start of <|e|>).
async *reply(message, { temperature, topK, maxReply, shouldStop, pause }) {
const prompt = [...this.END.subarray(this.endSeen), ...this.USER, ...message, ...this.END, ...this.ASSISTANT];
for (let i = 0; i < prompt.length; ++i) {
this.logits = this.engine.step(prompt[i]);
yield { reading: i + 1, total: prompt.length };
await pause();
}
const reply = [];
let shown = 0, ended = false;
while (reply.length < maxReply) {
if (reply.length > 0 && shouldStop()) break;
const byte = sampleByte(this.logits, temperature, topK);
reply.push(byte);
const n = reply.length, E = this.END.length;
if (n >= E && this.END.every((b, i) => reply[n - E + i] === b)) {
reply.length = n - E;
ended = true;
break; // its last byte waits for the next message
}
const hold = this.partialEnd(reply);
if (shown < reply.length - hold) {
yield { bytes: reply.slice(shown, reply.length - hold) };
shown = reply.length - hold;
}
this.logits = this.engine.step(byte);
await pause();
}
if (ended) {
this.endSeen = this.END.length - 1;
} else {
this.endSeen = this.partialEnd(reply);
reply.length -= this.endSeen;
}
if (shown < reply.length) yield { bytes: reply.slice(shown) };
}
}
// --- engine end
// --- The page ---------------------------------------------------------------------
const $ = id => document.getElementById(id);
const notice = (text, error = false) => { const el = $("empty"); if (el) { el.textContent = text; el.classList.toggle("error", error); } };
let chat = null, busy = false, stopRequested = false;
// Let the page breathe between bytes without the 4 ms setTimeout clamp.
const channel = new MessageChannel();
const waiting = [];
channel.port1.onmessage = () => waiting.shift()();
let lastBreath = performance.now();
function pause() {
if (performance.now() - lastBreath < 30) return Promise.resolve();
return new Promise(resolve => { waiting.push(() => { lastBreath = performance.now(); resolve(); }); channel.port2.postMessage(0); });
}
function ready(buffer) {
try {
chat = new Chat(new ChatGBAEngine(buffer));
} catch (e) {
notice(e.message, true);
return;
}
$("empty")?.remove(); // loaded: an empty chat
$("input").disabled = false;
$("send").disabled = false;
$("input").focus();
}
// The weights, from next to this page.
fetch("chatgba-float32.gab")
.then(r => { if (!r.ok) throw new Error(r.status + " " + r.statusText); return r.arrayBuffer(); })
.then(ready)
.catch(e => notice("Could not load the weights (chatgba-float32.gab): " + e.message, true));
function addTurn(kind, label, text) {
$("empty")?.remove();
const turn = document.createElement("div");
turn.className = "turn " + kind;
turn.innerHTML = `<div class="who"></div><div><div class="text"></div><div class="meta"></div></div>`;
turn.querySelector(".who").textContent = label;
turn.querySelector(".text").textContent = text;
$("log").appendChild(turn);
$("log").scrollTop = $("log").scrollHeight;
return turn;
}
function number(id, fallback) {
const v = Number($(id).value);
return Number.isFinite(v) ? v : fallback;
}
$("form").addEventListener("submit", async e => {
e.preventDefault();
if (busy) { stopRequested = true; return; }
const text = $("input").value.trim();
if (!text || !chat) return;
$("input").value = "";
addTurn("user", "you", text);
const turn = addTurn("model", "chatgba", "");
const out = turn.querySelector(".text"), meta = turn.querySelector(".meta");
out.classList.add("busy");
busy = true; stopRequested = false;
$("send").textContent = "STOP";
$("input").disabled = true;
const decoder = new TextDecoder();
let replyBytes = 0;
const started = performance.now();
let replyStarted = 0;
try {
for await (const event of chat.reply(new TextEncoder().encode(text), {
temperature: number("temperature", 0.7), topK: Math.round(number("top-k", 40)), maxReply: Math.round(number("max-reply", 300)),
shouldStop: () => stopRequested, pause,
})) {
if (event.reading) {
meta.textContent = `reading ${event.reading}/${event.total} bytes`;
} else {
if (!replyStarted) replyStarted = performance.now();
out.textContent += decoder.decode(new Uint8Array(event.bytes), { stream: true });
replyBytes += event.bytes.length;
meta.textContent = `${replyBytes} bytes`;
$("log").scrollTop = $("log").scrollHeight;
}
}
out.textContent += decoder.decode();
const seconds = (performance.now() - (replyStarted || started)) / 1000;
meta.textContent = `${replyBytes} bytes · ${replyBytes && seconds > 0 ? Math.round(replyBytes / seconds) : 0} bytes/s · position ${chat.engine.position.toLocaleString("en-US")}${stopRequested ? " · stopped" : ""}`;
} catch (err) {
meta.textContent = "error: " + err.message;
}
out.classList.remove("busy");
busy = false;
$("send").textContent = "SEND";
$("input").disabled = false;
$("input").focus();
});
// Enter sends, Shift+Enter starts a new line.
$("input").addEventListener("keydown", e => {
if (e.key === "Enter" && !e.shiftKey) { e.preventDefault(); $("form").requestSubmit(); }
});
$("input").addEventListener("input", () => { const t = $("input"); t.style.height = "auto"; t.style.height = Math.min(t.scrollHeight, 160) + "px"; });
</script>
</body>
</html>