Spaces:
Running
Running
Download inference.html from gszauer/TryChatGBA: direct link, hf CLI and curl.
- Browser
- Download file 20 kB
-
https://huggingface.co/spaces/gszauer/TryChatGBA/resolve/main/inference.html
- Command line
-
hf download hf://spaces/gszauer/TryChatGBA/inference.html
-
curl -L -o inference.html https://huggingface.co/spaces/gszauer/TryChatGBA/resolve/main/inference.html
20 kB
| <html lang="en"> | |
| <head> | |
| <meta charset="utf-8"> | |
| <meta name="viewport" content="width=device-width, initial-scale=1"> | |
| <title>ChatGBA Inference</title> | |
| <meta name="description" content="Chat with ChatGBA's float32 weights in the browser: a JavaScript reference engine for the 4-million-parameter Game Boy Advance model."> | |
| <link rel="preconnect" href="https://fonts.googleapis.com"> | |
| <link rel="preconnect" href="https://fonts.gstatic.com" crossorigin> | |
| <link href="https://fonts.googleapis.com/css2?family=Press+Start+2P&family=Inter:wght@400;600;800&family=JetBrains+Mono:wght@400;600&display=swap" rel="stylesheet"> | |
| <style> | |
| :root { | |
| --page: #14121f; | |
| --page-2: #1c1930; | |
| --ink: #ecebf5; | |
| --ink-dim: #a9a6c4; | |
| --accent: #8f7bff; | |
| --accent-2: #ffcf4d; | |
| --line: #2f2b4a; | |
| --code: #0f0d19; | |
| } | |
| * { box-sizing: border-box; font-variant-ligatures: none; font-feature-settings: "liga" 0, "calt" 0; } | |
| html, body { height: 100%; } | |
| body { | |
| margin: 0; display: flex; flex-direction: column; | |
| background: radial-gradient(1200px 600px at 50% -200px, #2a2350 0%, var(--page) 60%) fixed, var(--page); | |
| color: var(--ink); font: 16px/1.6 Inter, system-ui, sans-serif; | |
| } | |
| .wrap { width: 100%; max-width: 820px; margin: 0 auto; padding: 0 16px; } | |
| a { color: var(--accent-2); } | |
| .settings { display: flex; flex-wrap: wrap; gap: 8px 18px; align-items: center; padding: 12px 0 0; border-top: 1px solid var(--line); font-size: 14px; color: var(--ink-dim); } | |
| .settings label { display: flex; align-items: center; gap: 6px; } | |
| .settings input[type=number] { width: 72px; background: var(--code); color: var(--ink); border: 1px solid var(--line); border-radius: 6px; padding: 4px 6px; font: 14px "JetBrains Mono", monospace; } | |
| .log { flex: 1; overflow-y: auto; padding: 16px 0; } | |
| .turn { margin: 0 0 14px; display: flex; gap: 10px; } | |
| .who { flex: none; width: 76px; font: 12px/1.9 "JetBrains Mono", monospace; color: var(--ink-dim); text-align: right; } | |
| .turn.user .who { color: var(--accent); } | |
| .turn.model .who { color: var(--accent-2); } | |
| .text { white-space: pre-wrap; overflow-wrap: anywhere; min-width: 0; } | |
| .turn.model .text.busy::after { content: "▍"; color: var(--accent-2); animation: blink 1s steps(2) infinite; } | |
| @keyframes blink { 50% { opacity: 0; } } | |
| .meta { font: 12px "JetBrains Mono", monospace; color: var(--ink-dim); margin-top: 2px; } | |
| .empty { color: var(--ink-dim); text-align: center; padding: 40px 0; } | |
| .empty.error { color: #ff8a8a; } | |
| form { display: flex; gap: 8px; padding: 10px 0 20px; } | |
| textarea { | |
| flex: 1; resize: none; min-height: 44px; max-height: 160px; background: var(--page-2); color: var(--ink); | |
| border: 1px solid var(--line); border-radius: 10px; padding: 10px 12px; font: 16px/1.4 Inter, system-ui, sans-serif; | |
| } | |
| textarea:focus { outline: none; border-color: var(--accent); } | |
| button { | |
| font: 10px "Press Start 2P", monospace; background: var(--accent); color: #fff; border: 0; border-radius: 8px; | |
| padding: 0 14px; cursor: pointer; min-height: 44px; | |
| } | |
| button:disabled { opacity: .45; cursor: default; } | |
| @media (max-width: 560px) { .who { width: 44px; } } | |
| </style> | |
| </head> | |
| <body> | |
| <div class="log wrap" id="log"><div class="empty" id="empty">Loading the weights (15 MB)…</div></div> | |
| <div class="wrap"> | |
| <div class="settings"> | |
| <label>Temperature <input type="number" id="temperature" value="0.7" min="0" max="2" step="0.05"></label> | |
| <label>Top-k <input type="number" id="top-k" value="40" min="0" max="256" step="1"></label> | |
| <label>Max reply <input type="number" id="max-reply" value="300" min="1" max="2000" step="10"></label> | |
| </div> | |
| <form id="form"> | |
| <textarea id="input" rows="1" placeholder="Say something…" disabled></textarea> | |
| <button id="send" type="submit" disabled>SEND</button> | |
| </form> | |
| </div> | |
| <script> | |
| // --- The engine ------------------------------------------------------------------- | |
| // A float32 reference engine for ChatGBA, inference only. The math | |
| // is GabGPTInfer.js's; the cache is the GBA engine's: one byte at a time, each | |
| // block keeping a ring of its last window positions (or the last 512 for a | |
| // block that sees the whole context), so the context slides and positions | |
| // never run out. RoPE uses the true position; attention only sees differences. | |
| // --- engine start | |
| const CONTEXT = 512; | |
| const EPS = 1e-5; | |
| function decodeWeightFile(buffer) { | |
| const bytes = new Uint8Array(buffer); | |
| if (buffer.byteLength < 12 || String.fromCharCode(...bytes.subarray(0, 4)) !== "GAB_") { | |
| throw new Error("not a GabGPT weight file: missing the GAB_ magic"); | |
| } | |
| const view = new DataView(buffer); | |
| if (view.getUint32(4, true) !== 1) throw new Error("unsupported weight file version " + view.getUint32(4, true)); | |
| const headerLength = view.getUint32(8, true); | |
| const header = JSON.parse(new TextDecoder().decode(bytes.subarray(12, 12 + headerLength))); | |
| const count = (buffer.byteLength - 12 - headerLength) / 4; | |
| if (count !== header.parameterCount) throw new Error(`the file holds ${count} weights, but its header says ${header.parameterCount}`); | |
| return { header, weights: new Float32Array(buffer, 12 + headerLength, count) }; | |
| } | |
| class ChatGBAEngine { | |
| constructor(buffer) { | |
| const { header, weights } = decodeWeightFile(buffer); | |
| if (header.architecture !== "chatgba") throw new Error(`this engine runs chatgba, not ${header.architecture}`); | |
| this.header = header; | |
| const D = this.D = header.featureDim, H = this.H = header.headDim, F = this.F = header.mlpHiddenDim; | |
| this.V = header.vocabSize; | |
| this.kvHeads = header.numKVHeads; | |
| this.perGroup = header.numHeads / header.numKVHeads; | |
| this.kernel = header.convKernel; | |
| // Views into the file, in its order (model.json's "weights.order"). | |
| let offset = 0; | |
| const take = n => { const t = weights.subarray(offset, offset + n); offset += n; return t; }; | |
| this.embedding = take(this.V * D); // [vocab, D], also the output projection | |
| this.blocks = []; | |
| for (let b = 0; b < header.numBlocks; ++b) { | |
| const block = { attentionNorm: take(D), canonA: take(this.kernel * D), groups: [] }; | |
| for (let g = 0; g < this.kvHeads; ++g) { | |
| const group = { K: take(D * H), V: take(D * H), keyNorm: take(H), queries: [] }; | |
| for (let i = 0; i < this.perGroup; ++i) group.queries.push({ Q: take(D * H), O: take(H * D), queryNorm: take(H) }); | |
| block.groups.push(group); | |
| } | |
| block.mlpNorm = take(D); block.canonC = take(this.kernel * D); | |
| block.gate = take(D * F); block.up = take(D * F); block.down = take(F * D); | |
| block.window = header.attentionWindows[b]; | |
| block.capacity = block.window || CONTEXT; // positions the ring keeps | |
| this.blocks.push(block); | |
| } | |
| this.finalNorm = take(D); | |
| if (offset !== weights.length) throw new Error("the weight file's layout does not match chatgba"); | |
| // RoPE frequencies for adjacent pairs (0,1), (2,3), ...: base^(-2i/H). | |
| this.freqs = new Float64Array(H / 2); | |
| for (let i = 0; i < H / 2; ++i) this.freqs[i] = 1 / Math.pow(header.ropeBase, (2 * i) / H); | |
| // Working vectors for one position. | |
| this.x = new Float32Array(D); this.a = new Float32Array(D); this.mixed = new Float32Array(D); | |
| this.q = new Float32Array(H); this.k = new Float32Array(H); this.v = new Float32Array(H); this.head = new Float32Array(H); | |
| this.gateOut = new Float32Array(F); this.upOut = new Float32Array(F); | |
| this.scores = new Float64Array(CONTEXT); | |
| this.logits = new Float32Array(this.V); | |
| this.reset(); | |
| } | |
| // A fresh conversation: empty caches, position 0. | |
| reset() { | |
| this.position = 0; | |
| for (const block of this.blocks) { | |
| block.keys = block.groups.map(() => new Float32Array(block.capacity * this.H)); | |
| block.values = block.groups.map(() => new Float32Array(block.capacity * this.H)); | |
| block.historyA = new Float32Array((this.kernel - 1) * this.D); // row j-1: the input j positions back | |
| block.historyC = new Float32Array((this.kernel - 1) * this.D); | |
| } | |
| } | |
| // out = x · W, W row-major [x.length, out.length]. | |
| static matvec(x, W, out) { | |
| const n = out.length; | |
| const sum = new Float64Array(n); | |
| for (let i = 0; i < x.length; ++i) { | |
| const xi = x[i]; | |
| if (xi === 0) continue; | |
| const base = i * n; | |
| for (let j = 0; j < n; ++j) sum[j] += xi * W[base + j]; | |
| } | |
| out.set(sum); | |
| } | |
| static rmsNorm(x, gamma, out) { | |
| let sumSq = 0; | |
| for (let i = 0; i < x.length; ++i) sumSq += x[i] * x[i]; | |
| const inverse = 1 / Math.sqrt(sumSq / x.length + EPS); | |
| for (let i = 0; i < x.length; ++i) out[i] = x[i] * inverse * gamma[i]; | |
| } | |
| // A canon layer in place: a[t] += sum over j of w[j] * input[t - j], then | |
| // the input joins the history. | |
| canon(a, weight, history) { | |
| const D = this.D, taps = this.kernel - 1; | |
| const input = Float32Array.from(a); | |
| for (let c = 0; c < D; ++c) { | |
| let sum = weight[c] * input[c]; | |
| for (let j = 1; j <= taps; ++j) sum += weight[j * D + c] * history[(j - 1) * D + c]; | |
| a[c] = input[c] + sum; | |
| } | |
| history.copyWithin(D, 0, (taps - 1) * D); | |
| history.set(input, 0); | |
| } | |
| rope(vector, position) { | |
| for (let i = 0; i < this.H / 2; ++i) { | |
| const angle = position * this.freqs[i]; | |
| const cos = Math.cos(angle), sin = Math.sin(angle); | |
| const p = vector[2 * i], r = vector[2 * i + 1]; | |
| vector[2 * i] = p * cos - r * sin; | |
| vector[2 * i + 1] = p * sin + r * cos; | |
| } | |
| } | |
| // Run one byte through the model; returns the next byte's logits. | |
| step(byte) { | |
| const { D, H } = this; | |
| const x = this.x, a = this.a; | |
| const pos = this.position; | |
| x.set(this.embedding.subarray(byte * D, (byte + 1) * D)); | |
| const scale = 1 / Math.sqrt(H); | |
| for (const block of this.blocks) { | |
| // Attention half. | |
| ChatGBAEngine.rmsNorm(x, block.attentionNorm, a); | |
| this.canon(a, block.canonA, block.historyA); | |
| const slot = pos % block.capacity; | |
| const seen = Math.min(pos + 1, block.capacity); // cached positions this query sees, the newest last | |
| this.mixed.fill(0); | |
| for (let g = 0; g < block.groups.length; ++g) { | |
| const group = block.groups[g], keys = block.keys[g], values = block.values[g]; | |
| ChatGBAEngine.matvec(a, group.K, this.k); | |
| ChatGBAEngine.rmsNorm(this.k, group.keyNorm, this.k); | |
| this.rope(this.k, pos); | |
| ChatGBAEngine.matvec(a, group.V, this.v); | |
| keys.set(this.k, slot * H); | |
| values.set(this.v, slot * H); | |
| for (const query of group.queries) { | |
| ChatGBAEngine.matvec(a, query.Q, this.q); | |
| ChatGBAEngine.rmsNorm(this.q, query.queryNorm, this.q); | |
| this.rope(this.q, pos); | |
| let best = -Infinity; | |
| for (let n = 0; n < seen; ++n) { | |
| const s = ((pos - n) % block.capacity) * H; // n positions back | |
| let dot = 0; | |
| for (let i = 0; i < H; ++i) dot += this.q[i] * keys[s + i]; | |
| this.scores[n] = dot * scale; | |
| if (this.scores[n] > best) best = this.scores[n]; | |
| } | |
| let total = 0; | |
| for (let n = 0; n < seen; ++n) { this.scores[n] = Math.exp(this.scores[n] - best); total += this.scores[n]; } | |
| this.head.fill(0); | |
| for (let n = 0; n < seen; ++n) { | |
| const s = ((pos - n) % block.capacity) * H, w = this.scores[n] / total; | |
| for (let i = 0; i < H; ++i) this.head[i] += w * values[s + i]; | |
| } | |
| // mixed += head · O | |
| for (let i = 0; i < H; ++i) { | |
| const hi = this.head[i], base = i * D; | |
| for (let c = 0; c < D; ++c) this.mixed[c] += hi * query.O[base + c]; | |
| } | |
| } | |
| } | |
| for (let c = 0; c < D; ++c) x[c] += this.mixed[c]; // residual add | |
| // MLP half. | |
| ChatGBAEngine.rmsNorm(x, block.mlpNorm, a); | |
| this.canon(a, block.canonC, block.historyC); | |
| ChatGBAEngine.matvec(a, block.gate, this.gateOut); | |
| ChatGBAEngine.matvec(a, block.up, this.upOut); | |
| for (let j = 0; j < this.F; ++j) { | |
| const g = this.gateOut[j]; | |
| this.gateOut[j] = g / (1 + Math.exp(-g)) * this.upOut[j]; // silu(gate) × up | |
| } | |
| ChatGBAEngine.matvec(this.gateOut, block.down, this.mixed); | |
| for (let c = 0; c < D; ++c) x[c] += this.mixed[c]; // residual add | |
| } | |
| // Final norm, then the tied unembedding: logits = x · Embeddingᵀ. | |
| ChatGBAEngine.rmsNorm(x, this.finalNorm, a); | |
| for (let t = 0; t < this.V; ++t) { | |
| let dot = 0; | |
| const base = t * D; | |
| for (let c = 0; c < D; ++c) dot += a[c] * this.embedding[base + c]; | |
| this.logits[t] = dot; | |
| } | |
| this.position = pos + 1; | |
| return this.logits; | |
| } | |
| } | |
| // The cartridge's sampling: temperature, then keep the top k (0: all). Temperature 0 is greedy. | |
| function sampleByte(logits, temperature, topK, random = Math.random) { | |
| let best = 0; | |
| for (let i = 1; i < logits.length; ++i) if (logits[i] > logits[best]) best = i; | |
| if (!(temperature > 0)) return best; | |
| const weights = Array.from(logits, (l, i) => ({ i, w: Math.exp((l - logits[best]) / temperature) })); | |
| weights.sort((p, r) => r.w - p.w); | |
| const kept = topK > 0 ? weights.slice(0, topK) : weights; | |
| let total = 0; | |
| for (const k of kept) total += k.w; | |
| let target = random() * total; | |
| for (const k of kept) { if (target < k.w) return k.i; target -= k.w; } | |
| return best; | |
| } | |
| // A conversation, as the cartridge's chat.c runs it: one endless stream of | |
| // <|u|>message<|e|><|a|>reply<|e|> turns through the sliding cache. The last | |
| // byte of a reply's closing <|e|> is sampled but not run until the next | |
| // message, and a reply cut off partway into <|e|> keeps that start. | |
| class Chat { | |
| constructor(engine) { | |
| const markers = engine.header.chat || { user: "<|u|>", assistant: "<|a|>", end: "<|e|>" }; | |
| const encode = s => new TextEncoder().encode(s); | |
| this.engine = engine; | |
| this.USER = encode(markers.user); this.ASSISTANT = encode(markers.assistant); this.END = encode(markers.end); | |
| this.reset(); | |
| } | |
| reset() { | |
| this.engine.reset(); | |
| this.endSeen = this.END.length; // bytes of the last reply's <|e|> the engine has run | |
| this.logits = null; | |
| } | |
| // The longest k such that the reply ends with the first k bytes of <|e|>. | |
| partialEnd(reply) { | |
| for (let k = this.END.length - 1; k > 0; --k) { | |
| if (k <= reply.length && this.END.subarray(0, k).every((b, i) => reply[reply.length - k + i] === b)) return k; | |
| } | |
| return 0; | |
| } | |
| // Yields { reading, total } while it reads the turn, then { bytes } as reply | |
| // bytes become final (not a possible start of <|e|>). | |
| async *reply(message, { temperature, topK, maxReply, shouldStop, pause }) { | |
| const prompt = [...this.END.subarray(this.endSeen), ...this.USER, ...message, ...this.END, ...this.ASSISTANT]; | |
| for (let i = 0; i < prompt.length; ++i) { | |
| this.logits = this.engine.step(prompt[i]); | |
| yield { reading: i + 1, total: prompt.length }; | |
| await pause(); | |
| } | |
| const reply = []; | |
| let shown = 0, ended = false; | |
| while (reply.length < maxReply) { | |
| if (reply.length > 0 && shouldStop()) break; | |
| const byte = sampleByte(this.logits, temperature, topK); | |
| reply.push(byte); | |
| const n = reply.length, E = this.END.length; | |
| if (n >= E && this.END.every((b, i) => reply[n - E + i] === b)) { | |
| reply.length = n - E; | |
| ended = true; | |
| break; // its last byte waits for the next message | |
| } | |
| const hold = this.partialEnd(reply); | |
| if (shown < reply.length - hold) { | |
| yield { bytes: reply.slice(shown, reply.length - hold) }; | |
| shown = reply.length - hold; | |
| } | |
| this.logits = this.engine.step(byte); | |
| await pause(); | |
| } | |
| if (ended) { | |
| this.endSeen = this.END.length - 1; | |
| } else { | |
| this.endSeen = this.partialEnd(reply); | |
| reply.length -= this.endSeen; | |
| } | |
| if (shown < reply.length) yield { bytes: reply.slice(shown) }; | |
| } | |
| } | |
| // --- engine end | |
| // --- The page --------------------------------------------------------------------- | |
| const $ = id => document.getElementById(id); | |
| const notice = (text, error = false) => { const el = $("empty"); if (el) { el.textContent = text; el.classList.toggle("error", error); } }; | |
| let chat = null, busy = false, stopRequested = false; | |
| // Let the page breathe between bytes without the 4 ms setTimeout clamp. | |
| const channel = new MessageChannel(); | |
| const waiting = []; | |
| channel.port1.onmessage = () => waiting.shift()(); | |
| let lastBreath = performance.now(); | |
| function pause() { | |
| if (performance.now() - lastBreath < 30) return Promise.resolve(); | |
| return new Promise(resolve => { waiting.push(() => { lastBreath = performance.now(); resolve(); }); channel.port2.postMessage(0); }); | |
| } | |
| function ready(buffer) { | |
| try { | |
| chat = new Chat(new ChatGBAEngine(buffer)); | |
| } catch (e) { | |
| notice(e.message, true); | |
| return; | |
| } | |
| $("empty")?.remove(); // loaded: an empty chat | |
| $("input").disabled = false; | |
| $("send").disabled = false; | |
| $("input").focus(); | |
| } | |
| // The weights, from next to this page. | |
| fetch("chatgba-float32.gab") | |
| .then(r => { if (!r.ok) throw new Error(r.status + " " + r.statusText); return r.arrayBuffer(); }) | |
| .then(ready) | |
| .catch(e => notice("Could not load the weights (chatgba-float32.gab): " + e.message, true)); | |
| function addTurn(kind, label, text) { | |
| $("empty")?.remove(); | |
| const turn = document.createElement("div"); | |
| turn.className = "turn " + kind; | |
| turn.innerHTML = `<div class="who"></div><div><div class="text"></div><div class="meta"></div></div>`; | |
| turn.querySelector(".who").textContent = label; | |
| turn.querySelector(".text").textContent = text; | |
| $("log").appendChild(turn); | |
| $("log").scrollTop = $("log").scrollHeight; | |
| return turn; | |
| } | |
| function number(id, fallback) { | |
| const v = Number($(id).value); | |
| return Number.isFinite(v) ? v : fallback; | |
| } | |
| $("form").addEventListener("submit", async e => { | |
| e.preventDefault(); | |
| if (busy) { stopRequested = true; return; } | |
| const text = $("input").value.trim(); | |
| if (!text || !chat) return; | |
| $("input").value = ""; | |
| addTurn("user", "you", text); | |
| const turn = addTurn("model", "chatgba", ""); | |
| const out = turn.querySelector(".text"), meta = turn.querySelector(".meta"); | |
| out.classList.add("busy"); | |
| busy = true; stopRequested = false; | |
| $("send").textContent = "STOP"; | |
| $("input").disabled = true; | |
| const decoder = new TextDecoder(); | |
| let replyBytes = 0; | |
| const started = performance.now(); | |
| let replyStarted = 0; | |
| try { | |
| for await (const event of chat.reply(new TextEncoder().encode(text), { | |
| temperature: number("temperature", 0.7), topK: Math.round(number("top-k", 40)), maxReply: Math.round(number("max-reply", 300)), | |
| shouldStop: () => stopRequested, pause, | |
| })) { | |
| if (event.reading) { | |
| meta.textContent = `reading ${event.reading}/${event.total} bytes`; | |
| } else { | |
| if (!replyStarted) replyStarted = performance.now(); | |
| out.textContent += decoder.decode(new Uint8Array(event.bytes), { stream: true }); | |
| replyBytes += event.bytes.length; | |
| meta.textContent = `${replyBytes} bytes`; | |
| $("log").scrollTop = $("log").scrollHeight; | |
| } | |
| } | |
| out.textContent += decoder.decode(); | |
| const seconds = (performance.now() - (replyStarted || started)) / 1000; | |
| meta.textContent = `${replyBytes} bytes · ${replyBytes && seconds > 0 ? Math.round(replyBytes / seconds) : 0} bytes/s · position ${chat.engine.position.toLocaleString("en-US")}${stopRequested ? " · stopped" : ""}`; | |
| } catch (err) { | |
| meta.textContent = "error: " + err.message; | |
| } | |
| out.classList.remove("busy"); | |
| busy = false; | |
| $("send").textContent = "SEND"; | |
| $("input").disabled = false; | |
| $("input").focus(); | |
| }); | |
| // Enter sends, Shift+Enter starts a new line. | |
| $("input").addEventListener("keydown", e => { | |
| if (e.key === "Enter" && !e.shiftKey) { e.preventDefault(); $("form").requestSubmit(); } | |
| }); | |
| $("input").addEventListener("input", () => { const t = $("input"); t.style.height = "auto"; t.style.height = Math.min(t.scrollHeight, 160) + "px"; }); | |
| </script> | |
| </body> | |
| </html> | |