/* app.js — the DMCRA Lab controller: real quantisation and a real llama forward pass in the page. * * WHAT THIS FILE DOES * It loads a GGUF in the browser, runs the REAL transformer over the REAL weight bytes, and measures * the claims of the DMCRA document on the actual output of the model. It is the page half of * `scripts/dmcra_tests/elastic.test.mjs` — the Node suite already measures all of this — and the * algorithms here (average ranks, Spearman with ties, the per-block elastic plan, the per-block * quantiser) are ports of that suite's, deliberately ported rather than reinvented so the page and * the suite cannot disagree about what a number means. * * THE INSTRUMENTS, AND WHY NOT PERPLEXITY * Perplexity was the obvious quality metric and had to be dropped: on the fixture the suite uses, the * exact pass's perplexity measures WORSE than a uniform distribution while its top-1 is 16.9%, so any * noise that shrinks the logits toward uniform "improves" it. Perplexity is therefore not shown at * all. The page scores with instruments that are invariant to logit scale, so a quantiser cannot buy * a better figure by shrinking its logits: * * mean rank of the true token — chance is (vocab+1)/2; continuous, so tighter than a hit rate. * Spearman rho against EXACT — rank correlation of the full logit vector against the exact * pass: how much of the model's own preference order a quantiser * destroyed. Primary instrument. * top-1 rate — reported with its standard error so its weakness is visible. * * HONESTY RULES ENFORCED HERE * - Every figure displayed is measured by this run; nothing is quoted from a report. * - bits/weight comes from the pass itself (`meta.meanBitsPerWeight`), never recomputed here. * - Anything that cannot run is refused in words with the reason, and the panel stays usable. * - The zero-channel test reports exact equality or difference, whichever the numbers give. * * Dependency globals, loaded by index.html in this order: HDCGGUF, HDCWeights, HDCBPE, DMCRAQUANT, * HDCFORWARD. Same UMD shape as quant.js, so it also loads under Node with a stubbed DOM. */ (function (root, factory) { var api = factory(root); if (typeof module !== "undefined" && module.exports) module.exports = api; root.DMCRAAPP = api; })(typeof globalThis !== "undefined" ? globalThis : this, function (root) { "use strict"; var G = root.HDCGGUF, W = root.HDCWeights, BPE = root.HDCBPE, Q = root.DMCRAQUANT, FW = root.HDCFORWARD; var BLOCK = 32; // weights per quantisation block (one 4-byte scale each) var LADDER = ["binary", "quad", "ternary", "penta", "hepta", "oct", "nona"]; var MAX_BYTES = 48 * 1024 * 1024; // above this the tab stops being usable; refused, not hung var LOG_CAP = 400; /* The passage the suites score on: in-distribution tiny-story English, so the fixture is not * handicapped by text it was never trained on. */ var DEFAULT_TEXT = "Once upon a time there was a little girl named Lily . She lived in a big house " + "with her mother and father . One day she went to the park to play with her friends " + "and they had a lot of fun . The sun was hot and the sky was blue and the girl was happy ."; var state = { buf: null, label: null, header: null, vocab: null, tk: null, text: DEFAULT_TEXT, ids: [], matrices: [], totalWeights: 0, exact: null, positions: 16, busy: false, plan: null, requirements: null }; var els = {}; /* ------------------------------------------------------------------ dom plumbing */ function el(id) { return (typeof document !== "undefined" && document.getElementById) ? document.getElementById(id) : null; } /* Re-resolved at each entry point rather than cached once at load, so the Node harness can seed the * DOM and then call into this module without caring about ordering. */ function refreshEls() { els = { modelSelect: el("model-select"), loadBtn: el("load-btn"), loadStatus: el("load-status"), modelBody: el("model-tbody"), prompt: el("prompt"), runBtn: el("run-btn"), inferStatus: el("infer-status"), topkBody: el("topk-tbody"), positions: el("positions"), positionsLabel: el("positions-label"), ladderBtn: el("ladder-btn"), exactStatus: el("exact-status"), ladderStatus: el("ladder-status"), ladderBody: el("ladder-tbody"), zeroBtn: el("zero-btn"), zeroStatus: el("zero-status"), zeroBody: el("zero-tbody"), planSelect: el("plan-select"), planBtn: el("plan-btn"), planStatus: el("plan-status"), planBody: el("plan-tbody"), gridCanvas: el("grid-canvas"), log: el("log"), terminal: el("terminal"), cursor: el("cursor"), clearBtn: el("clear-btn") }; return els; } function log(msg, cls) { if (!els.log) return; var line = document.createElement("div"); line.className = "logline" + (cls ? " " + cls : ""); line.textContent = msg; els.log.appendChild(line); while (els.log.childNodes.length > LOG_CAP) els.log.removeChild(els.log.firstChild); els.log.scrollTop = els.log.scrollHeight; } function status(node, msg) { if (node) node.textContent = msg; } /* Status text and a log line together, so nothing is reported in one place only. */ function say(node, msg, cls) { status(node, msg); log(msg, cls); } function term(text, cls) { if (!els.terminal) return; var line = document.createElement("div"); line.className = "termlines" + (cls ? " " + cls : ""); line.textContent = text; els.terminal.appendChild(line); while (els.terminal.childNodes.length > LOG_CAP) els.terminal.removeChild(els.terminal.firstChild); if (els.cursor) els.terminal.appendChild(els.cursor); els.terminal.scrollTop = els.terminal.scrollHeight; } function yieldUI() { return new Promise(function (r) { setTimeout(r, 0); }); } function num(n, d) { if (n === null || n === undefined || !isFinite(n)) return "—"; return Number(n).toFixed(d === undefined ? 2 : d); } function int(n) { if (n === null || n === undefined || !isFinite(n)) return "—"; var s = String(Math.round(n)); return s.replace(/\B(?=(\d{3})+(?!\d))/g, ","); } function fmtBytes(n) { if (!n) return "—"; if (n < 1024) return n + " B"; if (n < 1024 * 1024) return (n / 1024).toFixed(1) + " KB"; return (n / (1024 * 1024)).toFixed(2) + " MB"; } function fmtBits(b) { return (b === null || b === undefined) ? "exact" : num(b, 3) + " b/w"; } /* Table helpers. First cell is the label column, the rest are numbers, which the stylesheet * right-aligns; the row class carries that intent so the markup and this file agree. */ function clearTable(tbody) { if (tbody) tbody.innerHTML = ""; } function addRow(tbody, cells, cls) { if (!tbody) return null; var tr = document.createElement("tr"); if (cls) tr.className = cls; for (var i = 0; i < cells.length; i++) { var td = document.createElement("td"); td.className = i === 0 ? "c-label" : "c-num"; td.textContent = cells[i] === null || cells[i] === undefined ? "—" : String(cells[i]); tr.appendChild(td); } tbody.appendChild(tr); return tr; } function addModelRow(label, value) { return addRow(els.modelBody, [label, value]); } /* ------------------------------------------------------------------ statistics * Ports of the Node suite's helpers. Average ranks for ties, because quantised models produce exact * ties constantly and a naive rank would make Spearman undefined exactly where it matters. */ function avgRanks(a) { var n = a.length, idx = new Array(n), i; for (i = 0; i < n; i++) idx[i] = i; idx.sort(function (x, y) { return a[x] - a[y]; }); var r = new Float64Array(n); i = 0; while (i < n) { var j = i; while (j + 1 < n && a[idx[j + 1]] === a[idx[i]]) j++; var avg = (i + j) / 2 + 1; for (var k = i; k <= j; k++) r[idx[k]] = avg; i = j + 1; } return r; } function pearson(a, b) { var n = a.length, i, ma = 0, mb = 0; for (i = 0; i < n; i++) { ma += a[i]; mb += b[i]; } ma /= n; mb /= n; var dot = 0, va = 0, vb = 0, da, db; for (i = 0; i < n; i++) { da = a[i] - ma; db = b[i] - mb; dot += da * db; va += da * da; vb += db * db; } var den = Math.sqrt(va * vb); return den > 0 ? dot / den : 0; } function spearman(a, b) { return pearson(avgRanks(a), avgRanks(b)); } /* Average rank of one token's logit within the whole distribution: 1 is best, vocab is worst. */ function rankOfTrue(logits, id) { var tv = logits[id], greater = 0, equal = 0; for (var i = 0; i < logits.length; i++) { if (logits[i] > tv) greater++; else if (logits[i] === tv) equal++; } return greater + 1 + (equal - 1) / 2; } function median(list) { var s = list.slice().sort(function (x, y) { return x - y; }), n = s.length; if (!n) return 0; return n % 2 ? s[(n - 1) / 2] : (s[n / 2 - 1] + s[n / 2]) / 2; } /* ------------------------------------------------------------------ quantisers * A quantiser is what forward.js calls per matrix: it returns the replacement values and, when it * can be described, its own bit cost, which the pass adds up and reports as meta.meanBitsPerWeight. * Returning `bits` is what makes the page's cost column a measurement by the pass rather than * arithmetic done here. */ function gridQuantiser(grid) { return function (values) { var q = Q.quantise(values, { grid: grid, blockSize: BLOCK }); return { values: q.values, bits: q.bits.effBits }; }; } /* Per-block plan application: each 32-weight block gets its own grid from the planner. A matrix * absent from the plan is returned untouched, and that is reported rather than hidden, because * silently quantising it with some default would make the plan's bit figure wrong. */ function planQuantiser(plan, missing) { return function (values, name) { var gs = plan.gridByBlock[name]; if (!gs) { if (missing && missing.indexOf(name) < 0) missing.push(name); return { values: values }; } var out = new Float32Array(values.length), bits = 0, b, off, len, q; for (b = 0; b < gs.length; b++) { off = b * BLOCK; len = Math.min(BLOCK, values.length - off); if (len <= 0) break; q = Q.quantiseBlock(values, off, len, gs[b], "lsq"); out.set(q.values, off); bits += q.bits.effBits * len; } return { values: out, bits: bits / values.length }; }; } /* ------------------------------------------------------------------ measurement core */ /* Run the pass over scored positions and collect everything the panels need. `stride` lets a cheap * probe score every k-th position when only a ranking is needed. */ async function evaluate(label, opts, positions, stride) { stride = stride || 1; var limit = Math.min(positions, state.ids.length - 1); var t0 = Date.now(); var logitsAt = new Map(), perPos = [], i, t; var hit = 0, top5 = 0, total = 0, bad = false, bits = null, rankSum = 0; for (t = 1; t <= limit; t++) { if ((t - 1) % stride) continue; var r = FW.run(state.buf, state.header, state.ids.slice(0, t), opts); var L = r.logits; var best = -Infinity, arg = -1; for (i = 0; i < L.length; i++) { if (!isFinite(L[i])) bad = true; if (L[i] > best) { best = L[i]; arg = i; } } var rank = rankOfTrue(L, state.ids[t]); perPos.push(rank); rankSum += rank; if (arg === state.ids[t]) hit++; if (rank <= 5) top5++; logitsAt.set(t, L); total++; if (r.meta && r.meta.meanBitsPerWeight !== null && r.meta.meanBitsPerWeight !== undefined) { bits = r.meta.meanBitsPerWeight; } if (total % 4 === 0) await yieldUI(); } if (!total) throw new Error("no positions were scored"); var meanRank = rankSum / total; var sd = 0; for (i = 0; i < perPos.length; i++) sd += (perPos[i] - meanRank) * (perPos[i] - meanRank); sd = Math.sqrt(sd / Math.max(1, total - 1)); return { label: label, total: total, perPos: perPos, logitsAt: logitsAt, meanRank: meanRank, rankSE: sd / Math.sqrt(total), medianRank: median(perPos), acc: hit / total, top1: hit / total, top5: top5 / total, accSE: Math.sqrt((hit / total) * (1 - hit / total) / total), bits: bits, ms: Date.now() - t0, msPerToken: (Date.now() - t0) / total, finite: !bad, spearman: null, rmse: null }; } /* Spearman of a configuration's logits against the exact pass at the same positions. */ function vsExact(cfg) { if (!state.exact) return null; var sum = 0, n = 0; cfg.logitsAt.forEach(function (L, t) { var E = state.exact.logitsAt.get(t); if (!E || E.length !== L.length) return; sum += spearman(L, E); n++; }); return n ? sum / n : null; } /* Paired comparison over identical positions: the correct test, and far tighter than comparing two * independent means because both configurations are scored on the same text. */ function pairedRank(a, b) { var n = Math.min(a.perPos.length, b.perPos.length), d = [], i; for (i = 0; i < n; i++) d.push(a.perPos[i] - b.perPos[i]); if (!n) return { mean: 0, se: 0, n: 0, t: 0 }; var mean = 0; for (i = 0; i < n; i++) mean += d[i]; mean /= n; var v = 0; for (i = 0; i < n; i++) v += (d[i] - mean) * (d[i] - mean); var se = Math.sqrt(v / Math.max(1, n - 1)) / Math.sqrt(n); return { mean: mean, se: se, n: n, t: se > 0 ? mean / se : 0 }; } /* Mean reconstruction RMSE over the quantised matrices, at the same grid and block size the pass * uses, so the error column describes the weights the pass actually ran on. */ function meanRMSE(grid) { var sum = 0; for (var i = 0; i < state.matrices.length; i++) { sum += Q.quantise(state.matrices[i].values, { grid: grid, blockSize: BLOCK }).error.rmse; } return state.matrices.length ? sum / state.matrices.length : 0; } /* ------------------------------------------------------------------ model loading */ function setText(text) { var before = state.ids.join(","); state.text = text || DEFAULT_TEXT; if (!state.tk || !state.vocab) { state.ids = []; return state.ids; } var body = state.tk.ids(state.text); var bos = (state.vocab.bosId === null || state.vocab.bosId === undefined) ? 1 : state.vocab.bosId; state.ids = [bos].concat(body); /* A reference measurement is only a reference for the token sequence it was taken on. Every rho * on this page matches positions by index, so if the passage changes, the exact pass's logits at * position t belong to a DIFFERENT token — comparing them would be silently meaningless rather * than wrong-looking. The baseline is therefore dropped the moment the sequence changes. */ if (state.ids.join(",") !== before) state.exact = null; return state.ids; } /* The exact baseline, re-measured when it does not cover what is about to be compared against it * (the visitor raised the position count, or the passage changed under it). */ async function ensureExact() { refreshEls(); if (state.exact && state.exact.total >= state.positions) return state.exact; try { var limit = Math.min(state.positions, state.ids.length - 1); log("re-measuring the EXACT baseline over " + limit + " positions — the reference has to belong " + "to the same text and position count as what is compared to it"); status(els.exactStatus, "measuring the exact pass…"); state.exact = await evaluate("exact", {}, state.positions); status(els.exactStatus, "exact pass · mean rank " + num(state.exact.meanRank, 1) + " · top-1 " + num(100 * state.exact.acc, 1) + "% (chance " + num(100 / state.vocab.tokens.length, 2) + "%)"); log("exact: mean rank " + num(state.exact.meanRank, 1) + " of " + int(state.vocab.tokens.length) + " · top-1 " + num(100 * state.exact.acc, 1) + "% ± " + num(100 * state.exact.accSE, 1) + " points", "ok"); } catch (e) { say(els.exactStatus, "refused: the exact pass could not be measured — " + e.message, "bad"); return null; } return state.exact; } /* Discover exactly the matrices the pass quantises by recording one short pass with a quantiser * that returns its input. Planning over a guessed list could plan for tensors the pass never reads, * or miss ones it does, and either would make the plan's bit figure a fiction. */ function inventory() { var seen = []; FW.run(state.buf, state.header, state.ids.slice(0, 4), { quantize: function (values, name) { seen.push({ name: name, values: values }); return { values: values }; }, quantizeName: "inventory" }); var matrices = [], total = 0, i, k, n, v, sum, sq, mean; for (i = 0; i < seen.length; i++) { v = seen[i].values; n = v.length; sum = 0; sq = 0; for (k = 0; k < n; k++) { sum += v[k]; sq += v[k] * v[k]; } mean = sum / n; matrices.push({ name: seen[i].name, n: n, values: v, variance: Math.max(0, sq / n - mean * mean), mse: { binary: Q.quantise(v, { grid: "binary", blockSize: BLOCK }).error.mse, ternary: Q.quantise(v, { grid: "ternary", blockSize: BLOCK }).error.mse, hepta: Q.quantise(v, { grid: "hepta", blockSize: BLOCK }).error.mse } }); total += n; } state.matrices = matrices; state.totalWeights = total; return matrices; } /* Everything the load button does, over an ArrayBuffer the caller already has. Kept separate from * the fetch so the Node harness can load the fixture directly. */ async function loadFromBuffer(buffer, label) { refreshEls(); state.buf = buffer; state.label = label || "buffer"; state.exact = null; state.plan = null; clearTable(els.modelBody); clearTable(els.topkBody); clearTable(els.ladderBody); clearTable(els.zeroBody); clearTable(els.planBody); if (!G || !FW || !Q || !W) { say(els.loadStatus, "refused: a required module is missing (gguf.js, weights.js, quant.js, forward.js)", "bad"); return { ok: false, reason: "missing module" }; } log("parsing " + label + " (" + fmtBytes(buffer.byteLength) + ") — header, tensor index, metadata KVs"); state.header = G.parse(buffer, { metadataOnly: false }); state.vocab = G.vocab(state.header); state.tk = BPE.fromVocab(state.vocab); var req = FW.requirements(state.header); state.requirements = req; addModelRow("file", state.label); addModelRow("bytes", fmtBytes(buffer.byteLength)); addModelRow("architecture", String(state.header.architecture || "—")); addModelRow("gguf version", String(state.header.version)); addModelRow("tensors", int(state.header.tensors.length)); addModelRow("vocabulary", int(state.vocab.tokens.length) + " symbols"); addModelRow("layers", int(req.layers) + " · " + int(req.meta && req.meta.nEmbd) + "-dim hidden"); addModelRow("tokenizer", state.tk.exact ? "byte-level BPE (exact)" : "greedy longest match (no merge table)"); if (!req.feasible) { say(els.loadStatus, "refused: " + req.reason, "bad"); term("REFUSED: " + req.reason, "warn"); return { ok: false, reason: req.reason }; } if (buffer.byteLength > MAX_BYTES) { var why = "the file is " + fmtBytes(buffer.byteLength) + ", over this page's " + fmtBytes(MAX_BYTES) + " in-tab ceiling — a pure-JS pass over it would hang the tab, so it is refused rather than attempted"; say(els.loadStatus, "refused: " + why, "bad"); term("REFUSED: " + why, "warn"); return { ok: false, reason: why }; } if (req.note) { log(req.note, "warn"); term(req.note, "warn"); } var ids = setText(state.text); if (ids.length < 3) { say(els.loadStatus, "refused: the passage tokenised to " + ids.length + " token(s)", "bad"); return { ok: false, reason: "passage too short" }; } var mats = inventory(); var weights = 0; for (var i = 0; i < mats.length; i++) weights += mats[i].n; addModelRow("quantisable matrices", int(mats.length) + " · " + int(weights) + " weights"); addModelRow("scored positions", int(state.positions) + " of " + int(ids.length - 1)); say(els.loadStatus, "loaded " + state.label + " · " + int(mats.length) + " matrices / " + int(weights) + " weights quantisable", "ok"); log("inventory: the pass quantises " + mats.length + " matrices (" + int(weights) + " weights); norms and embeddings stay float, as they do in the Node suite"); /* The exact baseline is measured at load, because every quality figure afterwards is expressed * against it and a missing reference would make the panels silently meaningless. */ log("measuring the EXACT pass over " + Math.min(state.positions, ids.length - 1) + " positions — this is the reference every later figure is compared to"); status(els.exactStatus, "measuring the exact pass…"); state.exact = await evaluate("exact", {}, state.positions); status(els.exactStatus, "exact pass · mean rank " + num(state.exact.meanRank, 1) + " · top-1 " + num(100 * state.exact.acc, 1) + "% (chance " + num(100 / state.vocab.tokens.length, 2) + "%)"); log("exact: mean rank " + num(state.exact.meanRank, 1) + " of " + int(state.vocab.tokens.length) + " (chance " + num((state.vocab.tokens.length + 1) / 2, 1) + ") · top-1 " + num(100 * state.exact.acc, 1) + "% ± " + num(100 * state.exact.accSE, 1) + " points", "ok"); term("exact F32 · mean rank " + num(state.exact.meanRank, 1) + " · top-1 " + num(100 * state.exact.acc, 1) + "% · " + num(state.exact.msPerToken, 0) + " ms/token"); term("the top-1 standard error at " + state.exact.total + " positions is ± " + num(100 * state.exact.accSE, 1) + " points, so smaller differences are NOT resolvable — " + "ranks and rho are used instead", "warn"); return { ok: true, matrices: mats.length, weights: weights, exact: state.exact }; } async function loadFromUrl(url, label) { refreshEls(); say(els.loadStatus, "fetching " + label + "…"); log("fetching " + url, null); var res = await fetch(url); if (!res.ok) throw new Error("HTTP " + res.status + " for " + url); var total = Number(res.headers && res.headers.get ? (res.headers.get("content-length") || 0) : 0); var buffer; if (res.body && res.body.getReader) { var reader = res.body.getReader(), chunks = [], got = 0, step; while (true) { step = await reader.read(); if (step.done) break; if (!step.value || !step.value.length) continue; chunks.push(step.value); got += step.value.length; if (total) status(els.loadStatus, "fetching " + label + " — " + fmtBytes(got) + " / " + fmtBytes(total)); if (got % (4 * 1024 * 1024) < step.value.length) await yieldUI(); } var joined = new Uint8Array(got), at = 0; for (var i = 0; i < chunks.length; i++) { joined.set(chunks[i], at); at += chunks[i].length; } buffer = joined.buffer; } else { buffer = await res.arrayBuffer(); } return loadFromBuffer(buffer, label); } /* ------------------------------------------------------------------ panel: inference */ async function runInference() { refreshEls(); if (!state.buf) { say(els.inferStatus, "refused: load a model first", "bad"); return null; } var text = els.prompt && typeof els.prompt.value === "string" ? els.prompt.value : state.text; if (!text || !text.replace(/\s+/g, "")) { say(els.inferStatus, "refused: the prompt is empty — the pass needs tokens", "bad"); return null; } var ids = setText(text); if (ids.length < 2) { say(els.inferStatus, "refused: the prompt tokenised to " + ids.length + " token(s)", "bad"); return null; } clearTable(els.topkBody); try { log("forward pass: " + (ids.length - 1) + " tokens through " + state.requirements.layers + " layers over the file's own weight bytes"); status(els.inferStatus, "running…"); var t0 = Date.now(); var r = FW.run(state.buf, state.header, ids, {}); var passMs = Date.now() - t0; var top = FW.topK(r.logits, 6); var rows = 0; for (var i = 0; i < top.length; i++) { var id = top[i].id; addRow(els.topkBody, [ "#" + (i + 1), JSON.stringify(FW.surface(state.vocab.tokens[id])), num(top[i].logit, 3), num(100 * top[i].p, 2) + "%" ]); rows++; } if (!rows) throw new Error("the pass returned no candidates"); /* Next-token continuation on the page's own terminal, so the visitor sees the model speak. */ var line = top.slice(0, 3).map(function (c) { return FW.surface(state.vocab.tokens[c.id]); }).join(" "); term("next after " + JSON.stringify(text.slice(-24)) + " -> " + line); status(els.inferStatus, "done · " + ids.length + " tokens in " + num(passMs, 0) + " ms"); log("top-6 next tokens written · " + ids.length + " tokens in " + num(passMs, 0) + " ms (" + num(passMs / ids.length, 1) + " ms/token); exact weights, no quantisation in this panel", "ok"); return { ids: ids.length, top: top, ms: passMs }; } catch (e) { say(els.inferStatus, "refused: " + e.message, "bad"); term("forward pass failed: " + e.message, "bad"); return null; } } /* ------------------------------------------------------------------ panel: the ladder */ async function runLadder() { refreshEls(); if (!state.buf) { say(els.ladderStatus, "refused: load a model first", "bad"); return null; } if (!(await ensureExact())) { say(els.ladderStatus, "refused: no exact baseline", "bad"); return null; } clearTable(els.ladderBody); var results = { exact: state.exact }, i; log("ladder: " + LADDER.length + " level sets × " + state.exact.total + " positions through the real pass. Each row is scored against the exact pass at the same position."); for (i = 0; i < LADDER.length; i++) { var g = LADDER[i]; var info = Q.gridInfo(g); status(els.ladderStatus, "running " + g + " (" + info.count + " channels, " + info.indexBits + " index bits) …"); try { var cfg = await evaluate(g, { quantize: gridQuantiser(g) }, state.positions); cfg.spearman = vsExact(cfg); cfg.rmse = meanRMSE(g); results[g] = cfg; addRow(els.ladderBody, [ g + " {" + info.levels.join(",") + "}", fmtBits(cfg.bits), num(cfg.rmse, 5), num(cfg.meanRank, 1), num(100 * cfg.acc, 1) + "%", cfg.spearman === null ? "—" : num(cfg.spearman, 4) ]); log(" " + g + ": " + fmtBits(cfg.bits) + " · rmse " + num(cfg.rmse, 5) + " · mean rank " + num(cfg.meanRank, 1) + " · top-1 " + num(100 * cfg.acc, 1) + "% · rho vs exact " + num(cfg.spearman, 4), "ok"); } catch (e) { addRow(els.ladderBody, [g, "refused", "—", "—", "—", "—"], "bad"); log(" " + g + " refused: " + e.message, "bad"); } } /* The comparisons that decide the document's claims, stated as paired measurements. */ var tern = results.ternary, quad = results.quad, hept = results.hepta; if (tern && quad) { var pq = pairedRank(quad, tern); term("equal 2 index bits: quad (no zero) vs ternary -> mean rank " + (pq.mean > 0 ? "+" : "") + num(pq.mean, 2) + " (t " + num(pq.t, 2) + "), rho " + num(quad.spearman, 4) + " vs " + num(tern.spearman, 4)); } if (tern && hept) { var ph = pairedRank(hept, tern); term("wider grid: hepta vs ternary -> mean rank " + (ph.mean > 0 ? "+" : "") + num(ph.mean, 2) + " (t " + num(ph.t, 2) + ")"); } var exactRank = num(state.exact.meanRank, 1); term("ladder done · exact mean rank " + exactRank + " · best quantised " + num(Math.min.apply(null, LADDER.map(function (g) { return (results[g] || {}).meanRank || Infinity; })), 1)); status(els.ladderStatus, "done — " + LADDER.length + " level sets measured"); return results; } /* ------------------------------------------------------------------ panel: the 0/0/0 test */ async function runZeroTest() { refreshEls(); if (!state.buf) { say(els.zeroStatus, "refused: load a model first", "bad"); return null; } if (!(await ensureExact())) { say(els.zeroStatus, "refused: no exact baseline", "bad"); return null; } clearTable(els.zeroBody); log("zero test: quantising with {0,±1} and with the document's {0,0,0,±1} and comparing every " + "logit for EXACT equality — the claim under test is identity, not closeness"); status(els.zeroStatus, "running both…"); try { var one = await evaluate("ternary", { quantize: gridQuantiser("ternary") }, state.positions); one.spearman = vsExact(one); var three = await evaluate("ternaryDup", { quantize: gridQuantiser("ternaryDup") }, state.positions); three.spearman = vsExact(three); var identical = true, compared = 0, firstDiff = null; three.logitsAt.forEach(function (L, t) { var T = one.logitsAt.get(t); if (!T) return; for (var i = 0; i < L.length; i++) { if (L[i] !== T[i]) { identical = false; if (!firstDiff) firstDiff = { t: t, i: i, a: T[i], b: L[i] }; } } compared++; }); addRow(els.zeroBody, ["{0,±1} 3 channels", fmtBits(one.bits), identical ? "true" : "false", num(one.meanRank, 1), num(100 * one.acc, 1) + "%"]); addRow(els.zeroBody, ["{0,0,0,±1} 5 channels", fmtBits(three.bits), identical ? "true" : "false", num(three.meanRank, 1), num(100 * three.acc, 1) + "%"], identical ? "" : "bad"); var info1 = Q.gridInfo("ternary"), info3 = Q.gridInfo("ternaryDup"); term("logits compared bit-for-bit at " + compared + " positions, " + int(state.vocab.tokens.length) + " logits each: identical = " + identical); term("channels " + info1.count + " -> " + info3.count + " (decodable values " + info1.distinctLevels + " -> " + info3.distinctLevels + ", so " + info3.wastedChannels + " channels are unreachable) · " + "index bits " + info1.indexBits + " -> " + info3.indexBits); if (identical) { log("result: the widened zero changes NOTHING in the output while its accounting charges more " + "bits per weight — 3 zero channels decode to the single value 0", "ok"); term("RESULT: the multi-channel zero is not an information channel — it reconstructs and predicts " + "IDENTICALLY, and costs " + num((three.bits || 0) - (one.bits || 0), 3) + " more bits/weight to do so"); } else { log("result: the two configurations DID differ — first difference at position " + firstDiff.t + ", logit " + firstDiff.i + ": " + firstDiff.a + " vs " + firstDiff.b, "warn"); term("RESULT: the logits differ, so report the difference rather than the claim", "warn"); } status(els.zeroStatus, "done — logits identical: " + identical); return { identical: identical, one: one, three: three }; } catch (e) { say(els.zeroStatus, "refused: " + e.message, "bad"); return null; } } /* ------------------------------------------------------------------ panel: the elastic plan */ /* The document's mechanism, made decidable: every 32-weight block of every quantisable matrix is * ordered by a measured key, and while the mass promoted to the 3-bit grid and the mass demoted to * the 1-bit grid stay balanced, the most important blocks are unclamped and the least important are * clamped. The balance is what makes the average exactly 2.0 index bits — the same budget uniform * ternary spends — so the comparison is at equal bits and not merely at equal grid count. */ function elasticPlan(keyName) { var units = [], i, off, m, key; for (i = 0; i < state.matrices.length; i++) { m = state.matrices[i]; key = keyName === "variance" ? m.variance : (m.mse.binary - m.mse.hepta); for (off = 0; off < m.n; off += BLOCK) units.push({ mi: i, off: off, key: key }); } if (!units.length) return null; units.sort(function (a, b) { return b.key - a.key; }); var gridByBlock = {}; for (i = 0; i < state.matrices.length; i++) { m = state.matrices[i]; gridByBlock[m.name] = new Array(Math.ceil(m.n / BLOCK)); for (var b = 0; b < gridByBlock[m.name].length; b++) gridByBlock[m.name][b] = "ternary"; } var massUp = 0, massDown = 0, hi = 0, lo = units.length - 1, u; while (hi <= lo) { if (massUp <= massDown) { u = units[hi]; gridByBlock[state.matrices[u.mi].name][Math.floor(u.off / BLOCK)] = "hepta"; massUp += BLOCK; hi++; } else { u = units[lo]; gridByBlock[state.matrices[u.mi].name][Math.floor(u.off / BLOCK)] = "binary"; massDown += BLOCK; lo--; } } return { key: keyName, gridByBlock: gridByBlock, bits: 2 + (massUp - massDown) / state.totalWeights, promoted: massUp / BLOCK, demoted: massDown / BLOCK, blocks: units.length }; } function drawPlan(plan) { var c = els.gridCanvas; if (!c || !c.getContext) return false; var names = Object.keys(plan.gridByBlock), rows = names.length, rowH = 9, gap = 3; var w = 1024, h = rows * (rowH + gap) + 2; c.width = w; c.height = h; var ctx = c.getContext("2d"); ctx.fillStyle = "#06040f"; ctx.fillRect(0, 0, w, h); for (var r = 0; r < rows; r++) { var g = plan.gridByBlock[names[r]], nb = g.length, bw = w / nb; for (var b = 0; b < nb; b++) { /* These must match the legend in index.html, or the map mislabels itself. */ ctx.fillStyle = g[b] === "hepta" ? "#ff2d78" : (g[b] === "binary" ? "#3b82f6" : "#6b6480"); ctx.fillRect(b * bw, r * (rowH + gap), Math.max(1, bw), rowH); } } return true; } async function runPlan() { refreshEls(); if (!state.buf) { say(els.planStatus, "refused: load a model first", "bad"); return null; } if (!(await ensureExact())) { say(els.planStatus, "refused: no exact baseline", "bad"); return null; } var keyName = els.planSelect && els.planSelect.value === "variance" ? "variance" : "benefit"; clearTable(els.planBody); try { var plan = elasticPlan(keyName); if (!plan) { say(els.planStatus, "refused: no quantisable matrices to plan over", "bad"); return null; } state.plan = plan; log("plan '" + keyName + "': " + plan.blocks + " blocks of " + BLOCK + " weights ordered by the " + keyName + " key; " + plan.promoted + " block(s) unclamped to hepta and " + plan.demoted + " clamped to binary, which balances to " + num(plan.bits, 4) + " index bits/weight"); /* The baseline at the SAME budget: uniform ternary spends 2 index bits on every weight. */ status(els.planStatus, "running uniform ternary at 2.0 bits…"); /* No cross-panel cache. A baseline is only a baseline for the token sequence and position count * it was measured on, and `state.exact` is already nulled whenever the passage changes — a cache * keyed on anything less than the token sequence would compare two different passages. */ var tern = await evaluate("ternary", { quantize: gridQuantiser("ternary") }, state.positions); tern.spearman = vsExact(tern); status(els.planStatus, "running the elastic plan…"); var missing = []; var elastic = await evaluate("elastic:" + keyName, { quantize: planQuantiser(plan, missing), quantizeName: "elastic:" + keyName }, state.positions); elastic.spearman = vsExact(elastic); if (missing.length) log("note: " + missing.length + " matrix(es) were not in the plan and stayed " + "exact: " + missing.slice(0, 4).join(", "), "warn"); var pd = pairedRank(elastic, tern); addRow(els.planBody, ["uniform ternary", fmtBits(tern.bits), num(tern.meanRank, 1), num(tern.spearman, 4), "baseline"]); addRow(els.planBody, ["elastic:" + keyName, fmtBits(elastic.bits), num(elastic.meanRank, 1), num(elastic.spearman, 4), (pd.mean > 0 ? "+" : "") + num(pd.mean, 2) + " (t " + num(pd.t, 2) + ")"], pd.mean > 0 ? "bad" : "ok"); var drew = drawPlan(plan); if (drew) { term("grid map: one row per quantisable matrix, one cell per " + BLOCK + "-weight block — " + "pink = hepta (3 bits), grey = ternary (2 bits), blue = binary (1 bit)"); } term("equal budget (" + num(tern.bits, 3) + " vs " + num(elastic.bits, 3) + " b/w): uniform ternary mean rank " + num(tern.meanRank, 1) + " rho " + num(tern.spearman, 4) + " · elastic mean rank " + num(elastic.meanRank, 1) + " rho " + num(elastic.spearman, 4)); if (resolvable) { term("resolvable at this position count: |t| = " + num(Math.abs(pd.t), 2) + " >= 2"); } /* The verdict must respect the measurement's resolution. At the slider's default this comparison * is under-powered: the sign of the mean alone is not a finding, and printing one would be the * exact overclaiming this page exists to avoid. Below |t| = 2 the panel says so and points at the * suite that ran the positions this page cannot afford. */ var resolvable = Math.abs(pd.t) >= 2; var diff = (pd.mean > 0 ? "+" : "") + num(pd.mean, 1); if (!resolvable) { term("UNDECIDED at " + tern.total + " positions: the " + keyName + " plan's mean rank is " + diff + " versus uniform ternary, but t = " + num(pd.t, 2) + " — this run cannot resolve it. Raise the " + "slider and re-run."); term("The Node suite runs the same experiment over 89 positions, where it resolves: the benefit key " + "is +48.52 rank positions (t 3.21) and the variance key +46.35 (t 3.22), i.e. both WORSE than " + "uniform ternary at equal bits. Source: reports/dmcra-verdicts.md.", "warn"); log("result: elastic:" + keyName + " vs uniform ternary is UNDECIDED at " + tern.total + " positions (rank " + diff + ", t " + num(pd.t, 2) + ") — reporting no verdict rather than the " + "sign of an unresolved difference", "warn"); } else if (pd.mean > 0) { term("RESULT: unclamping by the " + keyName + " key is WORSE than spending the same bits uniformly " + "on a ternary grid — by " + num(pd.mean, 1) + " rank positions (t " + num(pd.t, 2) + ")", "bad"); log("result: elastic:" + keyName + " loses to uniform ternary at equal bits (rank +" + num(pd.mean, 1) + ", t " + num(pd.t, 2) + ")", "warn"); } else { term("RESULT: unclamping by the " + keyName + " key BEATS uniform ternary at equal bits — by " + num(-pd.mean, 1) + " rank positions (t " + num(pd.t, 2) + ")", "ok"); log("result: elastic:" + keyName + " beats uniform ternary at equal bits", "ok"); } if (elastic.bits !== null && Math.abs(elastic.bits - (plan.bits + 1)) > 1e-6) { log("note: the pass measured " + num(elastic.bits, 3) + " b/w for the plan, which is not the " + "planned " + num(plan.bits + 1, 3) + " including its scale bits — reported as measured", "warn"); } status(els.planStatus, "done — plan " + num(plan.bits, 4) + " index bits/weight over " + plan.blocks + " blocks"); return { plan: plan, elastic: elastic, ternary: tern, rankDiff: pd }; } catch (e) { say(els.planStatus, "refused: " + e.message, "bad"); log("plan failed: " + e.message, "bad"); return null; } } /* ------------------------------------------------------------------ wiring */ function readPositions() { refreshEls(); var v = els.positions && els.positions.value !== undefined ? parseInt(els.positions.value, 10) : NaN; if (isFinite(v)) state.positions = Math.max(4, Math.min(32, v)); status(els.positionsLabel, state.positions + " positions"); return state.positions; } function busy(on, btn) { state.busy = !!on; if (btn) btn.disabled = !!on; } function on(id, event, fn) { var node = el(id); if (node && node.addEventListener) node.addEventListener(event, fn); } function init() { refreshEls(); on("load-btn", "click", async function () { busy(true, els.loadBtn); try { var sel = els.modelSelect; var url = sel && sel.value; var label = url ? url.split("/").pop() : "model"; await loadFromUrl(url, label); } catch (e) { say(els.loadStatus, "refused: " + e.message, "bad"); log("load failed: " + e.message, "bad"); } finally { busy(false, els.loadBtn); } }); on("run-btn", "click", async function () { busy(true, els.runBtn); try { await runInference(); } finally { busy(false, els.runBtn); } }); on("ladder-btn", "click", async function () { readPositions(); busy(true, els.ladderBtn); try { await runLadder(); } finally { busy(false, els.ladderBtn); } }); on("zero-btn", "click", async function () { readPositions(); busy(true, els.zeroBtn); try { await runZeroTest(); } finally { busy(false, els.zeroBtn); } }); on("plan-btn", "click", async function () { readPositions(); busy(true, els.planBtn); try { await runPlan(); } finally { busy(false, els.planBtn); } }); on("positions", "input", readPositions); on("clear-btn", "click", function () { clearTable(els.ladderBody); clearTable(els.zeroBody); clearTable(els.planBody); clearTable(els.topkBody); if (els.log) els.log.innerHTML = ""; if (els.terminal) { els.terminal.innerHTML = ""; if (els.cursor) els.terminal.appendChild(els.cursor); } log("cleared", null); }); readPositions(); if (els.prompt && (!els.prompt.value || !els.prompt.value.length)) els.prompt.value = DEFAULT_TEXT; term("DMCRA Lab — ready. Load a GGUF; the exact pass is measured first, because every figure " + "afterwards is expressed against it."); log("ready: " + LADDER.length + " level sets available, " + int(Object.keys(Q.GRIDS).length) + " grids in the quantiser", "ok"); return true; } var api = { els: els, refreshEls: refreshEls, log: log, status: status, state: state, init: init, loadFromBuffer: loadFromBuffer, loadFromUrl: loadFromUrl, runInference: runInference, runLadder: runLadder, runZeroTest: runZeroTest, runPlan: runPlan, elasticPlan: elasticPlan, gridQuantiser: gridQuantiser, planQuantiser: planQuantiser, evaluate: evaluate, ensureExact: ensureExact, setText: setText, meanRMSE: meanRMSE, pairedRank: pairedRank, stats: { avgRanks: avgRanks, spearman: spearman, pearson: pearson, rankOfTrue: rankOfTrue, median: median }, config: { BLOCK: BLOCK, LADDER: LADDER, MAX_BYTES: MAX_BYTES, DEFAULT_TEXT: DEFAULT_TEXT } }; return api; });