Franok / source /tools /quantization_check.mjs
yunusemrejr's picture
Publish source, film and evaluations from 699be55df232
2bff5be verified
Raw History Blame Contribute Delete
2.36 kB
// Check actual WASM logits against FP32 on fixed validation prefixes.
import fs from 'node:fs';
import path from 'node:path';
import { createEngine } from '../web/engine.mjs';
const [directory, bits, casesFile, report] = process.argv.slice(2);
if (!report || !['q4', 'q8'].includes(bits))
throw Error('Pass an export directory, q4 or q8, cases and report.');
const spec = JSON.parse(
fs.readFileSync(path.join(directory, bits === 'q4' ? 'model.json' : 'model-q8.json')),
);
const model = await createEngine(
fs.readFileSync('web/franok.wasm'),
spec,
fs.readFileSync(path.join(directory, 'model.' + bits)),
);
const cases = JSON.parse(fs.readFileSync(casesFile));
const metrics = {};
const rows = [];
for (const c of cases) {
model.reset();
let nll = 0,
full = 0,
kl = 0,
agree = 0;
for (let t = 0; t < c.ids.length - 1; t++) {
const j = t + 1 - c.at;
const scores = model.step(c.ids[t], j >= 0);
if (j < 0) continue;
let peak = 0;
for (let k = 1; k < scores.length; k++) if (scores[k] > scores[peak]) peak = k;
let z = 0;
for (const s of scores) z += Math.exp(s - scores[peak]);
const logz = Math.log(z) + scores[peak];
nll += logz - scores[c.ids[t + 1]];
full -= c.gold[j];
agree += Number(peak === c.top[j][0]);
let baseTail = 1,
quantTail = 1;
for (let k = 0; k < c.top[j].length; k++) {
const p = c.probs[j][k],
lq = scores[c.top[j][k]] - logz;
kl += p * (Math.log(p) - lq);
baseTail -= p;
quantTail -= Math.exp(lq);
}
if (baseTail > 1e-9) kl += baseTail * Math.log(baseTail / Math.max(1e-10, quantTail));
}
const n = c.gold.length;
rows.push({ key: c.key, lang: c.lang, group: c.group, source: c.source, n, nll, full, kl, agree });
for (const group of ['all', c.group, c.lang]) {
const tab = (metrics[group] ||= { n: 0, modelNll: 0, fullNll: 0, coarseKl: 0, top1: 0 });
tab.n += n;
tab.modelNll += nll;
tab.fullNll += full;
tab.coarseKl += kl;
tab.top1 += agree;
}
}
for (const tab of Object.values(metrics)) {
for (const key of ['modelNll', 'fullNll', 'coarseKl', 'top1']) tab[key] /= tab.n;
}
fs.writeFileSync(
report,
JSON.stringify(
{ format: spec.format, sha256: spec.sha256, scale: spec.scale_method, metrics, rows },
null,
2,
) + '\n',
);
console.log(metrics);