Download source/tools/quantization_check.mjs from yunusemrejr/Franok: direct link, hf CLI and curl.
- Browser
- Download file 2.36 kB
-
https://huggingface.co/yunusemrejr/Franok/resolve/main/source/tools/quantization_check.mjs
- Command line
-
hf download hf://yunusemrejr/Franok/source/tools/quantization_check.mjs
-
curl -L -o quantization_check.mjs https://huggingface.co/yunusemrejr/Franok/resolve/main/source/tools/quantization_check.mjs
2.36 kB
| // Check actual WASM logits against FP32 on fixed validation prefixes. | |
| import fs from 'node:fs'; | |
| import path from 'node:path'; | |
| import { createEngine } from '../web/engine.mjs'; | |
| const [directory, bits, casesFile, report] = process.argv.slice(2); | |
| if (!report || !['q4', 'q8'].includes(bits)) | |
| throw Error('Pass an export directory, q4 or q8, cases and report.'); | |
| const spec = JSON.parse( | |
| fs.readFileSync(path.join(directory, bits === 'q4' ? 'model.json' : 'model-q8.json')), | |
| ); | |
| const model = await createEngine( | |
| fs.readFileSync('web/franok.wasm'), | |
| spec, | |
| fs.readFileSync(path.join(directory, 'model.' + bits)), | |
| ); | |
| const cases = JSON.parse(fs.readFileSync(casesFile)); | |
| const metrics = {}; | |
| const rows = []; | |
| for (const c of cases) { | |
| model.reset(); | |
| let nll = 0, | |
| full = 0, | |
| kl = 0, | |
| agree = 0; | |
| for (let t = 0; t < c.ids.length - 1; t++) { | |
| const j = t + 1 - c.at; | |
| const scores = model.step(c.ids[t], j >= 0); | |
| if (j < 0) continue; | |
| let peak = 0; | |
| for (let k = 1; k < scores.length; k++) if (scores[k] > scores[peak]) peak = k; | |
| let z = 0; | |
| for (const s of scores) z += Math.exp(s - scores[peak]); | |
| const logz = Math.log(z) + scores[peak]; | |
| nll += logz - scores[c.ids[t + 1]]; | |
| full -= c.gold[j]; | |
| agree += Number(peak === c.top[j][0]); | |
| let baseTail = 1, | |
| quantTail = 1; | |
| for (let k = 0; k < c.top[j].length; k++) { | |
| const p = c.probs[j][k], | |
| lq = scores[c.top[j][k]] - logz; | |
| kl += p * (Math.log(p) - lq); | |
| baseTail -= p; | |
| quantTail -= Math.exp(lq); | |
| } | |
| if (baseTail > 1e-9) kl += baseTail * Math.log(baseTail / Math.max(1e-10, quantTail)); | |
| } | |
| const n = c.gold.length; | |
| rows.push({ key: c.key, lang: c.lang, group: c.group, source: c.source, n, nll, full, kl, agree }); | |
| for (const group of ['all', c.group, c.lang]) { | |
| const tab = (metrics[group] ||= { n: 0, modelNll: 0, fullNll: 0, coarseKl: 0, top1: 0 }); | |
| tab.n += n; | |
| tab.modelNll += nll; | |
| tab.fullNll += full; | |
| tab.coarseKl += kl; | |
| tab.top1 += agree; | |
| } | |
| } | |
| for (const tab of Object.values(metrics)) { | |
| for (const key of ['modelNll', 'fullNll', 'coarseKl', 'top1']) tab[key] /= tab.n; | |
| } | |
| fs.writeFileSync( | |
| report, | |
| JSON.stringify( | |
| { format: spec.format, sha256: spec.sha256, scale: spec.scale_method, metrics, rows }, | |
| null, | |
| 2, | |
| ) + '\n', | |
| ); | |
| console.log(metrics); | |