File size: 2,355 Bytes
2bff5be
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
// Check actual WASM logits against FP32 on fixed validation prefixes.
import fs from 'node:fs';
import path from 'node:path';
import { createEngine } from '../web/engine.mjs';

const [directory, bits, casesFile, report] = process.argv.slice(2);
if (!report || !['q4', 'q8'].includes(bits))
  throw Error('Pass an export directory, q4 or q8, cases and report.');
const spec = JSON.parse(
  fs.readFileSync(path.join(directory, bits === 'q4' ? 'model.json' : 'model-q8.json')),
);
const model = await createEngine(
  fs.readFileSync('web/franok.wasm'),
  spec,
  fs.readFileSync(path.join(directory, 'model.' + bits)),
);
const cases = JSON.parse(fs.readFileSync(casesFile));
const metrics = {};
const rows = [];
for (const c of cases) {
  model.reset();
  let nll = 0,
    full = 0,
    kl = 0,
    agree = 0;
  for (let t = 0; t < c.ids.length - 1; t++) {
    const j = t + 1 - c.at;
    const scores = model.step(c.ids[t], j >= 0);
    if (j < 0) continue;
    let peak = 0;
    for (let k = 1; k < scores.length; k++) if (scores[k] > scores[peak]) peak = k;
    let z = 0;
    for (const s of scores) z += Math.exp(s - scores[peak]);
    const logz = Math.log(z) + scores[peak];
    nll += logz - scores[c.ids[t + 1]];
    full -= c.gold[j];
    agree += Number(peak === c.top[j][0]);
    let baseTail = 1,
      quantTail = 1;
    for (let k = 0; k < c.top[j].length; k++) {
      const p = c.probs[j][k],
        lq = scores[c.top[j][k]] - logz;
      kl += p * (Math.log(p) - lq);
      baseTail -= p;
      quantTail -= Math.exp(lq);
    }
    if (baseTail > 1e-9) kl += baseTail * Math.log(baseTail / Math.max(1e-10, quantTail));
  }
  const n = c.gold.length;
  rows.push({ key: c.key, lang: c.lang, group: c.group, source: c.source, n, nll, full, kl, agree });
  for (const group of ['all', c.group, c.lang]) {
    const tab = (metrics[group] ||= { n: 0, modelNll: 0, fullNll: 0, coarseKl: 0, top1: 0 });
    tab.n += n;
    tab.modelNll += nll;
    tab.fullNll += full;
    tab.coarseKl += kl;
    tab.top1 += agree;
  }
}
for (const tab of Object.values(metrics)) {
  for (const key of ['modelNll', 'fullNll', 'coarseKl', 'top1']) tab[key] /= tab.n;
}
fs.writeFileSync(
  report,
  JSON.stringify(
    { format: spec.format, sha256: spec.sha256, scale: spec.scale_method, metrics, rows },
    null,
    2,
  ) + '\n',
);
console.log(metrics);