File size: 1,627 Bytes
dd53693
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
// 1) JavaScript inference must reproduce PyTorch token for token (tests/parity.json, from ml/infer.py --parity).
// 2) Hand-written prompts that are NOT produced by the training templates (tests/natural.json).
//   node tests/check_model.mjs
import fs from 'node:fs';
import { TinyGPT } from '../web/js/nn/model.js';

const rd = (p) => fs.readFileSync(new URL(p, import.meta.url));
const buf = rd('../web/model/weights.bin');
const model = new TinyGPT(JSON.parse(rd('../web/model/model.json')), buf.buffer.slice(buf.byteOffset, buf.byteOffset + buf.byteLength));
console.log('model', model.meta.meta);

let fail = 0;
if (fs.existsSync(new URL('./parity.json', import.meta.url))) {
  const ref = JSON.parse(rd('./parity.json'));
  let same = 0, ms = 0;
  for (const [prompt, want] of ref) {
    const got = model.generate(prompt);
    ms += got.ms;
    if (got.tokens.join(' ') === want.join(' ')) same++;
    else if (fail++ < 5) console.log('PARITY', JSON.stringify(prompt), '\n  js:', got.tokens.join(' '), '\n  py:', want.join(' '));
  }
  console.log(`parity with PyTorch: ${same}/${ref.length} identical, ${(ms / ref.length).toFixed(0)} ms per prompt`);
}

const nat = JSON.parse(rd('./natural.json'));
let ok = 0, total = 0;
for (const [prompt, want] of nat) {
  const got = model.generate(prompt).tokens.join(' ');
  if (want == null) { console.log(`  (out of vocabulary) ${prompt}\n      -> ${got}`); continue; }
  total++;
  if (got === want) ok++;
  else console.log(`  MISS ${prompt}\n      got  ${got}\n      want ${want}`);
}
console.log(`hand-written prompts: ${ok}/${total} exact`);
process.exit(fail ? 1 : 0);