// 1) JavaScript inference must reproduce PyTorch token for token (tests/parity.json, from ml/infer.py --parity). // 2) Hand-written prompts that are NOT produced by the training templates (tests/natural.json). // node tests/check_model.mjs import fs from 'node:fs'; import { TinyGPT } from '../web/js/nn/model.js'; const rd = (p) => fs.readFileSync(new URL(p, import.meta.url)); const buf = rd('../web/model/weights.bin'); const model = new TinyGPT(JSON.parse(rd('../web/model/model.json')), buf.buffer.slice(buf.byteOffset, buf.byteOffset + buf.byteLength)); console.log('model', model.meta.meta); let fail = 0; if (fs.existsSync(new URL('./parity.json', import.meta.url))) { const ref = JSON.parse(rd('./parity.json')); let same = 0, ms = 0; for (const [prompt, want] of ref) { const got = model.generate(prompt); ms += got.ms; if (got.tokens.join(' ') === want.join(' ')) same++; else if (fail++ < 5) console.log('PARITY', JSON.stringify(prompt), '\n js:', got.tokens.join(' '), '\n py:', want.join(' ')); } console.log(`parity with PyTorch: ${same}/${ref.length} identical, ${(ms / ref.length).toFixed(0)} ms per prompt`); } const nat = JSON.parse(rd('./natural.json')); let ok = 0, total = 0; for (const [prompt, want] of nat) { const got = model.generate(prompt).tokens.join(' '); if (want == null) { console.log(` (out of vocabulary) ${prompt}\n -> ${got}`); continue; } total++; if (got === want) ok++; else console.log(` MISS ${prompt}\n got ${got}\n want ${want}`); } console.log(`hand-written prompts: ${ok}/${total} exact`); process.exit(fail ? 1 : 0);