tiny-scene-three / tests /check_model.mjs
paralaif's picture
Tiny Scene: text to animated three.js scenes with a 4.85M Transformer
dd53693 verified
Raw History Blame Contribute Delete
1.63 kB
// 1) JavaScript inference must reproduce PyTorch token for token (tests/parity.json, from ml/infer.py --parity).
// 2) Hand-written prompts that are NOT produced by the training templates (tests/natural.json).
// node tests/check_model.mjs
import fs from 'node:fs';
import { TinyGPT } from '../web/js/nn/model.js';
const rd = (p) => fs.readFileSync(new URL(p, import.meta.url));
const buf = rd('../web/model/weights.bin');
const model = new TinyGPT(JSON.parse(rd('../web/model/model.json')), buf.buffer.slice(buf.byteOffset, buf.byteOffset + buf.byteLength));
console.log('model', model.meta.meta);
let fail = 0;
if (fs.existsSync(new URL('./parity.json', import.meta.url))) {
const ref = JSON.parse(rd('./parity.json'));
let same = 0, ms = 0;
for (const [prompt, want] of ref) {
const got = model.generate(prompt);
ms += got.ms;
if (got.tokens.join(' ') === want.join(' ')) same++;
else if (fail++ < 5) console.log('PARITY', JSON.stringify(prompt), '\n js:', got.tokens.join(' '), '\n py:', want.join(' '));
}
console.log(`parity with PyTorch: ${same}/${ref.length} identical, ${(ms / ref.length).toFixed(0)} ms per prompt`);
}
const nat = JSON.parse(rd('./natural.json'));
let ok = 0, total = 0;
for (const [prompt, want] of nat) {
const got = model.generate(prompt).tokens.join(' ');
if (want == null) { console.log(` (out of vocabulary) ${prompt}\n -> ${got}`); continue; }
total++;
if (got === want) ok++;
else console.log(` MISS ${prompt}\n got ${got}\n want ${want}`);
}
console.log(`hand-written prompts: ${ok}/${total} exact`);
process.exit(fail ? 1 : 0);