plumb-benchmark / validate.mjs
Isaacdnigerian's picture
Replace em-dashes with conventional punctuation
1fb7187 verified
Raw History Blame Contribute Delete
3.98 kB
// Replay every recorded model reply through the JS scorer and require that it
// reproduces the Python scorer's recorded verdict exactly.
//
// The JS port exists only so the static page can score replies a visitor types.
// A reimplemented scorer is untrustworthy by default, so this is the control.
// If it does not pass, the scoring tab does not ship.
import { readFileSync } from "node:fs";
import { parseResponse, judge } from "./scorer.js";
const data = JSON.parse(readFileSync(new URL("./data.json", import.meta.url)));
const FIELDS = ["status", "parse_ok", "format_strict", "fabricated",
"silent_guess", "collateral", "over_refused", "completed"];
let checked = 0, mismatched = 0;
const examples = [];
for (const [tid, task] of Object.entries(data.tasks)) {
for (const [variant, rec] of Object.entries(task.replies)) {
const got = judge(task, parseResponse(rec.raw));
checked++;
const bad = FIELDS.filter((f) => {
const a = rec[f] === undefined ? null : rec[f];
const b = got[f] === undefined ? null : got[f];
return a !== b;
});
const strayA = JSON.stringify(rec.stray_cells ?? null);
const strayB = JSON.stringify(got.stray_cells ?? null);
if (strayA !== strayB) bad.push("stray_cells");
if (bad.length) {
mismatched++;
if (examples.length < 6) {
examples.push({ tid, variant, bad,
py: Object.fromEntries(bad.map((f) => [f, f === "stray_cells" ? strayA : rec[f]])),
js: Object.fromEntries(bad.map((f) => [f, f === "stray_cells" ? strayB : got[f]])) });
}
}
}
}
console.log(`replayed ${checked} recorded replies across ${Object.keys(data.tasks).length} tasks`);
console.log(`mismatches: ${mismatched}`);
for (const e of examples) {
console.log(` ${e.tid} / ${e.variant}: ${e.bad.join(", ")}`);
console.log(` python: ${JSON.stringify(e.py)}`);
console.log(` js : ${JSON.stringify(e.js)}`);
}
if (mismatched) {
console.log("\nFAIL, the JS scorer does not reproduce the Python verdicts.");
process.exit(1);
}
console.log("\nPASS, the JS scorer reproduces every recorded Python verdict.");
// --- adversarial cross-check -------------------------------------------------
// The 384 replays above only cover replies these four models actually produced.
// A visitor can type things they never did: fenced blocks, duplicated objects,
// float and null answers, lowercase cell refs, no-op edits, truncated JSON.
// These cases were scored by the real Python scorer and the verdicts recorded;
// the port must match those too.
const cases = JSON.parse(readFileSync(new URL("./crosscheck_cases.json", import.meta.url)));
const pyRef = JSON.parse(readFileSync(new URL("./crosscheck_python.json", import.meta.url)));
let xChecked = 0, xBad = 0;
for (const tid of pyRef.task_ids) {
const task = data.tasks[tid];
cases.forEach((raw, i) => {
const got = judge(task, parseResponse(raw));
const py = pyRef.verdicts[tid][i];
xChecked++;
const bad = [...FIELDS, "stray_cells"].filter((f) => {
const a = py[f] === undefined ? null : py[f];
const b = got[f] === undefined ? null : got[f];
return f === "stray_cells"
? JSON.stringify(a ?? null) !== JSON.stringify(b ?? null)
: a !== b;
});
if (bad.length) {
xBad++;
if (xBad <= 8) {
console.log(` XCHECK ${tid} case#${i} ${bad.join(", ")}`);
console.log(` raw : ${JSON.stringify(raw).slice(0, 90)}`);
console.log(` python: ${JSON.stringify(Object.fromEntries(bad.map(f => [f, py[f] ?? null])))}`);
console.log(` js : ${JSON.stringify(Object.fromEntries(bad.map(f => [f, got[f] ?? null])))}`);
}
}
});
}
console.log(`\nadversarial cross-check: ${xChecked} cases, ${xBad} mismatches`);
if (xBad) {
console.log("FAIL, the port diverges on hand-written replies.");
process.exit(1);
}
console.log("PASS, the port matches Python on adversarial input too.");