import fs from 'node:fs' import path from 'node:path' import {performance} from 'node:perf_hooks' import {ContractEngine} from '../server/contract-engine.mjs' const out=path.resolve('../output/kimi-jev-evaluation-20261013') const correctedReferences=JSON.parse(fs.readFileSync(path.join(out,'metadata-corrections.json'),'utf8')).planReferences const records=[],checks=[] const folders=fs.readdirSync(out).filter(n=>n==='runs'||n.startsWith('runs-')) for(const folder of folders)for(const file of fs.readdirSync(path.join(out,folder)).filter(n=>/^[GH]\d+\.json$/.test(n))){ const run=JSON.parse(fs.readFileSync(path.join(out,folder,file),'utf8'));if(run.status!=='complete')continue const plan=JSON.parse(fs.readFileSync(path.join(out,correctedReferences[folder+'/'+file]||run.planFile),'utf8')).plan const engine=new ContractEngine(plan,{cameraId:run.clipId,sourceId:run.sourceId}) const base=Date.parse(run.baseTime)/1000 const signals=Object.fromEntries(run.simulatedSignals.map(k=>[k,{value:true,observedAt:base-1,validUntil:base+86400,source:'explicit_test_injection'}])) for(const observation of run.observations){ const answers=Object.fromEntries(observation.predictions.map(p=>[p.observer,p.answer||{choice:null,abstained:true}])) const start=performance.now() engine.observe({...observation.trace,cameraId:run.clipId,sourceId:run.sourceId,quality:run.groundingPassed?'usable':'unregistered_target'},answers,{signals,manual:plan.rules.every(r=>r.mode==='now')}) records.push({run:folder+'/'+file,t:observation.t,engineMs:+(performance.now()-start).toFixed(4)}) } if(run.summaries.length)engine.finalize(run.summaries[0].at) const same=JSON.stringify(engine.notices)===JSON.stringify(run.runtime.notices) checks.push({run:folder+'/'+file,sameNotices:same,notices:engine.notices.length}) } const values=records.map(r=>r.engineMs).sort((a,b)=>a-b),pct=p=>values[Math.ceil(values.length*p)-1] const result={method:'Single serial CPU replay of saved visual outputs; no model calls and no network time included.',decisions:records.length,runs:checks.length,matchedRuns:checks.filter(c=>c.sameNotices).length,p50Ms:pct(.5),p95Ms:pct(.95),maximumMs:values.at(-1),checks,records} fs.writeFileSync(path.join(out,'decision-replay.json'),JSON.stringify(result,null,2)) console.log(JSON.stringify({decisions:result.decisions,runs:result.runs,matched:result.matchedRuns,p50Ms:result.p50Ms,p95Ms:result.p95Ms})) if(result.matchedRuns!==result.runs)process.exitCode=1