OneAlign / web /scripts /replay-decisions.mjs
loveaa's picture
Deploy Home Pulse: grounded setup, isolated workspaces and shared HF visual inference
90d354b verified
Raw History Blame Contribute Delete
2.48 kB
import fs from 'node:fs'
import path from 'node:path'
import {performance} from 'node:perf_hooks'
import {ContractEngine} from '../server/contract-engine.mjs'
const out=path.resolve('../output/kimi-jev-evaluation-20261013')
const correctedReferences=JSON.parse(fs.readFileSync(path.join(out,'metadata-corrections.json'),'utf8')).planReferences
const records=[],checks=[]
const folders=fs.readdirSync(out).filter(n=>n==='runs'||n.startsWith('runs-'))
for(const folder of folders)for(const file of fs.readdirSync(path.join(out,folder)).filter(n=>/^[GH]\d+\.json$/.test(n))){
const run=JSON.parse(fs.readFileSync(path.join(out,folder,file),'utf8'));if(run.status!=='complete')continue
const plan=JSON.parse(fs.readFileSync(path.join(out,correctedReferences[folder+'/'+file]||run.planFile),'utf8')).plan
const engine=new ContractEngine(plan,{cameraId:run.clipId,sourceId:run.sourceId})
const base=Date.parse(run.baseTime)/1000
const signals=Object.fromEntries(run.simulatedSignals.map(k=>[k,{value:true,observedAt:base-1,validUntil:base+86400,source:'explicit_test_injection'}]))
for(const observation of run.observations){
const answers=Object.fromEntries(observation.predictions.map(p=>[p.observer,p.answer||{choice:null,abstained:true}]))
const start=performance.now()
engine.observe({...observation.trace,cameraId:run.clipId,sourceId:run.sourceId,quality:run.groundingPassed?'usable':'unregistered_target'},answers,{signals,manual:plan.rules.every(r=>r.mode==='now')})
records.push({run:folder+'/'+file,t:observation.t,engineMs:+(performance.now()-start).toFixed(4)})
}
if(run.summaries.length)engine.finalize(run.summaries[0].at)
const same=JSON.stringify(engine.notices)===JSON.stringify(run.runtime.notices)
checks.push({run:folder+'/'+file,sameNotices:same,notices:engine.notices.length})
}
const values=records.map(r=>r.engineMs).sort((a,b)=>a-b),pct=p=>values[Math.ceil(values.length*p)-1]
const result={method:'Single serial CPU replay of saved visual outputs; no model calls and no network time included.',decisions:records.length,runs:checks.length,matchedRuns:checks.filter(c=>c.sameNotices).length,p50Ms:pct(.5),p95Ms:pct(.95),maximumMs:values.at(-1),checks,records}
fs.writeFileSync(path.join(out,'decision-replay.json'),JSON.stringify(result,null,2))
console.log(JSON.stringify({decisions:result.decisions,runs:result.runs,matched:result.matchedRuns,p50Ms:result.p50Ms,p95Ms:result.p95Ms}))
if(result.matchedRuns!==result.runs)process.exitCode=1