OneAlign / web /scripts /evaluate-atoms.mjs
loveaa's picture
Deploy Home Pulse: grounded setup, isolated workspaces and shared HF visual inference
90d354b verified
Raw History Blame Contribute Delete
1.79 kB
import fs from 'node:fs'
import path from 'node:path'
import {performance} from 'node:perf_hooks'
const out=path.resolve('../output/kimi-jev-evaluation-20261013')
const truth=JSON.parse(fs.readFileSync(path.join(out,'atomic-ground-truth.json'),'utf8'))
const plans=Object.fromEntries([['human','A01'],['pet','A02']].map(([type,id])=>[type,JSON.parse(fs.readFileSync(path.join(out,'plans',id+'.json'),'utf8')).plan]))
const file=path.join(out,'atomic-predictions.jsonl'),existing=fs.existsSync(file)?fs.readFileSync(file,'utf8').trim().split('\n').filter(Boolean).map(JSON.parse):[]
const seen=new Set(existing.map(r=>r.file))
const questions=Object.fromEntries(Object.entries(plans).map(([type,p])=>[type,{type:'choice',instructions:p.observers[0].question_en,criteria:p.observers[0].options}]))
let count=0
for(const row of truth){
if(seen.has(row.file))continue
const begin=performance.now(),image='data:image/jpeg;base64,'+fs.readFileSync(path.join(out,row.file)).toString('base64')
try{
const res=await fetch('http://127.0.0.1:8765/v1/systemone',{method:'POST',headers:{'Content-Type':'application/json'},body:JSON.stringify({state:{source:'current camera view'},questions,images:[image]}),signal:AbortSignal.timeout(120000)})
if(!res.ok)throw new Error('HTTP '+res.status)
const data=await res.json()
const result={file:row.file,clipId:row.clipId,sourceId:row.sourceId,t:row.t,answers:data.answers,latencyMs:Math.round(performance.now()-begin),modelMs:data.usage?.total_ms}
fs.appendFileSync(file,JSON.stringify(result)+'\n')
}catch(e){fs.appendFileSync(file,JSON.stringify({file:row.file,clipId:row.clipId,t:row.t,error:e.message})+'\n')}
count++;if(count%20===0)console.log('ATOMS '+(existing.length+count)+' / '+truth.length)
}
console.log('ATOMIC CHECK COMPLETE')