Inkling-2-Mini-Fast-Browser-Use / benchmark /verify_browser_timings.py
husky8's picture
Publish the ordered browser film, verified timings and standalone checkpoint links
a55a3c9 verified
Raw History Blame Contribute Delete
3.56 kB
"""Check every full-task speed ratio and token count against the published episode timings."""
import argparse
import json
import math
from pathlib import Path
from zipfile import ZipFile
def verify(archive_path, pairs_path):
def check(condition, message):
if not condition:
raise ValueError(message)
pairs=json.loads(pairs_path.read_text())['pairs']
with ZipFile(archive_path) as archive:
names=[n for n in archive.namelist() if n.startswith('timing/')]
episodes={}
for name in names:
event=json.loads(archive.read(name))
grade=json.loads(archive.read(name.replace('timing/','receipts/',1)))
check(event['seconds']==grade['elapsed_s'] and event['score']==grade['score'],f'{name}: grade and timing disagree')
times=[s['start_s'] for s in event['steps']]+[event['seconds']]
check(bool(event['steps']) and all(math.isfinite(t) and t>=0 for t in times),f'{name}: invalid timestamps')
for start,end,s in zip(times,times[1:],event['steps']):
check(0<=s['inference_s']<=end-start+1e-6 and 0<=s['thinking']<=s['generated'],f'{name}: invalid step duration or token count')
digest=event['receipt_sha256']
check(digest not in episodes or episodes[digest]==event,f'{name}: conflicting episode identity')
episodes[digest]=event
used=set()
for p in pairs:
original,edited=(episodes[p[a]['receipt_sha256']] for a in ('original','edited'))
label=f"WebArena {p['task']}, {p['checkpoint']}, repeat {p['repeat']}"
check(original['steps'][0]['view_sha256']==edited['steps'][0]['view_sha256'],f'{label}: different initial observations')
for arm,event in [('original',original),('edited',edited)]:
result=p[arm]
check((p['task'],arm,result['seconds'],result['score'],result['shard_sha256'])==(event['task'],event['arm'],event['seconds'],event['score'],event['shard_sha256']),f'{label}: wrong {arm} episode')
check([s['action'] for s in event['steps']]==result['commands'],f'{label}: different {arm} actions')
check(sum(s['generated'] for s in event['steps'])==result['generated'],f'{label}: different {arm} token count')
check(sum(s['thinking'] for s in event['steps'])==result['thinking'],f'{label}: different {arm} reasoning token count')
check(math.isclose(sum(s['inference_s'] for s in event['steps']),result['inference_seconds'],rel_tol=1e-12),f'{label}: different {arm} inference time')
used.add(event['receipt_sha256'])
if p['comparable'] and original['score']==edited['score']==1:
check(edited['seconds']>0 and isinstance(p['ratio'],(int,float)) and math.isclose(original['seconds']/edited['seconds'],p['ratio'],rel_tol=1e-12),f'{label}: speed ratio disagrees with the clock')
else:
check(p['ratio'] is None,f'{label}: speed ratio on an unsuccessful or incomparable pair')
check(bool(pairs) and used==set(episodes) and len(names)==2*len(pairs),'Incomplete timing archive')
return len(pairs),len(names)
if __name__=='__main__':
parser=argparse.ArgumentParser(description=__doc__)
parser.add_argument('archive',type=Path)
parser.add_argument('pairs',type=Path)
args=parser.parse_args()
count,total=verify(args.archive,args.pairs)
print(f'Verified {count} paired comparisons against {total} recorded episode timings.')