visual-product-matching / evaluation /compare_runtime.py
Ypeng12's picture
Publish verified cache speedup, pixel-equivalence checks, UI results and Chinese evaluation
23cdd96 verified
Raw History Blame Contribute Delete
1.81 kB
"""Verify numerical equivalence before reporting repeated-image speed."""
import json
from pathlib import Path
BASE=Path(__file__).resolve().parent
def read(name):return [json.loads(line) for line in (BASE/f'runtime_{name}.jsonl').read_text().splitlines()]
before=read('before');after=read('after')
assert len(before)==len(after)==36
for a,b in zip(before,after):
assert (a['pair_id'],a['round'])==(b['pair_id'],b['round'])
for key in ['scores','score','decision','hashes','ssim_raw','delta_e','weights','warnings','threshold']:
assert a['diagnostics'][key]==b['diagnostics'][key],(a['pair_id'],key)
for key,value in a['diagnostics']['features'].items():
assert value==b['diagnostics']['features'][key],(a['pair_id'],key)
assert all(not r['diagnostics']['features']['cache_hits']['a'] and not r['diagnostics']['features']['cache_hits']['b'] for r in after[:12]),'First pass was not cold'
assert all(r['diagnostics']['features']['cache_hits']['a'] and r['diagnostics']['features']['cache_hits']['b'] for r in after[12:]),'Repeat pass missed the cache'
b=json.loads((BASE/'runtime_before.summary.json').read_text())
a=json.loads((BASE/'runtime_after.summary.json').read_text())
bg=b['groups']['repeat_passes'];ag=a['groups']['repeat_passes']
result={'before_revision':b['revision'],'after_revision':a['revision'],'comparisons_verified':36,
'cold_calls_verified':12,'warm_calls_verified':24,'before':bg,'after':ag,
'server_p50_reduction_pct':100*(1-ag['server_ms']['p50']/bg['server_ms']['p50']),
'scope':'Repeated images only; before/after sequential deployments on shared CPU Basic, not randomized A/B. First pass separate in source summaries.'}
(BASE/'runtime_comparison.json').write_text(json.dumps(result,indent=2)+'\n');print(json.dumps(result,indent=2))