visual-product-matching / evaluation /runtime_benchmark.py
Ypeng12's picture
Add evaluated calibration preset, bounded SIFT cache, and reproducible 360-pair results
ad7798f verified
Raw History Blame Contribute Delete
2.4 kB
"""Compare repeated-image runtime on the deployed Space, preserving scores."""
import argparse, datetime, json, time
from pathlib import Path
from gradio_client import Client, handle_file
from huggingface_hub import HfApi
import numpy as np
BASE=Path(__file__).resolve().parent
parser=argparse.ArgumentParser();parser.add_argument('--label',required=True);args=parser.parse_args()
selected=[json.loads(line) for line in (BASE/'pairs.jsonl').read_text().splitlines() if line.strip()]
selected=[r for r in selected if r['split']=='calibration' and r['pair_type']=='same_product'][::5]
assert len(selected)==12
repo='Ypeng12/visual-product-matching';info=HfApi().space_info(repo)
client=Client(repo,verbose=False,download_files=False)
def path(value):
parts=Path(value).parts;return BASE.joinpath(*parts[parts.index('.cache'):])
out=BASE/f'runtime_{args.label}.jsonl'
if out.exists():raise SystemExit('Preserve existing run; use another label.')
rows=[]
with out.open('w') as stream:
for round_index in range(3):
for pair in selected:
start=time.perf_counter()
result=client.predict(handle_file(str(path(pair['left_image']))),handle_file(str(path(pair['right_image']))),
.75,75,40,.30,.30,.20,.20,api_name='/compare')
row={'pair_id':pair['pair_id'],'round':round_index,'client_ms':(time.perf_counter()-start)*1000,
'diagnostics':result[5],'revision':info.sha}
rows.append(row);stream.write(json.dumps(row)+'\n');stream.flush()
print(f'{args.label}: round {round_index+1}/3 finished',flush=True)
summary={'revision':info.sha,'hardware':str(info.runtime.hardware),'created_utc':datetime.datetime.now(datetime.timezone.utc).isoformat(),
'method':'12 calibration positive pairs, one per category; three sequential rounds, fixed original settings; no generated-file downloads.',
'groups':{}}
for group,items in [('first_pass',rows[:12]),('repeat_passes',rows[12:])]:
summary['groups'][group]={'n':len(items)}
for name,values in [('server_ms',[r['diagnostics']['latency_ms'] for r in items]),('client_ms',[r['client_ms'] for r in items])]:
summary['groups'][group][name]={'p50':float(np.percentile(values,50)),'p95':float(np.percentile(values,95))}
out.with_suffix('.summary.json').write_text(json.dumps(summary,indent=2)+'\n');print(json.dumps(summary,indent=2))