visual-product-matching / evaluation /cache_validation.py
Ypeng12's picture
Add evaluated calibration preset, bounded SIFT cache, and reproducible 360-pair results
ad7798f verified
Raw History Blame Contribute Delete
6.99 kB
"""Validate feature-cache output preservation through the deployed Gradio API."""
from __future__ import annotations
import argparse
import concurrent.futures
from datetime import datetime, timezone
import hashlib
import json
from pathlib import Path
import threading
from gradio_client import Client, handle_file
from huggingface_hub import HfApi
from PIL import Image
BASE = Path(__file__).resolve().parent
ASSETS = BASE.parents[1] / 'visual-product-matching' / 'assets'
REPO = 'Ypeng12/visual-product-matching'
PAIRS = [
('identical_bag', 'patterned_bag.jpg', 'patterned_bag.jpg'),
('alternate_bag', 'patterned_bag.jpg', 'patterned_bag_alternate.jpg'),
('bag_vs_sneaker', 'patterned_bag.jpg', 'sneaker.jpg'),
]
def fingerprint(path):
with Image.open(path) as image:
rgb = image.convert('RGB')
return {'mode': 'RGB', 'size': list(rgb.size), 'sha256_pixels': hashlib.sha256(rgb.tobytes()).hexdigest()}
def compare(client, left, right):
result = client.predict(handle_file(str(left)), handle_file(str(right)),
.75, 75, 40, .30, .30, .20, .20, api_name='/compare')
return {'diagnostics': result[5], 'matches_rgb': fingerprint(result[2]), 'heatmap_rgb': fingerprint(result[3])}
def semantic(row):
data = row['diagnostics']
selected = {name: data[name] for name in ('score', 'decision', 'threshold', 'scores', 'weights',
'hashes', 'ssim_raw', 'delta_e', 'settings', 'warnings', 'images')}
selected['features'] = {key: value for key, value in data['features'].items()
if key not in {'cache_hits', 'cache_size', 'cache_capacity', 'feature_extraction_ms', 'flann_ms'}}
return selected
def main():
parser = argparse.ArgumentParser()
parser.add_argument('--phase', choices=['baseline', 'after'], required=True)
parser.add_argument('--expected-sha')
args = parser.parse_args()
destination = BASE / f'cache_validation_{args.phase}.json'
if destination.exists():
raise SystemExit(f'Preserving existing artifact: {destination.name}')
info = HfApi().space_info(REPO)
if args.expected_sha and info.sha != args.expected_sha:
raise SystemExit(f'Revision mismatch: {info.sha}')
report = {'phase': args.phase, 'revision': info.sha, 'started_utc': datetime.now(timezone.utc).isoformat(),
'method': 'Deployed Gradio API with explicit original settings. Output hashes use decoded RGB pixels, not encoded PNG files.',
'examples': {}, 'checks': [], 'passed': False}
def check(name, condition, evidence=None):
report['checks'].append({'name': name, 'passed': bool(condition), 'evidence': evidence})
if not condition:
raise AssertionError(f'{name}: {evidence}')
try:
client = Client(REPO, verbose=False)
if args.phase == 'baseline':
for name, left, right in PAIRS:
report['examples'][name] = compare(client, ASSETS / left, ASSETS / right)
print(f'Captured baseline {name}', flush=True)
else:
baseline = json.loads((BASE / 'cache_validation_baseline.json').read_text())
report['baseline_revision'] = baseline['revision']
for name, left, right in PAIRS:
first = compare(client, ASSETS / left, ASSETS / right)
warm = compare(client, ASSETS / left, ASSETS / right)
report['examples'][name] = {'first': first, 'warm': warm}
for stage, row in [('first', first), ('warm', warm)]:
check(f'{name}_{stage}_semantics_unchanged', semantic(row) == semantic(baseline['examples'][name]))
for output in ('matches_rgb', 'heatmap_rgb'):
check(f'{name}_{stage}_{output}_unchanged', row[output] == baseline['examples'][name][output], row[output])
check(f'{name}_warm_both_hits', warm['diagnostics']['features']['cache_hits'] == {'a': True, 'b': True})
check('first_asset_was_cold', report['examples']['identical_bag']['first']['diagnostics']['features']['cache_hits']['a'] is False)
# These dataset images remain private local inputs. Only diagnostics
# and identifiers are retained in this report; no outputs are copied.
pairs = [json.loads(line) for line in (BASE / 'pairs.jsonl').read_text().splitlines()]
eviction = [pair for pair in pairs if pair['split'] == 'calibration' and pair['pair_type'] == 'same_product'][:35]
check('eviction_has_70_distinct_input_paths', len({pair[key] for pair in eviction for key in ('left_image', 'right_image')}) == 70)
local = threading.local()
def evict(pair):
if not hasattr(local, 'client'):
local.client = Client(REPO, verbose=False, download_files=False)
paths = []
for key in ('left_image', 'right_image'):
parts = Path(pair[key]).parts
paths.append(BASE.joinpath(*parts[parts.index('.cache'):]))
result = local.client.predict(handle_file(str(paths[0])), handle_file(str(paths[1])),
.75, 75, 40, .30, .30, .20, .20, api_name='/compare')
return {'pair_id': pair['pair_id'], 'diagnostics': result[5]}
with concurrent.futures.ThreadPoolExecutor(max_workers=2) as pool:
report['eviction_calls'] = list(pool.map(evict, eviction))
sizes = [row['diagnostics']['features']['cache_size'] for row in report['eviction_calls']]
check('cache_never_exceeds_64', max(sizes) <= 64, {'min': min(sizes), 'max': max(sizes)})
check('cache_reaches_capacity', max(sizes) == 64)
for name, left, right in PAIRS:
row = compare(client, ASSETS / left, ASSETS / right)
report['examples'][name]['after_eviction'] = row
check(f'{name}_after_eviction_semantics_unchanged', semantic(row) == semantic(baseline['examples'][name]))
for output in ('matches_rgb', 'heatmap_rgb'):
check(f'{name}_after_eviction_{output}_unchanged', row[output] == baseline['examples'][name][output])
check('earliest_asset_evicted', report['examples']['identical_bag']['after_eviction']['diagnostics']['features']['cache_hits']['a'] is False)
check('revision_unchanged_during_run', HfApi().space_info(REPO).sha == info.sha)
report['passed'] = True
except Exception as exc:
report['error'] = f'{type(exc).__name__}: {exc}'
raise
finally:
report['finished_utc'] = datetime.now(timezone.utc).isoformat()
destination.write_text(json.dumps(report, indent=2) + '\n')
print(json.dumps({'artifact': destination.name, 'passed': report['passed'], 'checks': len(report['checks'])}), flush=True)
if __name__ == '__main__':
main()