Spaces:
Sleeping
Sleeping
Download evaluation/cache_validation.py from Ypeng12/visual-product-matching: direct link, hf CLI and curl.
- Browser
- Download file 6.99 kB
-
https://huggingface.co/spaces/Ypeng12/visual-product-matching/resolve/main/evaluation/cache_validation.py
- Command line
-
hf download hf://spaces/Ypeng12/visual-product-matching/evaluation/cache_validation.py
-
curl -L -o cache_validation.py https://huggingface.co/spaces/Ypeng12/visual-product-matching/resolve/main/evaluation/cache_validation.py
6.99 kB
| """Validate feature-cache output preservation through the deployed Gradio API.""" | |
| from __future__ import annotations | |
| import argparse | |
| import concurrent.futures | |
| from datetime import datetime, timezone | |
| import hashlib | |
| import json | |
| from pathlib import Path | |
| import threading | |
| from gradio_client import Client, handle_file | |
| from huggingface_hub import HfApi | |
| from PIL import Image | |
| BASE = Path(__file__).resolve().parent | |
| ASSETS = BASE.parents[1] / 'visual-product-matching' / 'assets' | |
| REPO = 'Ypeng12/visual-product-matching' | |
| PAIRS = [ | |
| ('identical_bag', 'patterned_bag.jpg', 'patterned_bag.jpg'), | |
| ('alternate_bag', 'patterned_bag.jpg', 'patterned_bag_alternate.jpg'), | |
| ('bag_vs_sneaker', 'patterned_bag.jpg', 'sneaker.jpg'), | |
| ] | |
| def fingerprint(path): | |
| with Image.open(path) as image: | |
| rgb = image.convert('RGB') | |
| return {'mode': 'RGB', 'size': list(rgb.size), 'sha256_pixels': hashlib.sha256(rgb.tobytes()).hexdigest()} | |
| def compare(client, left, right): | |
| result = client.predict(handle_file(str(left)), handle_file(str(right)), | |
| .75, 75, 40, .30, .30, .20, .20, api_name='/compare') | |
| return {'diagnostics': result[5], 'matches_rgb': fingerprint(result[2]), 'heatmap_rgb': fingerprint(result[3])} | |
| def semantic(row): | |
| data = row['diagnostics'] | |
| selected = {name: data[name] for name in ('score', 'decision', 'threshold', 'scores', 'weights', | |
| 'hashes', 'ssim_raw', 'delta_e', 'settings', 'warnings', 'images')} | |
| selected['features'] = {key: value for key, value in data['features'].items() | |
| if key not in {'cache_hits', 'cache_size', 'cache_capacity', 'feature_extraction_ms', 'flann_ms'}} | |
| return selected | |
| def main(): | |
| parser = argparse.ArgumentParser() | |
| parser.add_argument('--phase', choices=['baseline', 'after'], required=True) | |
| parser.add_argument('--expected-sha') | |
| args = parser.parse_args() | |
| destination = BASE / f'cache_validation_{args.phase}.json' | |
| if destination.exists(): | |
| raise SystemExit(f'Preserving existing artifact: {destination.name}') | |
| info = HfApi().space_info(REPO) | |
| if args.expected_sha and info.sha != args.expected_sha: | |
| raise SystemExit(f'Revision mismatch: {info.sha}') | |
| report = {'phase': args.phase, 'revision': info.sha, 'started_utc': datetime.now(timezone.utc).isoformat(), | |
| 'method': 'Deployed Gradio API with explicit original settings. Output hashes use decoded RGB pixels, not encoded PNG files.', | |
| 'examples': {}, 'checks': [], 'passed': False} | |
| def check(name, condition, evidence=None): | |
| report['checks'].append({'name': name, 'passed': bool(condition), 'evidence': evidence}) | |
| if not condition: | |
| raise AssertionError(f'{name}: {evidence}') | |
| try: | |
| client = Client(REPO, verbose=False) | |
| if args.phase == 'baseline': | |
| for name, left, right in PAIRS: | |
| report['examples'][name] = compare(client, ASSETS / left, ASSETS / right) | |
| print(f'Captured baseline {name}', flush=True) | |
| else: | |
| baseline = json.loads((BASE / 'cache_validation_baseline.json').read_text()) | |
| report['baseline_revision'] = baseline['revision'] | |
| for name, left, right in PAIRS: | |
| first = compare(client, ASSETS / left, ASSETS / right) | |
| warm = compare(client, ASSETS / left, ASSETS / right) | |
| report['examples'][name] = {'first': first, 'warm': warm} | |
| for stage, row in [('first', first), ('warm', warm)]: | |
| check(f'{name}_{stage}_semantics_unchanged', semantic(row) == semantic(baseline['examples'][name])) | |
| for output in ('matches_rgb', 'heatmap_rgb'): | |
| check(f'{name}_{stage}_{output}_unchanged', row[output] == baseline['examples'][name][output], row[output]) | |
| check(f'{name}_warm_both_hits', warm['diagnostics']['features']['cache_hits'] == {'a': True, 'b': True}) | |
| check('first_asset_was_cold', report['examples']['identical_bag']['first']['diagnostics']['features']['cache_hits']['a'] is False) | |
| # These dataset images remain private local inputs. Only diagnostics | |
| # and identifiers are retained in this report; no outputs are copied. | |
| pairs = [json.loads(line) for line in (BASE / 'pairs.jsonl').read_text().splitlines()] | |
| eviction = [pair for pair in pairs if pair['split'] == 'calibration' and pair['pair_type'] == 'same_product'][:35] | |
| check('eviction_has_70_distinct_input_paths', len({pair[key] for pair in eviction for key in ('left_image', 'right_image')}) == 70) | |
| local = threading.local() | |
| def evict(pair): | |
| if not hasattr(local, 'client'): | |
| local.client = Client(REPO, verbose=False, download_files=False) | |
| paths = [] | |
| for key in ('left_image', 'right_image'): | |
| parts = Path(pair[key]).parts | |
| paths.append(BASE.joinpath(*parts[parts.index('.cache'):])) | |
| result = local.client.predict(handle_file(str(paths[0])), handle_file(str(paths[1])), | |
| .75, 75, 40, .30, .30, .20, .20, api_name='/compare') | |
| return {'pair_id': pair['pair_id'], 'diagnostics': result[5]} | |
| with concurrent.futures.ThreadPoolExecutor(max_workers=2) as pool: | |
| report['eviction_calls'] = list(pool.map(evict, eviction)) | |
| sizes = [row['diagnostics']['features']['cache_size'] for row in report['eviction_calls']] | |
| check('cache_never_exceeds_64', max(sizes) <= 64, {'min': min(sizes), 'max': max(sizes)}) | |
| check('cache_reaches_capacity', max(sizes) == 64) | |
| for name, left, right in PAIRS: | |
| row = compare(client, ASSETS / left, ASSETS / right) | |
| report['examples'][name]['after_eviction'] = row | |
| check(f'{name}_after_eviction_semantics_unchanged', semantic(row) == semantic(baseline['examples'][name])) | |
| for output in ('matches_rgb', 'heatmap_rgb'): | |
| check(f'{name}_after_eviction_{output}_unchanged', row[output] == baseline['examples'][name][output]) | |
| check('earliest_asset_evicted', report['examples']['identical_bag']['after_eviction']['diagnostics']['features']['cache_hits']['a'] is False) | |
| check('revision_unchanged_during_run', HfApi().space_info(REPO).sha == info.sha) | |
| report['passed'] = True | |
| except Exception as exc: | |
| report['error'] = f'{type(exc).__name__}: {exc}' | |
| raise | |
| finally: | |
| report['finished_utc'] = datetime.now(timezone.utc).isoformat() | |
| destination.write_text(json.dumps(report, indent=2) + '\n') | |
| print(json.dumps({'artifact': destination.name, 'passed': report['passed'], 'checks': len(report['checks'])}), flush=True) | |
| if __name__ == '__main__': | |
| main() | |