File size: 6,994 Bytes
ad7798f
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
"""Validate feature-cache output preservation through the deployed Gradio API."""
from __future__ import annotations

import argparse
import concurrent.futures
from datetime import datetime, timezone
import hashlib
import json
from pathlib import Path
import threading

from gradio_client import Client, handle_file
from huggingface_hub import HfApi
from PIL import Image

BASE = Path(__file__).resolve().parent
ASSETS = BASE.parents[1] / 'visual-product-matching' / 'assets'
REPO = 'Ypeng12/visual-product-matching'
PAIRS = [
    ('identical_bag', 'patterned_bag.jpg', 'patterned_bag.jpg'),
    ('alternate_bag', 'patterned_bag.jpg', 'patterned_bag_alternate.jpg'),
    ('bag_vs_sneaker', 'patterned_bag.jpg', 'sneaker.jpg'),
]


def fingerprint(path):
    with Image.open(path) as image:
        rgb = image.convert('RGB')
        return {'mode': 'RGB', 'size': list(rgb.size), 'sha256_pixels': hashlib.sha256(rgb.tobytes()).hexdigest()}


def compare(client, left, right):
    result = client.predict(handle_file(str(left)), handle_file(str(right)),
                            .75, 75, 40, .30, .30, .20, .20, api_name='/compare')
    return {'diagnostics': result[5], 'matches_rgb': fingerprint(result[2]), 'heatmap_rgb': fingerprint(result[3])}


def semantic(row):
    data = row['diagnostics']
    selected = {name: data[name] for name in ('score', 'decision', 'threshold', 'scores', 'weights',
                 'hashes', 'ssim_raw', 'delta_e', 'settings', 'warnings', 'images')}
    selected['features'] = {key: value for key, value in data['features'].items()
                            if key not in {'cache_hits', 'cache_size', 'cache_capacity', 'feature_extraction_ms', 'flann_ms'}}
    return selected


def main():
    parser = argparse.ArgumentParser()
    parser.add_argument('--phase', choices=['baseline', 'after'], required=True)
    parser.add_argument('--expected-sha')
    args = parser.parse_args()
    destination = BASE / f'cache_validation_{args.phase}.json'
    if destination.exists():
        raise SystemExit(f'Preserving existing artifact: {destination.name}')
    info = HfApi().space_info(REPO)
    if args.expected_sha and info.sha != args.expected_sha:
        raise SystemExit(f'Revision mismatch: {info.sha}')
    report = {'phase': args.phase, 'revision': info.sha, 'started_utc': datetime.now(timezone.utc).isoformat(),
              'method': 'Deployed Gradio API with explicit original settings. Output hashes use decoded RGB pixels, not encoded PNG files.',
              'examples': {}, 'checks': [], 'passed': False}

    def check(name, condition, evidence=None):
        report['checks'].append({'name': name, 'passed': bool(condition), 'evidence': evidence})
        if not condition:
            raise AssertionError(f'{name}: {evidence}')

    try:
        client = Client(REPO, verbose=False)
        if args.phase == 'baseline':
            for name, left, right in PAIRS:
                report['examples'][name] = compare(client, ASSETS / left, ASSETS / right)
                print(f'Captured baseline {name}', flush=True)
        else:
            baseline = json.loads((BASE / 'cache_validation_baseline.json').read_text())
            report['baseline_revision'] = baseline['revision']
            for name, left, right in PAIRS:
                first = compare(client, ASSETS / left, ASSETS / right)
                warm = compare(client, ASSETS / left, ASSETS / right)
                report['examples'][name] = {'first': first, 'warm': warm}
                for stage, row in [('first', first), ('warm', warm)]:
                    check(f'{name}_{stage}_semantics_unchanged', semantic(row) == semantic(baseline['examples'][name]))
                    for output in ('matches_rgb', 'heatmap_rgb'):
                        check(f'{name}_{stage}_{output}_unchanged', row[output] == baseline['examples'][name][output], row[output])
                check(f'{name}_warm_both_hits', warm['diagnostics']['features']['cache_hits'] == {'a': True, 'b': True})
            check('first_asset_was_cold', report['examples']['identical_bag']['first']['diagnostics']['features']['cache_hits']['a'] is False)

            # These dataset images remain private local inputs. Only diagnostics
            # and identifiers are retained in this report; no outputs are copied.
            pairs = [json.loads(line) for line in (BASE / 'pairs.jsonl').read_text().splitlines()]
            eviction = [pair for pair in pairs if pair['split'] == 'calibration' and pair['pair_type'] == 'same_product'][:35]
            check('eviction_has_70_distinct_input_paths', len({pair[key] for pair in eviction for key in ('left_image', 'right_image')}) == 70)
            local = threading.local()

            def evict(pair):
                if not hasattr(local, 'client'):
                    local.client = Client(REPO, verbose=False, download_files=False)
                paths = []
                for key in ('left_image', 'right_image'):
                    parts = Path(pair[key]).parts
                    paths.append(BASE.joinpath(*parts[parts.index('.cache'):]))
                result = local.client.predict(handle_file(str(paths[0])), handle_file(str(paths[1])),
                                               .75, 75, 40, .30, .30, .20, .20, api_name='/compare')
                return {'pair_id': pair['pair_id'], 'diagnostics': result[5]}

            with concurrent.futures.ThreadPoolExecutor(max_workers=2) as pool:
                report['eviction_calls'] = list(pool.map(evict, eviction))
            sizes = [row['diagnostics']['features']['cache_size'] for row in report['eviction_calls']]
            check('cache_never_exceeds_64', max(sizes) <= 64, {'min': min(sizes), 'max': max(sizes)})
            check('cache_reaches_capacity', max(sizes) == 64)
            for name, left, right in PAIRS:
                row = compare(client, ASSETS / left, ASSETS / right)
                report['examples'][name]['after_eviction'] = row
                check(f'{name}_after_eviction_semantics_unchanged', semantic(row) == semantic(baseline['examples'][name]))
                for output in ('matches_rgb', 'heatmap_rgb'):
                    check(f'{name}_after_eviction_{output}_unchanged', row[output] == baseline['examples'][name][output])
            check('earliest_asset_evicted', report['examples']['identical_bag']['after_eviction']['diagnostics']['features']['cache_hits']['a'] is False)
        check('revision_unchanged_during_run', HfApi().space_info(REPO).sha == info.sha)
        report['passed'] = True
    except Exception as exc:
        report['error'] = f'{type(exc).__name__}: {exc}'
        raise
    finally:
        report['finished_utc'] = datetime.now(timezone.utc).isoformat()
        destination.write_text(json.dumps(report, indent=2) + '\n')
        print(json.dumps({'artifact': destination.name, 'passed': report['passed'], 'checks': len(report['checks'])}), flush=True)


if __name__ == '__main__':
    main()