Spaces:
Sleeping
Sleeping
Ruzzel Mendoza
Phase 5: rotation-aware global pre-filter + scoring refinements - global_prefilter_distance: query phash('0') vs ALL SIX stored transform-variant phashes (6 Hamming ops, no re-hashing); scan_database skips a row only when every variant exceeds GLOBAL_PREFILTER_MAX_DISTANCE (default 28, env-overridable) - pre-filter is performance-only: never sets or influences scores of rows that pass; skip count logged - scan_database/check_distance_between_serp now return true top-5 sorted by calibrated_confidence (primary score) with the full new field set per match (was: history of interim bests) - scoring refinement for rotate-and-repost: final = 0.3*T*transform_cap + 0.7*B*block_multiplier so block geometry can no longer zero out transform evidence (pure rotations: T~65, blocks 0 -> final ~20, flagged) - background-coincidence guard: transform resemblance without ANY block corroboration is voided unless the alignment is tight (avg dist <= STRICT/2) - fixes the matching-background false positive at the global level while preserving rotation/mirror/flip detection - regression fixture verified: legacy 96.48% (old bug) vs new 0.0; sparse-art exact copy stays 100.0 (no over-suppression); real-data baseline re-verified: 78/78 unrelated pairs score 0.0, prefilter dists 18-32 vs cutoff 28 (kept: conservative, true copies at <=10) - tests 5/6/7 added: sparse-art multiplier, original-bug regression, prefilter + scan_database
f83b9ab Download scripts/calibrate_baseline.py from thesis-sd3a/digital-art-api: direct link, hf CLI and curl.
- Browser
- Download file 13.6 kB
-
https://huggingface.co/spaces/thesis-sd3a/digital-art-api/resolve/main/scripts/calibrate_baseline.py
- Command line
-
hf download hf://spaces/thesis-sd3a/digital-art-api/scripts/calibrate_baseline.py
-
curl -L -o calibrate_baseline.py https://huggingface.co/spaces/thesis-sd3a/digital-art-api/resolve/main/scripts/calibrate_baseline.py
13.6 kB
| #!/usr/bin/env python | |
| """ | |
| Offline baseline calibration for the plagiarism confidence score. | |
| NOT wired into the API. Run manually (periodically, e.g. whenever the artwork | |
| catalog grows substantially): | |
| python scripts/calibrate_baseline.py | |
| python scripts/calibrate_baseline.py --sample 60 --max-pairs 1000 | |
| python scripts/calibrate_baseline.py --from-json scripts/hash_dump.json | |
| python scripts/calibrate_baseline.py --dump-json scripts/hash_dump.json | |
| What it does: | |
| 1. Fetches perceptual-hash rows of registered artworks from the database | |
| (lib.db_service.get_all_artwork_hashes) or loads a previously dumped | |
| JSON file. Registered artworks are assumed to be UNRELATED to each | |
| other, so cross-pairing them approximates the unrelated-pair baseline. | |
| 2. Runs the updated consensus pipeline (compare_hash_sets) over sampled | |
| cross-pairs and records the final_similarity distribution. The legacy | |
| single-minimum score is computed too, for before/after validation. | |
| 3. Persists the distribution (mean/stddev + percentile table) to | |
| config/calibration.json, which lib.phash loads at runtime to compute | |
| calibrated_confidence. | |
| Flagging threshold: | |
| plagiarism_threshold = 99th percentile of the unrelated-pair baseline | |
| distribution of the NEW consensus score, i.e. only ~1% of unrelated pairs | |
| are expected to score at or above it by chance. This replaces the previous | |
| uncalibrated 0-100 scale where arbitrary numbers (">70%") had no | |
| statistical meaning. | |
| Provisional constants (MIN_BLOCK_STDDEV, STRICT_MATCH_DISTANCE, | |
| GLOBAL_PREFILTER_MAX_DISTANCE) were chosen before this data existed - this | |
| script prints diagnostics relevant to each of them so they can be revisited; | |
| do not treat the defaults as final just because they are in code. | |
| """ | |
| import argparse | |
| import itertools | |
| import json | |
| import os | |
| import random | |
| import sys | |
| from datetime import datetime, timezone | |
| import numpy as np | |
| # Make the repository root importable when run as a plain script | |
| sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) | |
| from dotenv import load_dotenv # noqa: E402 | |
| load_dotenv() # Supabase credentials for local runs | |
| from lib.phash import ( # noqa: E402 | |
| GLOBAL_PREFILTER_MAX_DISTANCE, | |
| MAX_DISTANCE, | |
| STRICT_MATCH_DISTANCE, | |
| blocks_with_content, | |
| compare_blocks, | |
| compare_hash_sets, | |
| compare_transforms, | |
| global_prefilter_distance, | |
| weighted_distance, | |
| ) | |
| DEFAULT_OUTPUT = os.path.join( | |
| os.path.dirname(os.path.dirname(os.path.abspath(__file__))), | |
| "config", | |
| "calibration.json", | |
| ) | |
| PERCENTILE_POINTS = [0, 5, 25, 50, 75, 90, 95, 99, 99.9, 100] | |
| def fetch_artworks(sample=None, seed=0): | |
| """Fetch hash rows from the registered_arts table (network required).""" | |
| from lib.db_service import get_all_artwork_hashes | |
| artworks = get_all_artwork_hashes() | |
| if sample and len(artworks) > sample: | |
| rng = random.Random(seed) | |
| artworks = rng.sample(artworks, sample) | |
| return artworks | |
| def load_artworks(path): | |
| """Load hash rows from a JSON dump written by --dump-json.""" | |
| with open(path, "r", encoding="utf-8") as f: | |
| return json.load(f) | |
| def is_valid_hashset(hashes): | |
| """Skip rows missing the structure the comparison pipeline needs.""" | |
| return ( | |
| isinstance(hashes, dict) | |
| and isinstance(hashes.get("transforms"), dict) | |
| and isinstance(hashes.get("blocks"), dict) | |
| and len(hashes["transforms"]) > 0 | |
| and len(hashes["blocks"]) > 0 | |
| ) | |
| def min_dist_between_blocks(blocks1, blocks2): | |
| """Smallest weighted distance across eligible cross-pairs (diagnostic).""" | |
| eligible1 = blocks_with_content(blocks1) or list(blocks1.keys()) | |
| eligible2 = blocks_with_content(blocks2) or list(blocks2.keys()) | |
| best = float("inf") | |
| for b1 in eligible1: | |
| for b2 in eligible2: | |
| d = weighted_distance(blocks1[b1], blocks2[b2]) | |
| if d < best: | |
| best = d | |
| return best | |
| def distribution_stats(scores): | |
| arr = np.asarray(scores, dtype=np.float64) | |
| percentiles = { | |
| str(p): round(float(np.percentile(arr, p)), 4) for p in PERCENTILE_POINTS | |
| } | |
| return { | |
| "mean": round(float(np.mean(arr)), 4), | |
| "stddev": round(float(np.std(arr, ddof=1)) if len(arr) > 1 else 0.0, 4), | |
| "percentiles": percentiles, | |
| } | |
| def main(): | |
| parser = argparse.ArgumentParser( | |
| description="Offline baseline calibration for plagiarism confidence") | |
| parser.add_argument("--sample", type=int, default=None, | |
| help="sample at most N artworks from the DB (default: all)") | |
| parser.add_argument("--max-pairs", type=int, default=None, | |
| help="cap the number of cross-pairs evaluated") | |
| parser.add_argument("--seed", type=int, default=0, | |
| help="random seed for sampling (reproducibility)") | |
| parser.add_argument("--from-json", default=None, | |
| help="load hash rows from a JSON dump instead of the DB") | |
| parser.add_argument("--dump-json", default=None, | |
| help="also write the fetched hash rows to this JSON file") | |
| parser.add_argument("--output", default=DEFAULT_OUTPUT, | |
| help=f"calibration config destination (default: {DEFAULT_OUTPUT})") | |
| args = parser.parse_args() | |
| # ------------------------- fetch / load rows ------------------------- | |
| if args.from_json: | |
| artworks = load_artworks(args.from_json) | |
| print(f"Loaded {len(artworks)} artwork hash rows from {args.from_json}") | |
| else: | |
| artworks = fetch_artworks(sample=args.sample, seed=args.seed) | |
| print(f"Fetched {len(artworks)} artwork hash rows from the database") | |
| if args.dump_json: | |
| with open(args.dump_json, "w", encoding="utf-8") as f: | |
| json.dump(artworks, f) | |
| print(f"Dumped hash rows to {args.dump_json}") | |
| valid = [a for a in artworks if is_valid_hashset(a.get("perceptual_hash"))] | |
| skipped = len(artworks) - len(valid) | |
| if skipped: | |
| print(f"Skipped {skipped} rows with missing/incomplete hash payloads") | |
| if len(valid) < 2: | |
| print("Need at least 2 valid artwork hash rows to calibrate. Aborting.") | |
| sys.exit(1) | |
| # ----------------------------- cross-pair ---------------------------- | |
| rng = random.Random(args.seed) | |
| pairs = list(itertools.combinations(valid, 2)) | |
| if args.max_pairs and len(pairs) > args.max_pairs: | |
| pairs = rng.sample(pairs, args.max_pairs) | |
| print(f"Evaluating {len(pairs)} unrelated cross-pairs " | |
| f"({len(valid)} artworks, STRICT_MATCH_DISTANCE={STRICT_MATCH_DISTANCE})...") | |
| new_scores = [] | |
| legacy_scores = [] | |
| pairs_with_agreement = 0 | |
| pairs_with_transform_agreement = 0 | |
| pairs_low_content = 0 | |
| pairs_bg_suppressed = 0 | |
| best_block_dists = [] | |
| prefilter_dists = [] | |
| for i, (a, b) in enumerate(pairs, start=1): | |
| result = compare_hash_sets(a["perceptual_hash"], b["perceptual_hash"]) | |
| # final_similarity is the raw consensus score (calibration-independent) | |
| new_scores.append(result["final_similarity"]) | |
| legacy_scores.append(result["raw_similarity_legacy"]) | |
| if result["low_content_warning"]: | |
| pairs_low_content += 1 | |
| if result["background_coincidence_suppressed"]: | |
| pairs_bg_suppressed += 1 | |
| # diagnostics for the provisional constants | |
| block_result = compare_blocks(a["perceptual_hash"]["blocks"], | |
| b["perceptual_hash"]["blocks"]) | |
| if block_result["agreements"]: | |
| pairs_with_agreement += 1 | |
| transform_result = compare_transforms(a["perceptual_hash"]["transforms"], | |
| b["perceptual_hash"]["transforms"]) | |
| if transform_result["agreements"]: | |
| pairs_with_transform_agreement += 1 | |
| best_block_dists.append(min_dist_between_blocks( | |
| a["perceptual_hash"]["blocks"], b["perceptual_hash"]["blocks"])) | |
| prefilter_dists.append(global_prefilter_distance( | |
| a["perceptual_hash"], b["perceptual_hash"])) | |
| if i % 250 == 0 or i == len(pairs): | |
| print(f" {i}/{len(pairs)} pairs done") | |
| report_and_persist(new_scores, legacy_scores, pairs_with_agreement, | |
| pairs_with_transform_agreement, pairs_low_content, | |
| pairs_bg_suppressed, best_block_dists, | |
| prefilter_dists, len(valid), len(pairs), args.output) | |
| def report_and_persist(new_scores, legacy_scores, pairs_with_agreement, | |
| pairs_with_transform_agreement, pairs_low_content, | |
| pairs_bg_suppressed, best_block_dists, prefilter_dists, | |
| n_artworks, n_pairs, output_path): | |
| """Print the baseline report and write config/calibration.json.""" | |
| new_stats = distribution_stats(new_scores) | |
| legacy_stats = distribution_stats(legacy_scores) | |
| degenerate = new_stats["stddev"] == 0.0 | |
| if degenerate: | |
| # Degenerate baseline (e.g. every unrelated pair scores exactly 0 on | |
| # the current catalog - good separation, little data). A percentile | |
| # threshold would be 0 and meaningless, so place the threshold a | |
| # fixed margin above the best score any unrelated pair achieved. | |
| baseline_max = new_stats["percentiles"]["100"] | |
| threshold = min(100.0, round(baseline_max + 5.0, 4)) | |
| threshold_basis = ("degenerate baseline (stddev 0): best unrelated " | |
| "score + 5-point margin; switch to the 99th " | |
| "percentile once the catalog is large enough to " | |
| "produce a non-degenerate distribution") | |
| else: | |
| threshold = new_stats["percentiles"]["99"] | |
| threshold_basis = ("99th percentile of the unrelated-pair baseline " | |
| "distribution of the new consensus score; only ~1% " | |
| "of unrelated pairs are expected to reach it by " | |
| "chance") | |
| print("\n=== Unrelated-pair baseline: NEW consensus score (final_similarity) ===") | |
| print(json.dumps(new_stats, indent=2)) | |
| print("\n=== Unrelated-pair baseline: LEGACY single-minimum score (comparison) ===") | |
| print(json.dumps(legacy_stats, indent=2)) | |
| print(f"\nFlagging threshold (p99 of new baseline): raw score >= {threshold}") | |
| print(f"Pairs with >=1 agreeing block at STRICT_MATCH_DISTANCE=" | |
| f"{STRICT_MATCH_DISTANCE}: {pairs_with_agreement}/{n_pairs} " | |
| f"({100.0 * pairs_with_agreement / n_pairs:.1f}%)") | |
| print(f"Pairs with >=1 agreeing transform: " | |
| f"{pairs_with_transform_agreement}/{n_pairs} " | |
| f"({100.0 * pairs_with_transform_agreement / n_pairs:.1f}%)") | |
| print(f"Pairs voided by the background-coincidence guard: " | |
| f"{pairs_bg_suppressed}/{n_pairs}") | |
| print(f"Pairs that hit the low-content fallback: {pairs_low_content}/{n_pairs}") | |
| if best_block_dists: | |
| d = np.asarray(best_block_dists, dtype=np.float64) | |
| print(f"Best eligible-block distance between unrelated images: " | |
| f"median={np.median(d):.1f} p10={np.percentile(d, 10):.1f} " | |
| f"min={d.min():.1f} (max possible {MAX_DISTANCE})") | |
| if prefilter_dists: | |
| p = np.asarray(prefilter_dists, dtype=np.float64) | |
| print(f"Global pre-filter distance between unrelated images: " | |
| f"median={np.median(p):.1f} min={p.min():.1f} max={p.max():.1f} " | |
| f"(cutoff GLOBAL_PREFILTER_MAX_DISTANCE=" | |
| f"{GLOBAL_PREFILTER_MAX_DISTANCE}; rows above the cutoff on all " | |
| f"six variants are skipped)") | |
| print("\nRevisit MIN_BLOCK_STDDEV / STRICT_MATCH_DISTANCE / " | |
| "GLOBAL_PREFILTER_MAX_DISTANCE against these numbers before " | |
| "treating the defaults as final.") | |
| config = { | |
| "version": 1, | |
| "generated_at": datetime.now(timezone.utc).isoformat(), | |
| "n_artworks": n_artworks, | |
| "n_pairs": n_pairs, | |
| "method": ("cross-paired registered artworks (assumed unrelated); " | |
| "consensus pipeline final_similarity (entropy filter + " | |
| "agreement ratio + transform consistency multiplier)"), | |
| "mean": new_stats["mean"], | |
| "stddev": new_stats["stddev"], | |
| "percentiles": new_stats["percentiles"], | |
| "plagiarism_threshold": threshold, | |
| "threshold_basis": threshold_basis, | |
| "legacy_baseline": legacy_stats, | |
| "diagnostics": { | |
| "degenerate_baseline": degenerate, | |
| "pairs_with_block_agreement": pairs_with_agreement, | |
| "pairs_with_transform_agreement": pairs_with_transform_agreement, | |
| "pairs_background_coincidence_suppressed": pairs_bg_suppressed, | |
| "pairs_low_content_fallback": pairs_low_content, | |
| "strict_match_distance_used": STRICT_MATCH_DISTANCE, | |
| "global_prefilter_max_distance_used": GLOBAL_PREFILTER_MAX_DISTANCE, | |
| }, | |
| "notes": ("Consumed by lib.phash.calibrated_confidence. Regenerate " | |
| "when the artwork catalog changes substantially or when " | |
| "tunables change."), | |
| } | |
| os.makedirs(os.path.dirname(os.path.abspath(output_path)), exist_ok=True) | |
| with open(output_path, "w", encoding="utf-8") as f: | |
| json.dump(config, f, indent=2) | |
| print(f"\nCalibration config written to {output_path}") | |
| if __name__ == "__main__": | |
| main() | |