digital-art-api / scripts /calibrate_baseline.py
Ruzzel Mendoza
Phase 5: rotation-aware global pre-filter + scoring refinements - global_prefilter_distance: query phash('0') vs ALL SIX stored transform-variant phashes (6 Hamming ops, no re-hashing); scan_database skips a row only when every variant exceeds GLOBAL_PREFILTER_MAX_DISTANCE (default 28, env-overridable) - pre-filter is performance-only: never sets or influences scores of rows that pass; skip count logged - scan_database/check_distance_between_serp now return true top-5 sorted by calibrated_confidence (primary score) with the full new field set per match (was: history of interim bests) - scoring refinement for rotate-and-repost: final = 0.3*T*transform_cap + 0.7*B*block_multiplier so block geometry can no longer zero out transform evidence (pure rotations: T~65, blocks 0 -> final ~20, flagged) - background-coincidence guard: transform resemblance without ANY block corroboration is voided unless the alignment is tight (avg dist <= STRICT/2) - fixes the matching-background false positive at the global level while preserving rotation/mirror/flip detection - regression fixture verified: legacy 96.48% (old bug) vs new 0.0; sparse-art exact copy stays 100.0 (no over-suppression); real-data baseline re-verified: 78/78 unrelated pairs score 0.0, prefilter dists 18-32 vs cutoff 28 (kept: conservative, true copies at <=10) - tests 5/6/7 added: sparse-art multiplier, original-bug regression, prefilter + scan_database
f83b9ab
Raw History Blame Contribute Delete
13.6 kB
#!/usr/bin/env python
"""
Offline baseline calibration for the plagiarism confidence score.
NOT wired into the API. Run manually (periodically, e.g. whenever the artwork
catalog grows substantially):
python scripts/calibrate_baseline.py
python scripts/calibrate_baseline.py --sample 60 --max-pairs 1000
python scripts/calibrate_baseline.py --from-json scripts/hash_dump.json
python scripts/calibrate_baseline.py --dump-json scripts/hash_dump.json
What it does:
1. Fetches perceptual-hash rows of registered artworks from the database
(lib.db_service.get_all_artwork_hashes) or loads a previously dumped
JSON file. Registered artworks are assumed to be UNRELATED to each
other, so cross-pairing them approximates the unrelated-pair baseline.
2. Runs the updated consensus pipeline (compare_hash_sets) over sampled
cross-pairs and records the final_similarity distribution. The legacy
single-minimum score is computed too, for before/after validation.
3. Persists the distribution (mean/stddev + percentile table) to
config/calibration.json, which lib.phash loads at runtime to compute
calibrated_confidence.
Flagging threshold:
plagiarism_threshold = 99th percentile of the unrelated-pair baseline
distribution of the NEW consensus score, i.e. only ~1% of unrelated pairs
are expected to score at or above it by chance. This replaces the previous
uncalibrated 0-100 scale where arbitrary numbers (">70%") had no
statistical meaning.
Provisional constants (MIN_BLOCK_STDDEV, STRICT_MATCH_DISTANCE,
GLOBAL_PREFILTER_MAX_DISTANCE) were chosen before this data existed - this
script prints diagnostics relevant to each of them so they can be revisited;
do not treat the defaults as final just because they are in code.
"""
import argparse
import itertools
import json
import os
import random
import sys
from datetime import datetime, timezone
import numpy as np
# Make the repository root importable when run as a plain script
sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
from dotenv import load_dotenv # noqa: E402
load_dotenv() # Supabase credentials for local runs
from lib.phash import ( # noqa: E402
GLOBAL_PREFILTER_MAX_DISTANCE,
MAX_DISTANCE,
STRICT_MATCH_DISTANCE,
blocks_with_content,
compare_blocks,
compare_hash_sets,
compare_transforms,
global_prefilter_distance,
weighted_distance,
)
DEFAULT_OUTPUT = os.path.join(
os.path.dirname(os.path.dirname(os.path.abspath(__file__))),
"config",
"calibration.json",
)
PERCENTILE_POINTS = [0, 5, 25, 50, 75, 90, 95, 99, 99.9, 100]
def fetch_artworks(sample=None, seed=0):
"""Fetch hash rows from the registered_arts table (network required)."""
from lib.db_service import get_all_artwork_hashes
artworks = get_all_artwork_hashes()
if sample and len(artworks) > sample:
rng = random.Random(seed)
artworks = rng.sample(artworks, sample)
return artworks
def load_artworks(path):
"""Load hash rows from a JSON dump written by --dump-json."""
with open(path, "r", encoding="utf-8") as f:
return json.load(f)
def is_valid_hashset(hashes):
"""Skip rows missing the structure the comparison pipeline needs."""
return (
isinstance(hashes, dict)
and isinstance(hashes.get("transforms"), dict)
and isinstance(hashes.get("blocks"), dict)
and len(hashes["transforms"]) > 0
and len(hashes["blocks"]) > 0
)
def min_dist_between_blocks(blocks1, blocks2):
"""Smallest weighted distance across eligible cross-pairs (diagnostic)."""
eligible1 = blocks_with_content(blocks1) or list(blocks1.keys())
eligible2 = blocks_with_content(blocks2) or list(blocks2.keys())
best = float("inf")
for b1 in eligible1:
for b2 in eligible2:
d = weighted_distance(blocks1[b1], blocks2[b2])
if d < best:
best = d
return best
def distribution_stats(scores):
arr = np.asarray(scores, dtype=np.float64)
percentiles = {
str(p): round(float(np.percentile(arr, p)), 4) for p in PERCENTILE_POINTS
}
return {
"mean": round(float(np.mean(arr)), 4),
"stddev": round(float(np.std(arr, ddof=1)) if len(arr) > 1 else 0.0, 4),
"percentiles": percentiles,
}
def main():
parser = argparse.ArgumentParser(
description="Offline baseline calibration for plagiarism confidence")
parser.add_argument("--sample", type=int, default=None,
help="sample at most N artworks from the DB (default: all)")
parser.add_argument("--max-pairs", type=int, default=None,
help="cap the number of cross-pairs evaluated")
parser.add_argument("--seed", type=int, default=0,
help="random seed for sampling (reproducibility)")
parser.add_argument("--from-json", default=None,
help="load hash rows from a JSON dump instead of the DB")
parser.add_argument("--dump-json", default=None,
help="also write the fetched hash rows to this JSON file")
parser.add_argument("--output", default=DEFAULT_OUTPUT,
help=f"calibration config destination (default: {DEFAULT_OUTPUT})")
args = parser.parse_args()
# ------------------------- fetch / load rows -------------------------
if args.from_json:
artworks = load_artworks(args.from_json)
print(f"Loaded {len(artworks)} artwork hash rows from {args.from_json}")
else:
artworks = fetch_artworks(sample=args.sample, seed=args.seed)
print(f"Fetched {len(artworks)} artwork hash rows from the database")
if args.dump_json:
with open(args.dump_json, "w", encoding="utf-8") as f:
json.dump(artworks, f)
print(f"Dumped hash rows to {args.dump_json}")
valid = [a for a in artworks if is_valid_hashset(a.get("perceptual_hash"))]
skipped = len(artworks) - len(valid)
if skipped:
print(f"Skipped {skipped} rows with missing/incomplete hash payloads")
if len(valid) < 2:
print("Need at least 2 valid artwork hash rows to calibrate. Aborting.")
sys.exit(1)
# ----------------------------- cross-pair ----------------------------
rng = random.Random(args.seed)
pairs = list(itertools.combinations(valid, 2))
if args.max_pairs and len(pairs) > args.max_pairs:
pairs = rng.sample(pairs, args.max_pairs)
print(f"Evaluating {len(pairs)} unrelated cross-pairs "
f"({len(valid)} artworks, STRICT_MATCH_DISTANCE={STRICT_MATCH_DISTANCE})...")
new_scores = []
legacy_scores = []
pairs_with_agreement = 0
pairs_with_transform_agreement = 0
pairs_low_content = 0
pairs_bg_suppressed = 0
best_block_dists = []
prefilter_dists = []
for i, (a, b) in enumerate(pairs, start=1):
result = compare_hash_sets(a["perceptual_hash"], b["perceptual_hash"])
# final_similarity is the raw consensus score (calibration-independent)
new_scores.append(result["final_similarity"])
legacy_scores.append(result["raw_similarity_legacy"])
if result["low_content_warning"]:
pairs_low_content += 1
if result["background_coincidence_suppressed"]:
pairs_bg_suppressed += 1
# diagnostics for the provisional constants
block_result = compare_blocks(a["perceptual_hash"]["blocks"],
b["perceptual_hash"]["blocks"])
if block_result["agreements"]:
pairs_with_agreement += 1
transform_result = compare_transforms(a["perceptual_hash"]["transforms"],
b["perceptual_hash"]["transforms"])
if transform_result["agreements"]:
pairs_with_transform_agreement += 1
best_block_dists.append(min_dist_between_blocks(
a["perceptual_hash"]["blocks"], b["perceptual_hash"]["blocks"]))
prefilter_dists.append(global_prefilter_distance(
a["perceptual_hash"], b["perceptual_hash"]))
if i % 250 == 0 or i == len(pairs):
print(f" {i}/{len(pairs)} pairs done")
report_and_persist(new_scores, legacy_scores, pairs_with_agreement,
pairs_with_transform_agreement, pairs_low_content,
pairs_bg_suppressed, best_block_dists,
prefilter_dists, len(valid), len(pairs), args.output)
def report_and_persist(new_scores, legacy_scores, pairs_with_agreement,
pairs_with_transform_agreement, pairs_low_content,
pairs_bg_suppressed, best_block_dists, prefilter_dists,
n_artworks, n_pairs, output_path):
"""Print the baseline report and write config/calibration.json."""
new_stats = distribution_stats(new_scores)
legacy_stats = distribution_stats(legacy_scores)
degenerate = new_stats["stddev"] == 0.0
if degenerate:
# Degenerate baseline (e.g. every unrelated pair scores exactly 0 on
# the current catalog - good separation, little data). A percentile
# threshold would be 0 and meaningless, so place the threshold a
# fixed margin above the best score any unrelated pair achieved.
baseline_max = new_stats["percentiles"]["100"]
threshold = min(100.0, round(baseline_max + 5.0, 4))
threshold_basis = ("degenerate baseline (stddev 0): best unrelated "
"score + 5-point margin; switch to the 99th "
"percentile once the catalog is large enough to "
"produce a non-degenerate distribution")
else:
threshold = new_stats["percentiles"]["99"]
threshold_basis = ("99th percentile of the unrelated-pair baseline "
"distribution of the new consensus score; only ~1% "
"of unrelated pairs are expected to reach it by "
"chance")
print("\n=== Unrelated-pair baseline: NEW consensus score (final_similarity) ===")
print(json.dumps(new_stats, indent=2))
print("\n=== Unrelated-pair baseline: LEGACY single-minimum score (comparison) ===")
print(json.dumps(legacy_stats, indent=2))
print(f"\nFlagging threshold (p99 of new baseline): raw score >= {threshold}")
print(f"Pairs with >=1 agreeing block at STRICT_MATCH_DISTANCE="
f"{STRICT_MATCH_DISTANCE}: {pairs_with_agreement}/{n_pairs} "
f"({100.0 * pairs_with_agreement / n_pairs:.1f}%)")
print(f"Pairs with >=1 agreeing transform: "
f"{pairs_with_transform_agreement}/{n_pairs} "
f"({100.0 * pairs_with_transform_agreement / n_pairs:.1f}%)")
print(f"Pairs voided by the background-coincidence guard: "
f"{pairs_bg_suppressed}/{n_pairs}")
print(f"Pairs that hit the low-content fallback: {pairs_low_content}/{n_pairs}")
if best_block_dists:
d = np.asarray(best_block_dists, dtype=np.float64)
print(f"Best eligible-block distance between unrelated images: "
f"median={np.median(d):.1f} p10={np.percentile(d, 10):.1f} "
f"min={d.min():.1f} (max possible {MAX_DISTANCE})")
if prefilter_dists:
p = np.asarray(prefilter_dists, dtype=np.float64)
print(f"Global pre-filter distance between unrelated images: "
f"median={np.median(p):.1f} min={p.min():.1f} max={p.max():.1f} "
f"(cutoff GLOBAL_PREFILTER_MAX_DISTANCE="
f"{GLOBAL_PREFILTER_MAX_DISTANCE}; rows above the cutoff on all "
f"six variants are skipped)")
print("\nRevisit MIN_BLOCK_STDDEV / STRICT_MATCH_DISTANCE / "
"GLOBAL_PREFILTER_MAX_DISTANCE against these numbers before "
"treating the defaults as final.")
config = {
"version": 1,
"generated_at": datetime.now(timezone.utc).isoformat(),
"n_artworks": n_artworks,
"n_pairs": n_pairs,
"method": ("cross-paired registered artworks (assumed unrelated); "
"consensus pipeline final_similarity (entropy filter + "
"agreement ratio + transform consistency multiplier)"),
"mean": new_stats["mean"],
"stddev": new_stats["stddev"],
"percentiles": new_stats["percentiles"],
"plagiarism_threshold": threshold,
"threshold_basis": threshold_basis,
"legacy_baseline": legacy_stats,
"diagnostics": {
"degenerate_baseline": degenerate,
"pairs_with_block_agreement": pairs_with_agreement,
"pairs_with_transform_agreement": pairs_with_transform_agreement,
"pairs_background_coincidence_suppressed": pairs_bg_suppressed,
"pairs_low_content_fallback": pairs_low_content,
"strict_match_distance_used": STRICT_MATCH_DISTANCE,
"global_prefilter_max_distance_used": GLOBAL_PREFILTER_MAX_DISTANCE,
},
"notes": ("Consumed by lib.phash.calibrated_confidence. Regenerate "
"when the artwork catalog changes substantially or when "
"tunables change."),
}
os.makedirs(os.path.dirname(os.path.abspath(output_path)), exist_ok=True)
with open(output_path, "w", encoding="utf-8") as f:
json.dump(config, f, indent=2)
print(f"\nCalibration config written to {output_path}")
if __name__ == "__main__":
main()