digital-art-api / scripts /regression_absent_evidence.py
Ruzzel Mendoza
asd
5d0b1a5
Raw History Blame Contribute Delete
6.74 kB
#!/usr/bin/env python
"""
Offline regression bar for the dynamic reweighting of absent evidence
channels (compare_hash_sets in lib/phash.py).
Run it BEFORE and AFTER any change to the transform/block blend and compare
the two tables - the same before/after discipline as the previous passes:
python scripts/regression_absent_evidence.py
Cases:
1. crop/zoom fixture (image1.png vs image2.png, this thread's pair) -
transform evidence is structurally absent; raw_similarity must equal
the block-channel score, not 70% of it.
2. unrelated-pair baseline (78 cross-pairs from scripts/hash_dump.json) -
must stay at/near 0.0 even though most pairs also have
transform_agreements == 0 (weak block evidence must stay weak).
3. shared-background false positive (the original 11-image-set case,
replicated as background_art_a / background_art_b) - must stay a
clean negative.
4. genuine rotation case (noise artwork vs its 90-degree-rotated copy) -
transform evidence exists and is meaningful; the normal 0.3/0.7 blend
must still apply (the reweighting must not ignore transform evidence).
NOT wired into the API; read-only, writes nothing.
"""
import io
import itertools
import json
import os
import sys
import numpy as np
from PIL import Image, ImageDraw
ROOT = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
sys.path.insert(0, ROOT)
from lib import phash # noqa: E402
DUMP_PATH = os.path.join(ROOT, "scripts", "hash_dump.json")
# Fields reported for every case; absent fields (e.g. evidence-status fields
# added by the reweighting pass) are skipped so the script works before AND
# after the change.
REPORT_FIELDS = (
"raw_similarity", "raw_similarity_legacy", "block_similarity",
"transform_similarity", "transform_agreements", "block_agreements",
"content_blocks_used", "low_content_warning",
"background_coincidence_suppressed", "calibrated_confidence",
"calibration_loaded", "best_scale_pair",
"transform_evidence_status", "block_evidence_status",
)
def report(tag, r, threshold):
print(f" [{tag}]")
for k in REPORT_FIELDS:
if k in r:
print(f" {k}: {r[k]}")
if threshold is not None:
flagged = r["raw_similarity"] >= threshold
print(f" flagged (raw >= {threshold}): {flagged}")
def crop_fixture_hashes():
paths = [os.path.join(ROOT, "image1.png"), os.path.join(ROOT, "image2.png")]
return [phash.generate_hashes(open(p, "rb")) for p in paths]
def background_fixture_hashes():
"""Replicates tests/test_phash.py background_art_a / background_art_b."""
def art_a():
img = Image.new("RGB", (256, 256), (255, 255, 255))
ImageDraw.Draw(img).ellipse((10, 10, 90, 90), fill=(0, 0, 0))
return img
def art_b():
rng = np.random.default_rng(77)
patch = Image.fromarray(
rng.integers(0, 256, size=(80, 80), dtype=np.uint8), mode="L"
).convert("RGB")
img = Image.new("RGB", (256, 256), (255, 255, 255))
img.paste(patch, (170, 170))
return img
def to_stream(img):
buf = io.BytesIO()
img.save(buf, format="PNG")
buf.seek(0)
return buf
return [phash.generate_hashes(to_stream(art_a())),
phash.generate_hashes(to_stream(art_b()))]
def rotation_fixture_hashes():
rng = np.random.default_rng(91)
arr = rng.integers(0, 256, size=(256, 256), dtype=np.uint8)
img = Image.fromarray(arr, mode="L").convert("RGB")
def to_stream(im):
buf = io.BytesIO()
im.save(buf, format="PNG")
buf.seek(0)
return buf
return [phash.generate_hashes(to_stream(img)),
phash.generate_hashes(to_stream(img.rotate(90, expand=True)))]
def load_dump_artworks():
with open(DUMP_PATH, "r", encoding="utf-8") as f:
return json.load(f)
def main():
threshold = phash.plagiarism_threshold()
print(f"Flagging threshold: {threshold} "
f"(calibration_loaded={phash.load_calibration() is not None})")
print("\n=== Case 1: crop/zoom fixture (image1.png vs image2.png) ===")
h1, h2 = crop_fixture_hashes()
r = phash.compare_hash_sets(h1, h2)
report("upload vs db", r, threshold)
if "transform_evidence_status" in r:
expected = "raw == block_similarity" if r["block_agreements"] else "n/a"
ok = abs(r["raw_similarity"] - r["block_similarity"]) < 0.01
print(f" renormalization check: {expected} -> "
f"{'PASS' if ok else 'FAIL'}")
print("\n=== Case 2: unrelated-pair baseline (78 pairs from hash_dump.json) ===")
artworks = load_dump_artworks()
pairs = list(itertools.combinations(artworks, 2))
scores, blocks, zero_transform = [], 0, 0
top = []
for a, b in pairs:
r = phash.compare_hash_sets(a["perceptual_hash"], b["perceptual_hash"])
scores.append(r["raw_similarity"])
top.append((r["raw_similarity"], r["block_similarity"],
r["transform_agreements"], r["block_agreements"],
a["id"][:8], b["id"][:8]))
if r["block_similarity"] > 0:
blocks += 1
if r["transform_agreements"] == 0:
zero_transform += 1
scores_np = np.asarray(scores, dtype=np.float64)
print(f" n_pairs: {len(pairs)}")
print(f" max raw_similarity: {scores_np.max():.2f}")
print(f" mean raw_similarity: {scores_np.mean():.4f} "
f"stddev: {scores_np.std():.4f}")
print(f" pairs with transform_agreements == 0: {zero_transform}/{len(pairs)}")
print(f" pairs with block_similarity > 0: {blocks}/{len(pairs)}")
if threshold is not None:
print(f" pairs flagged (raw >= {threshold}): "
f"{sum(1 for s in scores if s >= threshold)}/{len(pairs)}")
print(" top 3 pairs by raw_similarity "
"(raw, block, t_agree, b_agree, idA, idB):")
for row in sorted(top, reverse=True)[:3]:
print(f" {row}")
print("\n=== Case 3: shared-background false positive (background_art_a/b) ===")
h1, h2 = background_fixture_hashes()
r = phash.compare_hash_sets(h1, h2)
report("art_a vs art_b", r, threshold)
print("\n=== Case 4: genuine rotation case (noise vs its 90-degree copy) ===")
h1, h2 = rotation_fixture_hashes()
r = phash.compare_hash_sets(h1, h2)
report("original vs rotated", r, threshold)
if "transform_evidence_status" in r:
print(f" blend check (0.3*t + 0.7*b with multiplier 1 expected): "
f"t_sim={r['transform_similarity']} b_sim={r['block_similarity']} "
f"raw={r['raw_similarity']}")
if __name__ == "__main__":
main()