digital-art-api / lib /phash.py
Ruzzel Mendoza
whash, phash, and dhash weight tuning
7c8d720
Raw History Blame Contribute Delete
51.9 kB
import os from io import BytesIO from typing import Dict, Any import json import math import os import requests from PIL import Image, ImageOps, ImageFilter import imagehash import numpy as np from concurrent.futures import ThreadPoolExecutor, as_completed HASH_SIZE = 8 MAX_DISTANCE = HASH_SIZE * HASH_SIZE # ----------------------------- # TUNABLE THRESHOLDS # ----------------------------- # Every threshold below is provisional until baseline calibration data exists # (see scripts/calibrate_baseline.py). Each one can be overridden via an # environment variable without a code change. def _env_float(name, default): try: return float(os.environ.get(name, default)) except (TypeError, ValueError): return default # Minimum stddev of a preprocessed grayscale block for it to count as # "content-bearing". Flat regions (solid backgrounds) hash to near-identical # "empty" fingerprints regardless of the real artwork, so blocks below this # threshold are excluded from block matching. Tune against calibration data. MIN_BLOCK_STDDEV = _env_float("PLAGIARISM_MIN_BLOCK_STDDEV", 5.0) # Maximum weighted hash distance (0..MAX_DISTANCE) for a block/transform pair # to count as an "agreement" in consensus scoring. Pairs farther apart than # this are treated as non-matching instead of dragging the average. STRICT_MATCH_DISTANCE = _env_float("PLAGIARISM_STRICT_MATCH_DISTANCE", 12.0) # Number of agreeing blocks required before full confidence is granted. A # single lucky matching block cannot establish geometric consistency on its # own, so the confidence multiplier is capped until this many blocks agree. # Never demands more agreeing blocks than the image actually has content # blocks (sparse/minimalist artwork must not be over-suppressed). MIN_CONSENSUS_BLOCKS = int(_env_float("PLAGIARISM_MIN_CONSENSUS_BLOCKS", 2)) # Maximum Hamming distance (0..MAX_DISTANCE) on the cheap global phash # pre-filter. scan_database skips a DB row before the expensive pipeline only # when the query phash is farther than this from ALL SIX precomputed # transform variants of the row (rotation-aware: a rotated copy matches one # of the variants closely and is never skipped). Performance-only; never # influences the score of rows that pass. GLOBAL_PREFILTER_MAX_DISTANCE = int( _env_float("PLAGIARISM_GLOBAL_PREFILTER_MAX_DISTANCE", 28)) # Minimum legacy score that arms the zero-consensus fallback (Option B gate). # When the consensus pipeline scores a pair at (near-)zero but the legacy # single-minimum algorithm scores at or above this, the legacy value is used # for plagiarism *ranking/flagging* decisions. Below this, a zero-consensus # pair stays a clean negative so the legacy algorithm's known background # inflation (unrelated baseline mean ~61.8, p95 ~66.9) does not flag # everything. Overridable via PLAGIARISM_LEGACY_FALLBACK_MIN_SCORE. LEGACY_FALLBACK_MIN_SCORE = _env_float("PLAGIARISM_LEGACY_FALLBACK_MIN_SCORE", 60.0) # Location of the baseline calibration config written by # scripts/calibrate_baseline.py. Overridable for tests/deployments. _REPO_ROOT = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) CALIBRATION_CONFIG_PATH = os.environ.get( "PLAGIARISM_CALIBRATION_CONFIG", os.path.join(_REPO_ROOT, "config", "calibration.json"), ) # ----------------------------- # MULTI-SCALE BLOCK MATCHING # ----------------------------- # Scale factors at which each image is block-hashed. A zoomed/re-cropped # copy of an artwork can then align with a differently-scaled version of # the original instead of requiring exact pixel-level quadrant alignment. # Keep to 3 scales initially: more scales directly multiply the comparison # count (5 blocks x N scales x 5 blocks x N scales) against the consensus # design's goal of controlling that count. SCALE_FACTORS = [0.625, 0.75, 1.0] # ----------------------------- # IMAGE PREPROCESSING # ----------------------------- def preprocess_image(img: Image.Image): if img.mode in ("RGBA", "LA"): bg = Image.new("RGB", img.size, (255, 255, 255)) bg.paste(img, mask=img.split()[-1]) img = bg else: img = img.convert("RGB") img = img.convert("L") img = img.filter(ImageFilter.GaussianBlur(radius=1)) return img # ----------------------------- # ENTROPY / CONTENT DETECTION # ----------------------------- def block_entropy(block: Image.Image) -> float: """ Content measure of a block: stddev of its preprocessed grayscale pixels. Expects the block to already be preprocessed (grayscale + Gaussian blur), i.e. exactly the pixel data that gets hashed. Flat/uniform regions such as solid backgrounds score near 0 and produce near-identical "empty" fingerprints regardless of the real artwork, so this value is used to exclude them from block matching. """ arr = np.asarray(block, dtype=np.float64) if arr.size == 0: return 0.0 return float(np.std(arr)) def blocks_with_content(blocks: Dict[str, Any]) -> list: """ Names of blocks whose recorded entropy is at or above MIN_BLOCK_STDDEV. Legacy hash payloads (old database rows) carry no "entropy" key; such blocks are treated as content-bearing so old data keeps working. """ eligible = [] for name, block_hashes in blocks.items(): entropy = block_hashes.get("entropy") if entropy is None or entropy >= MIN_BLOCK_STDDEV: eligible.append(name) return eligible def low_content_warning_for(hashset: Dict[str, Any]) -> bool: """ True when every block of this hash set is flat/low-detail. Comparison still falls back to using flat blocks in that case (otherwise there would be nothing to compare), but the flag propagates to the API response so clients can treat the score with caution. """ blocks = hashset.get("blocks") or {} if not blocks: return False return not blocks_with_content(blocks) # ----------------------------- # HASH GENERATION # ----------------------------- def compute_hashes(img: Image.Image): return { "phash": str(imagehash.phash(img, hash_size=HASH_SIZE)), "dhash": str(imagehash.dhash(img, hash_size=HASH_SIZE)), "whash": str(imagehash.whash(img, hash_size=HASH_SIZE)) } def generate_transform_hashes(img: Image.Image): transforms = { "0": img, "90": img.rotate(90, expand=True), "180": img.rotate(180, expand=True), "270": img.rotate(270, expand=True), "mirror": ImageOps.mirror(img), "flip": ImageOps.flip(img) } return {k: compute_hashes(v) for k, v in transforms.items()} def generate_block_hashes(img: Image.Image): w, h = img.size blocks = { "top_left": img.crop((0, 0, w//2, h//2)), "top_right": img.crop((w//2, 0, w, h//2)), "bottom_left": img.crop((0, h//2, w//2, h)), "bottom_right": img.crop((w//2, h//2, w, h)), "center": img.crop((w//4, h//4, 3*w//4, 3*h//4)) } result = {} for name, block in blocks.items(): preprocessed = preprocess_image(block) block_hashes = compute_hashes(preprocessed) # Tag each block with its entropy score so comparisons can exclude # flat/low-detail blocks that hash to near-identical fingerprints. block_hashes["entropy"] = round(block_entropy(preprocessed), 4) result[name] = block_hashes return result def generate_multiscale_block_hashes(img: Image.Image): """ Block hashes at multiple grid scales of the same (preprocessed) image. Instead of resizing the whole frame (which is a pHash no-op: pHash is scale-invariant and the relative 2x2+center split always covers the same proportional region), this function varies the *relative footprint* of each block while keeping the same 2x2+center structure, same entropy tagging/filtering, and same consensus downstream logic. For each factor in SCALE_FACTORS the block grid is scaled so that each block's coverage is `factor` times the standard coverage, anchored at the frame center (clamped to image bounds). A zoomed copy of an artwork can then align with a differently-grid-scaled version of the original instead of requiring exact pixel-level quadrant alignment. Block keys are prefixed with their grid scale (e.g. "0.75:center", "1.0:top_left") so results stay traceable and so the consensus code in compare_blocks can distinguish which scale a block came from. generate_block_hashes (single standard scale) remains intact and unit-testable; this function is its multi-scale wrapper. """ result = {} for factor in SCALE_FACTORS: blocks = _cut_grid_blocks(img, factor) for name, block_data in blocks.items(): key = f"{factor}:{name}" result[key] = block_data return result def _cut_grid_blocks(img, grid_scale): """Cut the 2x2+center block grid whose relative footprint is `grid_scale` times the standard split, anchored at the frame center and clamped to image bounds. Block hashes + entropy tagging use the same logic as generate_block_hashes.""" w, h = img.size s = grid_scale half = s / 2.0 def _clamp(c): return max(0.0, min(1.0, c)) # relative (0..1) bounding boxes anchored at frame center quads = [ ("top_left", (0.5 - half, 0.5 - half, 0.5, 0.5)), ("top_right", (0.5, 0.5 - half, 0.5 + half, 0.5)), ("bottom_left", (0.5 - half, 0.5, 0.5, 0.5 + half)), ("bottom_right", (0.5, 0.5, 0.5 + half, 0.5 + half)), ("center", (0.5 - s / 4, 0.5 - s / 4, 0.5 + s / 4, 0.5 + s / 4)), ] result = {} for name, (x0, y0, x1, y1) in quads: left, upper = int(round(_clamp(x0) * w)), int(round(_clamp(y0) * h)) right, lower = int(round(_clamp(x1) * w)), int(round(_clamp(y1) * h)) if right - left < 2 or lower - upper < 2: continue # degenerate at extreme scales; drop silently block = img.crop((left, upper, right, lower)) preprocessed = preprocess_image(block) block_hashes = compute_hashes(preprocessed) block_hashes["entropy"] = round(block_entropy(preprocessed), 4) result[name] = block_hashes return result def generate_hashes(image_file): try: img = Image.open(image_file) img = preprocess_image(img) return { "transforms": generate_transform_hashes(img), "blocks": generate_multiscale_block_hashes(img) } except Exception as e: print("Error generating hashes:", e) return None # Module-level session: reused connection pool, keep-alive, no repeated TLS handshakes _session = requests.Session() _session.headers.update({"User-Agent": "Mozilla/5.0"}) _adapter = requests.adapters.HTTPAdapter(pool_connections=20, pool_maxsize=20) _session.mount("https://", _adapter) _session.mount("http://", _adapter) def generate_hash_from_url(url): """Fetch one image URL and hash it. Retries transient failures once. Returns (hashes_or_None, reason_or_None) so callers can keep diagnostics without dropping the whole batch on one bad host. """ if not url: return None, "missing_url" from urllib.parse import urlparse import time as _time try: host = urlparse(url).netloc[:80] or "unknown_host" except Exception: host = "unknown_host" last_err = None last_reason = None for attempt in range(2): try: # Separate connect/read timeouts: slow CDNs (e.g. Instagram # lookaside) often connect fast then stall on read. One retry # with backoff instead of hammering immediately. resp = _session.get(url, timeout=(5, 20), stream=True) resp.raise_for_status() content_type = (resp.headers.get("Content-Type") or "").lower() if content_type and "image" not in content_type and "octet-stream" not in content_type: resp.close() return None, f"non_image_content_type:{content_type[:60]}" # Stream with a size cap so a huge/never-ending body can't OOM # the HF Space; most SERP thumbnails are well under this. chunks = [] total = 0 for chunk in resp.iter_content(chunk_size=256 * 1024): if not chunk: continue total += len(chunk) if total > 15 * 1024 * 1024: resp.close() return None, "body_too_large" chunks.append(chunk) resp.close() content = b"".join(chunks) if not content: return None, "empty_body" # Validate bytes are a decodable image BEFORE hashing so the # logs say *why* ("HTML error page", "truncated", ...) instead # of the generic "cannot identify image file". try: probe = Image.open(BytesIO(content)) fmt = (probe.format or "unknown").lower() probe.load() except Exception as probe_err: snippet = content[:200].lstrip()[:120] hint = "unknown_bytes" if snippet.startswith(b"<"): hint = "html_or_text_page_not_image" elif snippet.startswith(b"\x89PNG") or snippet.startswith(b"\xff\xd8"): hint = "truncated_image" fmt = locals().get("fmt", hint) return None, f"undecodable_image:{fmt}:{str(probe_err)[:80]}" hashes = generate_hashes(BytesIO(content)) if hashes is None: return None, "hash_generation_failed" return hashes, None except Exception as e: last_err = e msg = str(e) # Classify so web_diagnostics groups by cause, not by traceback. if "Read timed out" in msg or "ConnectTimeout" in type(e).__name__ or "Timeout" in type(e).__name__: last_reason = f"fetch_timeout:{host}" elif "403" in msg or "401" in msg: last_reason = f"fetch_forbidden:{host}" elif "404" in msg: last_reason = f"fetch_not_found:{host}" else: last_reason = f"fetch_failed:{host}:{msg[:60]}" if attempt == 0: _time.sleep(1.0) # backoff before retry; don't hammer slow CDNs # Log once with host + final reason (not once per attempt) to keep # HF Space logs readable when a whole CDN (e.g. lookaside) is slow. print(f"Error fetching URL {host}: {last_reason} ({last_err})") return None, last_reason or f"fetch_failed:{last_err}" # ----------------------------- # DISTANCE CALCULATION # ----------------------------- def hamming(hash1, hash2): return imagehash.hex_to_hash(hash1) - imagehash.hex_to_hash(hash2) def weighted_distance(hash1, hash2, weights=None): if weights is None: weights = {"phash": 0.7, "dhash": 0.05, "whash": 0.25} total, weight_sum = 0, 0 for algo, w in weights.items(): total += hamming(hash1[algo], hash2[algo]) * w weight_sum += w return total / weight_sum def _consensus_similarity(agreement_ratio, agreements): """ Derive a 0-100 similarity from (agreement ratio, average distance of the agreeing pairs). A single lucky match out of N eligible components can contribute at most ~100/N, so coincidence is no longer rewarded. """ if not agreements: return 0.0 avg_dist = sum(a["dist"] for a in agreements) / len(agreements) quality = max(0.0, 1.0 - avg_dist / MAX_DISTANCE) return 100.0 * agreement_ratio * quality # ----------------------------- # GEOMETRIC / TRANSFORM CONSISTENCY # ----------------------------- # For each of the six transforms applied when hashing, where each block # position ends up. Used to check whether agreeing block pairs tell a # geometrically consistent story (root cause: one lucky matching block # should not decide the whole score). TRANSFORM_BLOCK_MAP = { "0": { "top_left": "top_left", "top_right": "top_right", "bottom_left": "bottom_left", "bottom_right": "bottom_right", "center": "center", }, # PIL rotate(90) is counter-clockwise: top edge becomes left edge "90": { "top_left": "bottom_left", "top_right": "top_left", "bottom_left": "bottom_right", "bottom_right": "top_right", "center": "center", }, "180": { "top_left": "bottom_right", "top_right": "bottom_left", "bottom_left": "top_right", "bottom_right": "top_left", "center": "center", }, # rotate(270) counter-clockwise == 90 clockwise "270": { "top_left": "top_right", "top_right": "bottom_right", "bottom_left": "top_left", "bottom_right": "bottom_left", "center": "center", }, # ImageOps.mirror: horizontal (left/right) flip "mirror": { "top_left": "top_right", "top_right": "top_left", "bottom_left": "bottom_right", "bottom_right": "bottom_left", "center": "center", }, # ImageOps.flip: vertical (top/bottom) flip "flip": { "top_left": "bottom_left", "top_right": "bottom_right", "bottom_left": "top_left", "bottom_right": "top_right", "center": "center", }, } def implied_transforms(b1, b2): """ Which transforms would move block position b1 onto block position b2. Block keys may carry a multi-scale prefix ("0.75:top_left"); the prefix identifies which scale a block was hashed at and is ignored here - two blocks agreeing across different scales still vote for the transform their POSITIONS imply. A center match is geometrically uninformative (center maps to center under every transform), so center pairs return the full transform list and are excluded from voting by the caller. """ pos1 = b1.split(":", 1)[-1] pos2 = b2.split(":", 1)[-1] return [t for t, mapping in TRANSFORM_BLOCK_MAP.items() if mapping.get(pos1) == pos2] def compute_transform_consistency(agreements): """ Geometric consistency across agreeing block pairs. Each agreeing pair (b1 -> b2) votes for the transform(s) that would map b1 onto b2; ambiguous pairs split their vote fractionally. Returns (consistency, dominant_transform) where consistency is the fraction of votes held by the dominant transform. The result distinguishes two empty-signal cases: (0.0, None) when there are NO agreeing pairs at all - there is no geometric evidence, so consistency is zero (this makes the exposed transform_consistency field honest for unrelated images instead of a misleading "perfect" 1.0). (1.0, None) when agreements exist but every one is geometrically uninformative (e.g. only center matches, for which every transform maps center to center) - the neutral default. The MIN_CONSENSUS_BLOCKS evidence cap in compare_hash_sets independently guards against single coincidental matches. """ votes = {} voting_pairs = 0 n_transforms = len(TRANSFORM_BLOCK_MAP) for pair in agreements: candidates = implied_transforms(pair["b1"], pair["b2"]) # Fully ambiguous (center-center) or impossible: no geometric signal if not candidates or len(candidates) >= n_transforms: continue voting_pairs += 1 share = 1.0 / len(candidates) for t in candidates: votes[t] = votes.get(t, 0.0) + share if not agreements: return 0.0, None # agreements exist but every one was uninformative (e.g. center-center only) if voting_pairs == 0: return 1.0, None dominant = max(votes, key=lambda t: votes[t]) return votes[dominant] / voting_pairs, dominant def evidence_cap(n_agreements, n_eligible): """ Ramp 0 -> 1 as the number of agreements reaches MIN_CONSENSUS_BLOCKS. Never demands more agreements than there are eligible components, so an image with only 1-2 content blocks (sparse/minimalist artwork) or only a few comparable transforms is not punished beyond what its evidence allows. """ required = max(1, min(MIN_CONSENSUS_BLOCKS, n_eligible)) return min(1.0, n_agreements / required) def confidence_multiplier(agreements, n_eligible_blocks): """ Multiplier in [0, 1] applied to the block consensus score. = transform_consistency (geometric vote share across agreeing block pairs) * evidence cap. A single lucky matching block cannot establish geometric consistency on its own and rarely survives both factors. """ consistency, _ = compute_transform_consistency(agreements) return consistency * evidence_cap(len(agreements), n_eligible_blocks) # ----------------------------- # BASELINE CALIBRATION # ----------------------------- # The raw 0-100 consensus score was never calibrated against what unrelated # images actually score. scripts/calibrate_baseline.py cross-pairs registered # artworks (assumed unrelated) and writes config/calibration.json with the # resulting score distribution. calibrated_confidence() then reports where a # new raw score sits within that baseline distribution (percentile, 0-100). _calibration_cache = "unset" def reset_calibration_cache(): """Forget the cached calibration config (used by tests and reloads).""" global _calibration_cache _calibration_cache = "unset" def load_calibration(path=None): """ Load the calibration config written by scripts/calibrate_baseline.py. Returns None when the config is missing or unreadable; callers must treat that as "not calibrated yet". The default-path result is cached. """ global _calibration_cache if path is None: if _calibration_cache != "unset": return _calibration_cache target = CALIBRATION_CONFIG_PATH else: target = path try: with open(target, "r", encoding="utf-8") as f: cfg = json.load(f) except (OSError, ValueError): cfg = None if path is None: _calibration_cache = cfg return cfg def _normal_cdf_percent(score, mean, stddev): """Normal-CDF percentile of score given baseline mean/stddev.""" if stddev <= 0: return 100.0 if score >= mean else 0.0 z = (score - mean) / stddev return 100.0 * 0.5 * (1.0 + math.erf(z / math.sqrt(2.0))) def calibrated_confidence(raw_score, calibration=None): """ Convert a raw 0-100 consensus score into a calibrated confidence (0-100): the percentile of the unrelated-pair baseline distribution at or below the score (higher = more unusual among unrelated images = more likely to be a real match). Piecewise-linear interpolation inside the recorded percentile table; normal-CDF extrapolation in the tails. Until calibration exists (no config file) this falls back to the identity mapping so the field is always populated, and the calibration_loaded flag in comparison results tells clients which case applies. """ score = float(max(0.0, min(100.0, raw_score))) cfg = calibration if calibration is not None else load_calibration() if not cfg: return score try: table = sorted((float(p), float(v)) for p, v in cfg["percentiles"].items()) mean = float(cfg["mean"]) stddev = float(cfg["stddev"]) except (KeyError, TypeError, ValueError): return score if not table: return score low_pct, low_val = table[0] high_pct, high_val = table[-1] # Tails: extrapolate with the normal approximation of the baseline if score <= low_val: return max(0.0, min(low_pct, _normal_cdf_percent(score, mean, stddev))) if score >= high_val: return max(high_pct, min(100.0, _normal_cdf_percent(score, mean, stddev))) # Interior: piecewise-linear interpolation on the percentile table for (p0, v0), (p1, v1) in zip(table, table[1:]): if v0 <= score <= v1: frac = 0.5 if v1 == v0 else (score - v0) / (v1 - v0) return p0 + frac * (p1 - p0) return high_pct def plagiarism_threshold(calibration=None): """ Raw score at or above which a match is flagged as suspected plagiarism. Derived by scripts/calibrate_baseline.py as the 99th percentile of the unrelated-pair baseline distribution (i.e. only ~1% of unrelated pairs score at or above it by chance). None until calibration exists. """ cfg = calibration if calibration is not None else load_calibration() if not cfg: return None threshold = cfg.get("plagiarism_threshold") try: return float(threshold) except (TypeError, ValueError): return None def compare_transforms(hashes1, hashes2): """ Consensus-based whole-image transform comparison. For each transform of image 1 the single best (minimum distance) match among image 2's transforms is found. A pair "agrees" when that best distance is below STRICT_MATCH_DISTANCE. Similarity derives from how many transforms agree and how close the agreeing pairs are - not from the one best pair out of 36 cross-comparisons, which rewarded coincidence (multiple-comparisons problem). Returns a dict: similarity 0-100 consensus score (linear, uncalibrated) agreements list of agreeing pairs {t1, t2, dist} agreement_ratio fraction of image-1 transforms that agreed """ agreements = [] for t1, h1 in hashes1.items(): best_t2, best_dist = None, float("inf") for t2, h2 in hashes2.items(): d = weighted_distance(h1, h2) if d < best_dist: best_t2, best_dist = t2, d if best_dist < STRICT_MATCH_DISTANCE: agreements.append({"t1": t1, "t2": best_t2, "dist": best_dist}) n = len(hashes1) ratio = len(agreements) / n if n else 0.0 similarity = _consensus_similarity(ratio, agreements) return { "similarity": similarity, "agreements": agreements, "agreement_ratio": ratio, } def compare_transforms_legacy(hashes1, hashes2): """ Old single-minimum-distance transform scoring (kept for before/after validation via the raw_similarity_legacy field and offline tooling). """ min_dist = float("inf") for t1 in hashes1: for t2 in hashes2: d = weighted_distance(hashes1[t1], hashes2[t2]) if d < min_dist: min_dist = d return (1 - min_dist / MAX_DISTANCE) * 100 def _parse_block_scale(key): """Return (scale, position) for a block key with a 'scale:position' form (multi-scale) or (None, key) for an unprefixed legacy key.""" if ":" in key: scale, _, pos = key.partition(":") return scale, pos return None, key def _group_blocks_by_scale(blocks): """Split a flat block dict into {scale: {position: data}}. Unprefixed (legacy) keys are grouped under the sentinel scale key ''. If every key carries the same scale prefix (already uniform), returns a single group under that scale.""" groups = {} for key, data in blocks.items(): scale, pos = _parse_block_scale(key) groups.setdefault(scale if scale is not None else "", {})[pos] = data return groups def _single_mesh_compare(blocks1, blocks2): """ The core 5-block (single-scale) consensus used by both compare_blocks and compare_multiscale_blocks. Returns the same dict shape as compare_blocks. """ eligible1 = blocks_with_content(blocks1) eligible2 = blocks_with_content(blocks2) low_content_warning = False if not eligible1: eligible1 = list(blocks1.keys()) low_content_warning = True if not eligible2: eligible2 = list(blocks2.keys()) low_content_warning = True agreements = [] for b1 in eligible1: best_b2, best_dist = None, float("inf") for b2 in eligible2: d = weighted_distance(blocks1[b1], blocks2[b2]) if d < best_dist: best_b2, best_dist = b2, d if best_dist < STRICT_MATCH_DISTANCE: agreements.append({"b1": b1, "b2": best_b2, "dist": best_dist}) n_eligible = len(eligible1) ratio = len(agreements) / n_eligible if n_eligible else 0.0 similarity = _consensus_similarity(ratio, agreements) return { "similarity": similarity, "agreements": agreements, "agreement_ratio": ratio, "content_blocks_used": n_eligible, "low_content_warning": low_content_warning, "eligible_blocks": eligible1, } def compare_blocks(blocks1, blocks2): """ Consensus-based block comparison with entropy filtering. Flat/low-detail blocks (entropy below MIN_BLOCK_STDDEV) are excluded from pairing because they hash to near-identical "empty" fingerprints regardless of the real artwork, which rewards coincidence. If every block of an image is flat, all of its blocks are used as a fallback and low_content_warning is raised. For each eligible block of image 1 the single best (minimum distance) match among image 2's eligible blocks is found; a pair "agrees" when that best distance is below STRICT_MATCH_DISTANCE. Similarity derives from the agreement ratio and the average distance of agreeing pairs - not from the one best pair out of all cross-comparisons (multiple-comparisons problem). Returns a dict: similarity 0-100 consensus score (linear, uncalibrated) agreements list of agreeing pairs {b1, b2, dist} agreement_ratio fraction of eligible image-1 blocks that agreed content_blocks_used number of eligible (content-bearing) blocks of the first image that were used in the comparison low_content_warning True when at least one image had no content-bearing blocks and flat blocks had to be used anyway eligible_blocks names of the eligible blocks of the first image This operates on a single-scale (unprefixed, 5-block) dict, e.g. legacy database rows or a caller that already picked one scale. The multi-scale wrapper below reuses the same per-mesh consensus. """ return _single_mesh_compare(blocks1, blocks2) def compare_multiscale_blocks(blocks1, blocks2): """ Multi-scale block comparison: a zoomed/re-cropped copy of an artwork will align with exactly ONE of the scaled versions of the other image, not with all of them at once. So instead of treating every (scale, position) brick as an equal peer (which dilutes a genuine single-scale alignment), the block grids are re-grouped by scale and the existing 5-block consensus is evaluated independently for every (scale_of_img1, scale_of_img2) pair. The best agreeing scale-pair is reported. This keeps the exact same entropy filtering, agreement ratio, and geometric/transform-consistency logic from the single-scale consensus; it only fixes the denominator back to the 5-block mesh that the consensus design assumed. Legacy/unprefixed block dicts are treated as a single implicit scale and behave identically to compare_blocks. Returns the same shape as compare_blocks plus: best_scale_pair (s1, s2) that produced the best result """ g1 = _group_blocks_by_scale(blocks1) g2 = _group_blocks_by_scale(blocks2) results = [] for s1, sub1 in g1.items(): for s2, sub2 in g2.items(): mesh = _single_mesh_compare(sub1, sub2) mesh["best_scale_pair"] = (s1, s2) results.append(mesh) if not results: return { "similarity": 0.0, "agreements": [], "agreement_ratio": 0.0, "content_blocks_used": 0, "low_content_warning": True, "eligible_blocks": [], "best_scale_pair": (None, None), } return max(results, key=lambda m: (m["similarity"], len(m["agreements"]))) def compare_blocks_legacy(blocks1, blocks2): """ Old single-minimum-distance block scoring without entropy filtering (kept for before/after validation via the raw_similarity_legacy field and offline tooling). """ min_dist = float("inf") for b1 in blocks1: for b2 in blocks2: d = weighted_distance(blocks1[b1], blocks2[b2]) if d < min_dist: min_dist = d return (1 - min_dist / MAX_DISTANCE) * 100 def compare_hash_sets(hashset1, hashset2): """ Full consensus comparison of two hash sets. Keeps the original field names (transform_similarity, block_similarity, final_similarity) and adds: raw_similarity new consensus algorithm, linear 0-100, uncalibrated (same value as final_similarity; consensus score multiplied by the transform-consistency confidence multiplier) raw_similarity_legacy old single-minimum algorithm, linear 0-100 (kept during the transition so results can be validated before/after; will be removed in a later cleanup) transform_consistency geometric consistency (0-1) of the agreeing block pairs (see compute_transform_consistency) dominant_transform the transform the agreeing blocks collectively imply (None when there is no geometric signal) background_coincidence_suppressed True when transform-level resemblance was voided for lack of block corroboration (shared background / layout false-positive guard) transform_evidence_status / block_evidence_status "absent" when that evidence channel structurally could not produce evidence (transform_agreements == 0 after the full 6x6 comparison; respectively content_blocks_used == 0), "checked" otherwise. In the "absent" case the score is renormalized fully onto the other channel instead of applying the fixed 0.3/0.7 split. """ transform_result = compare_transforms(hashset1["transforms"], hashset2["transforms"]) block_result = compare_multiscale_blocks(hashset1["blocks"], hashset2["blocks"]) transform_sim = transform_result["similarity"] block_sim = block_result["similarity"] # Block evidence: damped by geometric consistency across agreeing blocks # (low consistency suppresses the score even when raw distances are low) # and by the evidence cap (one lucky block cannot decide the score). consistency, dominant_transform = compute_transform_consistency(block_result["agreements"]) block_multiplier = confidence_multiplier(block_result["agreements"], block_result["content_blocks_used"]) # Transform evidence: damped only by the evidence cap. The block-based # geometric multiplier must not zero it out - a pure rotate-and-repost # copy matches through transforms while its position-bound blocks do not. transform_multiplier = evidence_cap(len(transform_result["agreements"]), len(hashset1["transforms"])) # Background-coincidence guard: transform-level resemblance without ANY # corroborating block agreement is presumed to be shared background / # layout rather than copied content - the exact false positive this # pipeline was built to fix. The exception is a genuinely tight alignment # (average agreeing distance <= STRICT_MATCH_DISTANCE / 2): real # rotate/mirror/flip reposts align almost exactly at the global level, # while background/layout coincidences only scrape into the strict band. # Fully flat images (low_content_warning) are left alone; they carry # their warning to the client instead. transform_agreements = transform_result["agreements"] if transform_agreements: avg_transform_dist = (sum(p["dist"] for p in transform_agreements) / len(transform_agreements)) else: avg_transform_dist = float("inf") tight_alignment = avg_transform_dist <= STRICT_MATCH_DISTANCE / 2 background_coincidence = bool( transform_agreements and not block_result["agreements"] and not tight_alignment and not block_result["low_content_warning"] and blocks_with_content(hashset1["blocks"]) and blocks_with_content(hashset2["blocks"]) ) if background_coincidence: transform_multiplier = 0.0 # Dynamic reweighting for ABSENT evidence channels. "Absent" means the # comparison structurally cannot produce evidence of that type, which is # different from "checked and failed" (a nonzero-but-low similarity, # which is real negative evidence and keeps the normal weighting): # - transform evidence is absent when transform_agreements == 0 after # the full 6x6 transform comparison (e.g. a pure crop/zoom copy: no # rotation/flip/mirror happened, so the whole-image transform # comparison cannot find a match no matter how strong the block # evidence is); # - block evidence is absent when content_blocks_used == 0 (no # content-bearing block mesh existed to compare). # In the absent case the fixed 0.3/0.7 split is NOT applied: the score is # renormalized fully onto the channel that does have evidence (the # explicit branches below, not a division-by-adjusted-weights trick). transform_evidence_status = ("checked" if transform_result["agreements"] else "absent") block_evidence_status = ("checked" if block_result["content_blocks_used"] else "absent") if (transform_evidence_status == "absent" and block_evidence_status == "absent"): # No usable evidence at all -> no score. final_score = 0.0 elif transform_evidence_status == "absent": # Renormalize fully onto the block channel. The block-side damping # (geometric consistency + evidence cap) still applies: it is part of # how block evidence is graded, not part of the 0.3/0.7 channel split. final_score = block_sim * block_multiplier elif block_evidence_status == "absent": # Symmetric case (degenerate in practice): renormalize onto the # transform channel, keeping the transform evidence cap. final_score = transform_sim * transform_multiplier else: # Both channels have evidence: the fixed 0.3/0.7 split is unchanged. final_score = (0.3 * transform_sim * transform_multiplier + 0.7 * block_sim * block_multiplier) legacy_score = ( 0.3 * compare_transforms_legacy(hashset1["transforms"], hashset2["transforms"]) + 0.7 * compare_blocks_legacy(hashset1["blocks"], hashset2["blocks"]) ) # Calibrated confidence: the new primary score. Percentile of the # unrelated-pair baseline distribution (see BASELINE CALIBRATION above). calibration = load_calibration() calibrated = calibrated_confidence(final_score, calibration=calibration) return { "transform_similarity": round(transform_sim, 2), "block_similarity": round(block_sim, 2), "final_similarity": round(final_score, 2), "raw_similarity": round(final_score, 2), "raw_similarity_legacy": round(legacy_score, 2), "calibrated_confidence": round(calibrated, 2), "calibration_loaded": calibration is not None, "background_coincidence_suppressed": background_coincidence, "transform_consistency": round(consistency, 4), "dominant_transform": dominant_transform, "transform_agreements": len(transform_result["agreements"]), "block_agreements": len(block_result["agreements"]), "content_blocks_used": block_result["content_blocks_used"], "low_content_warning": block_result["low_content_warning"], "best_scale_pair": block_result.get("best_scale_pair"), "transform_evidence_status": transform_evidence_status, "block_evidence_status": block_evidence_status, } # ----------------------------- # GLOBAL PRE-FILTER (performance only) # ----------------------------- def global_prefilter_distance(upload_hashes, db_hashes): """ Cheap rotation-aware global triage distance. Single non-blocked phash comparison: the query's orientation-0 phash against ALL SIX precomputed transform-variant phashes of the database row (already stored, so this is six Hamming comparisons - no re-hashing). Comparing all six keeps rotation-robustness: a genuinely rotated copy is close to one of the variants and must never be triaged away. Returns MAX_DISTANCE when the payloads are not comparable. """ try: query_phash = upload_hashes["transforms"]["0"]["phash"] except (KeyError, TypeError): return MAX_DISTANCE best = MAX_DISTANCE for variant in (db_hashes.get("transforms") or {}).values(): candidate = variant.get("phash") if isinstance(variant, dict) else None if not candidate: continue d = hamming(query_phash, candidate) if d < best: best = d return best def _match_fields(result): """Score fields shared by every match object (DB and internet results).""" fallback_used = legacy_fallback_applies(result) return { "similarity": result["final_similarity"], "raw_similarity": result["raw_similarity"], "raw_similarity_legacy": result["raw_similarity_legacy"], "calibrated_confidence": result["calibrated_confidence"], "transform_consistency": result["transform_consistency"], "transform_agreements": result["transform_agreements"], "block_agreements": result["block_agreements"], "content_blocks_used": result["content_blocks_used"], "low_content_warning": result["low_content_warning"], "transform_evidence_status": result["transform_evidence_status"], "block_evidence_status": result["block_evidence_status"], # Additive fallback signal: True when the consensus score is # (near-)zero but the legacy score is strong enough to drive # plagiarism ranking/flagging instead (see legacy_fallback_applies). "fallback_used": fallback_used, "effective_similarity": ( float(result["raw_similarity_legacy"]) if fallback_used else float(result["raw_similarity"]) ), } def _score_is_zero(value): """Treat missing/None/non-numeric/new-zero scores as zero.""" try: return float(value or 0.0) <= 0.0 except (TypeError, ValueError): return True def legacy_fallback_applies(match): """Whether the legacy fallback drives plagiarism decisions for a match. Option B gate: the consensus pipeline must score (near-)zero on similarity/raw_similarity while raw_similarity_legacy reaches LEGACY_FALLBACK_MIN_SCORE. Accepts both raw compare_hash_sets results and projected match dicts (tolerant to missing keys). """ try: similarity = match.get("similarity", match.get("final_similarity")) raw = match.get("raw_similarity") legacy = float(match.get("raw_similarity_legacy") or 0.0) except (TypeError, ValueError, AttributeError): return False return ( _score_is_zero(similarity) and _score_is_zero(raw) and legacy >= LEGACY_FALLBACK_MIN_SCORE ) def effective_score_for_plagiarism(match): """Score used for plagiarism ranking/flagging decisions. Returns raw_similarity_legacy when the Option B fallback gate applies (consensus zero + strong legacy), else the consensus score. Never rewrites the stored score fields; callers rank/flag on this value. """ try: if legacy_fallback_applies(match): return float(match.get("raw_similarity_legacy") or 0.0) return float( match.get( "raw_similarity", match.get("similarity", match.get("final_similarity")), ) or 0.0 ) except (TypeError, ValueError, AttributeError): return 0.0 def _ranking_score(match): """Ordering key for top-N match lists. Consensus matches rank by calibrated_confidence (existing behaviour). Fallback matches (consensus zero + strong legacy) rank by their effective_similarity so a 62-80 legacy hit surfaces instead of sinking to the bottom with the other zeros. Tuple second element keeps a deterministic order between the two groups on exact ties. """ try: if match.get("fallback_used") or legacy_fallback_applies(match): legacy = match.get( "effective_similarity", match.get("raw_similarity_legacy") ) return (float(legacy or 0.0), 1) return (float(match.get("calibrated_confidence") or 0.0), 0) except (TypeError, ValueError, AttributeError): return (0.0, 0) def scan_database(upload_hashes, artworks): """ Scan registered artworks and return the top 5 matches. Rows obviously unrelated to the query are skipped by the cheap global phash pre-filter (all six transform-variant distances above GLOBAL_PREFILTER_MAX_DISTANCE) before running the full block/transform/consensus pipeline. The pre-filter is performance-only: it never sets or influences the scores of rows that pass it. """ matches = [] skipped = 0 for art in artworks: db_hashes = art["perceptual_hash"] if global_prefilter_distance(upload_hashes, db_hashes) > GLOBAL_PREFILTER_MAX_DISTANCE: skipped += 1 continue result = compare_hash_sets(upload_hashes, db_hashes) match = {"artwork_id": art["id"], "url": art["c_secure_url"]} match.update(_match_fields(result)) matches.append(match) if skipped: print(f"Prefilter skipped {skipped}/{len(artworks)} " f"obviously-unrelated database rows") # Rank by the plagiarism decision score: consensus calibrated_confidence # normally, legacy score when the zero-consensus fallback applies. matches.sort(key=lambda m: _ranking_score(m), reverse=True) return matches[:5] # Candidate keys Serper/SerpAPI may use for the image-result list and URL. _SERP_LIST_KEYS = ("organic", "images", "similarImages", "results", "items") _SERP_URL_KEYS = ("imageUrl", "image_url", "thumbnailUrl", "thumbnail_url", "url") def _extract_serp_items(payload): """Tolerantly extract the image-result list from varying Serper shapes.""" if isinstance(payload, list): return payload, "list" if not isinstance(payload, dict): return [], "unknown" for key in _SERP_LIST_KEYS: val = payload.get(key) if isinstance(val, list): # Report "organic(empty)" instead of "none" when the key exists # but Serper returned zero candidates, so diagnostics can tell # "Serper found nothing" apart from "unexpected response shape". return val, key if val else f"{key}(empty)" # Fall back to first non-empty list value so error/empty shapes stay visible. for key, val in payload.items(): if isinstance(val, list) and val and isinstance(val[0], dict): return val, key return [], "none" def _extract_serp_image_url(item): for key in _SERP_URL_KEYS: val = item.get(key) if val: return val, key return None, None def generate_hash_from_serp_with_stats(payload, max_results=60, max_workers=8): """Schema-tolerant SERP hashing. Never raises on bad items. Returns (hashed_dict, stats) where stats carries diagnostics so the API can distinguish 'no online matches' from 'online check degraded'. """ payload_keys = ( list(payload.keys()) if isinstance(payload, dict) else [type(payload).__name__] ) raw_items, list_key = _extract_serp_items(payload or {}) items = (raw_items or [])[:max_results] hashed = {} failed = 0 failure_reasons = {} url_key_used = set() def _work(entry): idx, item = entry if not isinstance(item, dict): return idx, item, None, "non_dict_item", None img_url, found_key = _extract_serp_image_url(item) if not img_url: return idx, item, None, "missing_image_url", None h, reason = generate_hash_from_url(img_url) return idx, item, h, reason, found_key with ThreadPoolExecutor(max_workers=max_workers) as pool: futures = {pool.submit(_work, (i, it)): i for i, it in enumerate(items)} for future in as_completed(futures): try: i, item, h, reason, found_key = future.result() except Exception as e: # per-URL isolation: one bad item can't kill batch failed += 1 failure_reasons["worker_exception"] = ( failure_reasons.get("worker_exception", 0) + 1 ) print(f"Error hashing SERP item: {e}") continue if found_key: url_key_used.add(found_key) if h: hashed[str(i)] = { "hashes": h, "source": item.get("source"), "link": item.get("link"), "imageUrl": item.get("imageUrl") or item.get("image_url") or item.get("thumbnailUrl") or item.get("url"), } else: failed += 1 key = (reason or "unknown")[:80] failure_reasons[key] = failure_reasons.get(key, 0) + 1 stats = { "payload_keys": payload_keys, "list_key": list_key, "serp_returned": len(raw_items or []), "attempted": len(items), "hashed_ok": len(hashed), "hashed_failed": failed, "url_keys_seen": sorted(url_key_used), "failure_reasons": failure_reasons, } if failed: print(f"SERP hashing: {len(hashed)} ok / {failed} failed ({failure_reasons})") return hashed, stats def generate_hash_from_serp(payload, max_results=60, max_workers=8): """Backwards-compatible wrapper: returns just the hashed dict.""" hashed, _ = generate_hash_from_serp_with_stats( payload, max_results=max_results, max_workers=max_workers ) return hashed def check_distance_between_serp(uploaded, searched): """Compare the upload against hashed SERP images; return the top 5.""" results = [] for url, image_data in searched.items(): result = compare_hash_sets(uploaded, image_data["hashes"]) match = { "source": image_data["source"], "link": image_data["link"], "url": image_data["imageUrl"], } match.update(_match_fields(result)) results.append(match) # Rank by the plagiarism decision score (fallback-aware). results.sort(key=lambda m: _ranking_score(m), reverse=True) return results[:5]
import os
from io import BytesIO
from typing import Dict, Any
import json
import math
import os
import requests
from PIL import Image, ImageOps, ImageFilter
import imagehash
import numpy as np
from concurrent.futures import ThreadPoolExecutor, as_completed
HASH_SIZE = 8
MAX_DISTANCE = HASH_SIZE * HASH_SIZE
# -----------------------------
# TUNABLE THRESHOLDS
# -----------------------------
# Every threshold below is provisional until baseline calibration data exists
# (see scripts/calibrate_baseline.py). Each one can be overridden via an
# environment variable without a code change.
def _env_float(name, default):
try:
return float(os.environ.get(name, default))
except (TypeError, ValueError):
return default
# Minimum stddev of a preprocessed grayscale block for it to count as
# "content-bearing". Flat regions (solid backgrounds) hash to near-identical
# "empty" fingerprints regardless of the real artwork, so blocks below this
# threshold are excluded from block matching. Tune against calibration data.
MIN_BLOCK_STDDEV = _env_float("PLAGIARISM_MIN_BLOCK_STDDEV", 5.0)
# Maximum weighted hash distance (0..MAX_DISTANCE) for a block/transform pair
# to count as an "agreement" in consensus scoring. Pairs farther apart than
# this are treated as non-matching instead of dragging the average.
STRICT_MATCH_DISTANCE = _env_float("PLAGIARISM_STRICT_MATCH_DISTANCE", 12.0)
# Number of agreeing blocks required before full confidence is granted. A
# single lucky matching block cannot establish geometric consistency on its
# own, so the confidence multiplier is capped until this many blocks agree.
# Never demands more agreeing blocks than the image actually has content
# blocks (sparse/minimalist artwork must not be over-suppressed).
MIN_CONSENSUS_BLOCKS = int(_env_float("PLAGIARISM_MIN_CONSENSUS_BLOCKS", 2))
# Maximum Hamming distance (0..MAX_DISTANCE) on the cheap global phash
# pre-filter. scan_database skips a DB row before the expensive pipeline only
# when the query phash is farther than this from ALL SIX precomputed
# transform variants of the row (rotation-aware: a rotated copy matches one
# of the variants closely and is never skipped). Performance-only; never
# influences the score of rows that pass.
GLOBAL_PREFILTER_MAX_DISTANCE = int(
_env_float("PLAGIARISM_GLOBAL_PREFILTER_MAX_DISTANCE", 28))
# Minimum legacy score that arms the zero-consensus fallback (Option B gate).
# When the consensus pipeline scores a pair at (near-)zero but the legacy
# single-minimum algorithm scores at or above this, the legacy value is used
# for plagiarism *ranking/flagging* decisions. Below this, a zero-consensus
# pair stays a clean negative so the legacy algorithm's known background
# inflation (unrelated baseline mean ~61.8, p95 ~66.9) does not flag
# everything. Overridable via PLAGIARISM_LEGACY_FALLBACK_MIN_SCORE.
LEGACY_FALLBACK_MIN_SCORE = _env_float("PLAGIARISM_LEGACY_FALLBACK_MIN_SCORE", 60.0)
# Location of the baseline calibration config written by
# scripts/calibrate_baseline.py. Overridable for tests/deployments.
_REPO_ROOT = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
CALIBRATION_CONFIG_PATH = os.environ.get(
"PLAGIARISM_CALIBRATION_CONFIG",
os.path.join(_REPO_ROOT, "config", "calibration.json"),
)
# -----------------------------
# MULTI-SCALE BLOCK MATCHING
# -----------------------------
# Scale factors at which each image is block-hashed. A zoomed/re-cropped
# copy of an artwork can then align with a differently-scaled version of
# the original instead of requiring exact pixel-level quadrant alignment.
# Keep to 3 scales initially: more scales directly multiply the comparison
# count (5 blocks x N scales x 5 blocks x N scales) against the consensus
# design's goal of controlling that count.
SCALE_FACTORS = [0.625, 0.75, 1.0]
# -----------------------------
# IMAGE PREPROCESSING
# -----------------------------
def preprocess_image(img: Image.Image):
if img.mode in ("RGBA", "LA"):
bg = Image.new("RGB", img.size, (255, 255, 255))
bg.paste(img, mask=img.split()[-1])
img = bg
else:
img = img.convert("RGB")
img = img.convert("L")
img = img.filter(ImageFilter.GaussianBlur(radius=1))
return img