Spaces:
Runtime error
Runtime error
| """ | |
| duplicate_detector.py | |
| ---------------------- | |
| Two layers of duplicate detection: | |
| 1. Exact duplicates -- SHA256 of the raw PNG-encoded pixel bytes. | |
| 2. Visual (near) duplicates -- a perceptual hash (dHash, 64-bit) with | |
| Hamming-distance comparison. dHash is used instead of average-hash | |
| because it is far more robust to the flat color blocks typical of | |
| pixel art (aHash tends to false-positive heavily on sprites that | |
| share a dominant background/body color). | |
| No external `imagehash` dependency is required -- the hash is implemented | |
| directly with numpy/Pillow (both already required elsewhere in this | |
| project), which also keeps the Space's dependency footprint smaller. | |
| """ | |
| from __future__ import annotations | |
| from dataclasses import dataclass | |
| import numpy as np | |
| from PIL import Image | |
| from config import PHASH_DUPLICATE_DISTANCE_FRACTION, PHASH_SIZE | |
| from image_utils import sha256_bytes | |
| TOTAL_HASH_BITS = PHASH_SIZE * PHASH_SIZE + 4 * 4 * 3 # structural + color bits | |
| class DuplicateMatch: | |
| sprite_id: str | |
| exact: bool | |
| hamming_distance: int | |
| similarity_percent: float | |
| def compute_sha256(png_bytes: bytes) -> str: | |
| return sha256_bytes(png_bytes) | |
| def compute_dhash(rgba: np.ndarray, hash_size: int = PHASH_SIZE) -> str: | |
| """ | |
| Perceptual hash combining two signals so that visually distinct | |
| sprites of the same *shape* (e.g. two same-silhouette plants in | |
| different colors) are not conflated: | |
| 1. Structural dHash (grayscale gradient hash, `hash_size x hash_size` | |
| bits): captures the silhouette/shape/light-dark edge pattern. | |
| 2. Coarse color hash (4x4 average-color grid, 3 bits per cell -- | |
| one bit per RGB channel vs. that channel's sprite-wide median): | |
| captures roughly *which* colors dominate the sprite, cheaply. | |
| Both resizes are for HASH COMPUTATION ONLY -- never written to disk or | |
| used as the exported sprite -- so this does not violate the | |
| pixel-preservation rule, which applies to dataset *output*. | |
| Returns a hex string encoding the combined bit string. | |
| """ | |
| if rgba.shape[0] == 0 or rgba.shape[1] == 0: | |
| return "0" * ((hash_size * hash_size + 4 * 4 * 3) // 4 + 1) | |
| img = Image.fromarray(rgba, mode="RGBA") | |
| background = Image.new("RGBA", img.size, (128, 128, 128, 255)) | |
| composited = Image.alpha_composite(background, img) | |
| # --- structural component (grayscale gradient hash) --- | |
| gray = composited.convert("L") | |
| small_gray = gray.resize((hash_size + 1, hash_size), resample=Image.NEAREST) | |
| gray_pixels = np.asarray(small_gray, dtype=np.int16) | |
| structural_bits = (gray_pixels[:, 1:] > gray_pixels[:, :-1]).flatten() | |
| # --- color component (coarse 4x4 average-color grid, absolute-referenced) --- | |
| # Compared against a fixed midpoint (128) rather than the sprite's own | |
| # median: a self-referenced median captures "brighter/darker than this | |
| # sprite's own average", which duplicates the structural hash's job and | |
| # collapses to the identical bit pattern for e.g. a green sprite and a | |
| # magenta sprite of the same silhouette on the same background (their | |
| # own two-tone medians make every foreground pixel "above median" and | |
| # every background pixel "below" regardless of which color it actually | |
| # is). An absolute reference keeps the hash sensitive to hue and channel | |
| # balance, which is exactly the visual duplicate signal color is meant | |
| # to add on top of the structural hash. | |
| rgb_img = composited.convert("RGB") | |
| grid = 4 | |
| small_color = rgb_img.resize((grid, grid), resample=Image.NEAREST) | |
| color_pixels = np.asarray(small_color, dtype=np.int16).reshape(-1, 3) # (16, 3) | |
| color_bits = (color_pixels > 128).flatten() # 16*3 = 48 bits | |
| all_bits = np.concatenate([structural_bits, color_bits]) | |
| value = 0 | |
| for bit in all_bits: | |
| value = (value << 1) | int(bit) | |
| hex_digits = (len(all_bits) + 3) // 4 | |
| return format(value, f"0{hex_digits}x") | |
| def hamming_distance(hash_a: str, hash_b: str) -> int: | |
| int_a = int(hash_a, 16) | |
| int_b = int(hash_b, 16) | |
| return bin(int_a ^ int_b).count("1") | |
| def find_duplicates( | |
| sha256_hash: str, | |
| dhash: str, | |
| existing: list[dict], | |
| distance_threshold: int | None = None, | |
| ) -> list[DuplicateMatch]: | |
| """ | |
| `existing` is a list of dicts with at least {"id", "sha256", "phash"} | |
| representing already-accepted or already-staged sprites to compare | |
| against. `distance_threshold` defaults to | |
| PHASH_DUPLICATE_DISTANCE_FRACTION * TOTAL_HASH_BITS when not given. | |
| """ | |
| if distance_threshold is None: | |
| distance_threshold = round(PHASH_DUPLICATE_DISTANCE_FRACTION * TOTAL_HASH_BITS) | |
| matches: list[DuplicateMatch] = [] | |
| for item in existing: | |
| if item.get("sha256") == sha256_hash: | |
| matches.append(DuplicateMatch(sprite_id=item["id"], exact=True, hamming_distance=0, similarity_percent=100.0)) | |
| continue | |
| other_hash = item.get("phash") | |
| if not other_hash: | |
| continue | |
| dist = hamming_distance(dhash, other_hash) | |
| if dist <= distance_threshold: | |
| similarity = 100.0 * (1 - dist / max(1, TOTAL_HASH_BITS)) | |
| matches.append( | |
| DuplicateMatch(sprite_id=item["id"], exact=False, hamming_distance=dist, similarity_percent=similarity) | |
| ) | |
| matches.sort(key=lambda m: (not m.exact, m.hamming_distance)) | |
| return matches | |