Detect duplicate images during migration

app.py

@@ -108,6 +108,9 @@ PREVIEW_CACHE_SIZE = 256
 PREVIEW_QUALITY = 70
 PREVIEW_SIZE = 320
 IMAGE_KEY_CACHE_SIZE = 512
+DUPLICATE_HASH_SIZE = 16
+DUPLICATE_HASH_DISTANCE = 24
+DUPLICATE_COLOR_DISTANCE = 24
 DEFAULT_RETURN_SCALE = 1
 DEFAULT_BATCH_SIZE = 1
 DEFAULT_UI_BATCH_SIZE = 1
@@ -333,21 +336,30 @@ def cleanup_mount():
                 path.unlink()
                 log(f"Deleted img/{path.relative_to(IMAGE_DIR)}")
         if HAS_DATA_MOUNT:
-            pngs = list(IMAGE_DIR.glob("????-??-??/*.png"))
-            duplicates = {
-                path.with_suffix(".png")
-                for path in IMAGE_DIR.glob("????-??-??/*.epng")
-            }
-            for path in pngs:
-                if path in duplicates:
-                    path.unlink()
-                    log(f"Deleted img/{path.relative_to(IMAGE_DIR)}")
-            for path in pngs:
-                if path in duplicates:
-                    continue
+            fingerprints = []
+            for path in IMAGE_DIR.rglob("*.epng"):
+                with Image.open(BytesIO(stored_bytes(path))) as image:
+                    fingerprints.append(image_fingerprint(image))
+            for path in IMAGE_DIR.glob("????-??-??/*.png"):
                 with Image.open(path) as image:
-                    save_named_image(image, path.with_suffix(".epng"))
+                    fingerprint = image_fingerprint(image)
+                    duplicate = any(
+                        (fingerprint[0] ^ known[0]).bit_count()
+                        <= DUPLICATE_HASH_DISTANCE
+                        and sum(
+                            abs(left - right)
+                            for left, right in zip(fingerprint[1], known[1])
+                        )
+                        <= DUPLICATE_COLOR_DISTANCE
+                        for known in fingerprints
+                    )
+                    if not duplicate:
+                        save_named_image(image, path.with_suffix(".epng"))
                 path.unlink()
+                if duplicate:
+                    log(f"Deleted duplicate img/{path.relative_to(IMAGE_DIR)}")
+                else:
+                    fingerprints.append(fingerprint)
 
 
 def download(url, target):
@@ -1988,6 +2000,20 @@ def archive_image(image, from_api):
         log(f"Archive failed: {error}")
 
 
+def image_fingerprint(image):
+    pixels = np.asarray(
+        ImageOps.exif_transpose(image).convert("RGB").resize(
+            (DUPLICATE_HASH_SIZE + 1, DUPLICATE_HASH_SIZE),
+            Image.Resampling.LANCZOS,
+        )
+    )
+    differences = pixels[:, 1:] > pixels[:, :-1]
+    return (
+        int.from_bytes(np.packbits(differences).tobytes()),
+        tuple(int(value) for value in pixels.mean(axis=(0, 1))),
+    )
+
+
 def image_png_bytes(image):
     pnginfo = PngInfo()
     for key, value in image.info.items():