#!/usr/bin/env python3 """Build an audited 700-image dataset and freeze a fresh 20-image holdout.""" from __future__ import annotations import argparse import csv import hashlib import json import os import shutil from collections import Counter from pathlib import Path a import numpy as np from PIL import Image from iterstrat.ml_stratifiers import MultilabelStratifiedShuffleSplit from dataset_utils import read_rows def phash(path: Path, size: int = 32, low: int = 8) -> int: with Image.open(path) as image: array = np.asarray( image.convert("L").resize((size, size), Image.Resampling.LANCZOS), dtype=np.float64 ) n = np.arange(size) k = n[:, None] basis = np.cos(np.pi * (n + 0.5) * k / size) basis[0] /= np.sqrt(2) dct = (2 / size) * basis @ array @ basis.T block = dct[:low, :low] median = np.median(block.ravel()[1:]) value = 0 for bit in block.ravel() >= median: value = (value << 1) | int(bit) return value def add_origin(rows: list[dict[str, str]], root: Path, origin: str) -> list[dict[str, str]]: result = [] for row in rows: path = root / row["file_name"] if not path.is_file(): raise FileNotFoundError(path) result.append({ "file_name": row["file_name"], "characters": row["characters"], "origin": origin, "source_path": str(path.resolve()), }) return result def write_manifest(path: Path, rows: list[dict[str, str]], labels: bool = True) -> None: fields = ["file_name", "origin"] + (["characters"] if labels else []) path.parent.mkdir(parents=True, exist_ok=True) with path.open("w", newline="", encoding="utf-8") as handle: writer = csv.DictWriter(handle, fieldnames=fields) writer.writeheader() for row in rows: writer.writerow({field: row[field] for field in fields}) def count_rows(rows: list[dict[str, str]], classes: list[str]) -> dict[str, object]: cardinality = Counter(len(row["characters"].split("|")) for row in rows) return { "samples": len(rows), "classes": { name: sum(name in row["characters"].split("|") for row in rows) for name in classes }, "cardinality": {str(k): cardinality[k] for k in (1, 2, 3)}, } def main() -> None: parser = argparse.ArgumentParser() parser.add_argument("--old", type=Path, required=True) parser.add_argument("--new", type=Path, required=True) parser.add_argument("--extra", type=Path, required=True) parser.add_argument("--v2-splits", type=Path, required=True) parser.add_argument("--combined", type=Path, required=True) parser.add_argument("--splits", type=Path, required=True) parser.add_argument("--seed", type=int, default=700288) args = parser.parse_args() old_rows = add_origin(read_rows(args.old / "human-tagged.txt.txt"), args.old, "old440") new_rows = add_origin(read_rows(args.new / "human-tagged.txt.txt"), args.new, "new160") extra_rows = add_origin(read_rows(args.extra / "human-tagged.txt.txt"), args.extra, "extra100") all_rows = old_rows + new_rows + extra_rows if (len(old_rows), len(new_rows), len(extra_rows)) != (440, 160, 100): raise ValueError("expected 440 + 160 + 100 images") if len({row["file_name"] for row in all_rows}) != 700: raise ValueError("duplicate filenames") labels_by_name = {row["file_name"]: row["characters"] for row in all_rows} origin_by_name = {row["file_name"]: row["origin"] for row in all_rows} prior_dev = read_rows(args.v2_splits / "development.csv") prior_holdout = read_rows(args.v2_splits / "holdout_locked_labels.csv") for row in prior_dev + prior_holdout: if labels_by_name.get(row["file_name"]) != row["characters"]: raise ValueError(f"v2 label mismatch: {row['file_name']}") classes = sorted({label for row in all_rows for label in row["characters"].split("|")}) class_index = {name: index for index, name in enumerate(classes)} y = np.zeros((len(extra_rows), len(classes) + 3), dtype=np.uint8) for i, row in enumerate(extra_rows): labels = row["characters"].split("|") for label in labels: y[i, class_index[label]] = 1 y[i, len(classes) + len(labels) - 1] = 1 splitter = MultilabelStratifiedShuffleSplit( n_splits=1, test_size=20, random_state=args.seed ) extra_dev_idx, extra_test_idx = next(splitter.split(np.zeros(len(extra_rows)), y)) if len(extra_test_idx) != 20: raise RuntimeError(f"expected 20 holdout rows, got {len(extra_test_idx)}") extra_dev = [extra_rows[i] for i in sorted(extra_dev_idx)] extra_holdout = [extra_rows[i] for i in sorted(extra_test_idx)] development = [ { "file_name": row["file_name"], "characters": row["characters"], "origin": origin_by_name[row["file_name"]], } for row in prior_dev ] + extra_dev historical_holdout = [ { "file_name": row["file_name"], "characters": row["characters"], "origin": origin_by_name[row["file_name"]], } for row in prior_holdout ] if len(development) != 648 or len(historical_holdout) != 32: raise ValueError("unexpected development or historical holdout size") args.combined.mkdir(parents=True, exist_ok=True) for row in all_rows: source = Path(row["source_path"]) destination = args.combined / row["file_name"] if destination.exists(): if hashlib.sha256(destination.read_bytes()).digest() != hashlib.sha256(source.read_bytes()).digest(): raise ValueError(f"stale combined image: {destination}") if os.path.samefile(source, destination): temporary = destination.with_suffix(".copying") shutil.copy2(source, temporary) os.replace(temporary, destination) else: shutil.copy2(source, destination) with (args.combined / "human-tagged.txt.txt").open("w", newline="", encoding="utf-8") as handle: writer = csv.DictWriter(handle, fieldnames=["file_name", "characters", "index"]) writer.writeheader() for index, row in enumerate(all_rows): writer.writerow({"file_name": row["file_name"], "characters": row["characters"], "index": index}) public_rows = [{"file_name": row["file_name"], "origin": row["origin"]} for row in extra_holdout] write_manifest(args.splits / "all700.csv", all_rows) write_manifest(args.splits / "development648.csv", development) write_manifest(args.splits / "historical_holdout32.csv", historical_holdout) write_manifest(args.splits / "new_holdout20.csv", extra_holdout) write_manifest(args.splits / "new_holdout20_public.csv", public_rows, labels=False) sha_groups: dict[str, list[str]] = {} hashes: dict[str, int] = {} path_by_name = {row["file_name"]: Path(row["source_path"]) for row in all_rows} for name, path in path_by_name.items(): sha_groups.setdefault(hashlib.sha256(path.read_bytes()).hexdigest(), []).append(name) hashes[name] = phash(path) exact_duplicates = [group for group in sha_groups.values() if len(group) > 1] file_hashes = { name: hashlib.sha256(path.read_bytes()).hexdigest() for name, path in path_by_name.items() } (args.splits / "image_sha256.json").write_text( json.dumps(file_hashes, sort_keys=True, indent=2) + "\n", encoding="utf-8" ) development_names = [row["file_name"] for row in development] nearest = [] for row in extra_holdout: name = row["file_name"] neighbor, distance = min( ((candidate, (hashes[name] ^ hashes[candidate]).bit_count()) for candidate in development_names), key=lambda item: item[1], ) nearest.append({"holdout": name, "nearest_development": neighbor, "phash_hamming": distance}) report = { "seed": args.seed, "classes": classes, "all": count_rows(all_rows, classes), "development": count_rows(development, classes), "new_holdout": count_rows(extra_holdout, classes), "historical_holdout": count_rows(historical_holdout, classes), "exact_duplicate_groups": exact_duplicates, "minimum_new_holdout_to_development_phash_distance": min( item["phash_hamming"] for item in nearest ), "nearest_new_holdout_pairs": sorted(nearest, key=lambda item: item["phash_hamming"]), "policy": ( "Tune only on development648. Evaluate new_holdout20 once after model selection. " "Use historical_holdout32 only as a non-tuning reference to the published v2 result." ), } (args.splits / "split_report.json").write_text( json.dumps(report, ensure_ascii=False, indent=2) + "\n", encoding="utf-8" ) print(json.dumps(report, ensure_ascii=False, indent=2)) if __name__ == "__main__": main()