#!/usr/bin/env python3 """Extend the audited 700-image dataset and lock 32 of 100 new images.""" from __future__ import annotations import argparse import csv import hashlib import json import os import shutil from pathlib import Path import numpy as np from iterstrat.ml_stratifiers import MultilabelStratifiedShuffleSplit from dataset_utils import read_rows from prepare_data import count_rows, phash, write_manifest def copy_verified(source: Path, destination: Path) -> None: if destination.exists(): if hashlib.sha256(destination.read_bytes()).digest() != hashlib.sha256(source.read_bytes()).digest(): raise ValueError(f"stale combined image: {destination}") return shutil.copy2(source, destination) def main() -> None: parser = argparse.ArgumentParser() parser.add_argument("--prior-dataset", type=Path, required=True) parser.add_argument("--prior-splits", type=Path, required=True) parser.add_argument("--new", type=Path, required=True) parser.add_argument("--combined", type=Path, required=True) parser.add_argument("--splits", type=Path, required=True) # iterative-stratification may round the requested size for some seeds; # this audited seed yields exactly 68/32 while preserving both label and # label-cardinality balance. The explicit size assertion below is binding. parser.add_argument("--seed", type=int, default=800001) args = parser.parse_args() prior_labels = { row["file_name"]: row["characters"] for row in read_rows(args.prior_dataset / "human-tagged.txt.txt") } prior_rows = read_rows(args.prior_splits / "all700.csv") prior_development = read_rows(args.prior_splits / "development648.csv") prior_holdout20 = read_rows(args.prior_splits / "new_holdout20.csv") historical_holdout = read_rows(args.prior_splits / "historical_holdout32.csv") if tuple(map(len, (prior_rows, prior_development, prior_holdout20, historical_holdout))) != (700, 648, 20, 32): raise ValueError("expected prior all/development/new holdout/historical sizes 700/648/20/32") for row in prior_rows: if prior_labels.get(row["file_name"]) != row["characters"]: raise ValueError(f"prior label mismatch: {row['file_name']}") row["source_path"] = str((args.prior_dataset / row["file_name"]).resolve()) new_rows = read_rows(args.new / "human-tagged.txt.txt") if len(new_rows) != 100: raise ValueError(f"expected 100 new images, got {len(new_rows)}") for row in new_rows: row["origin"] = "incremental100" row["source_path"] = str((args.new / row["file_name"]).resolve()) if not Path(row["source_path"]).is_file(): raise FileNotFoundError(row["source_path"]) all_rows = prior_rows + new_rows if len({row["file_name"] for row in all_rows}) != 800: raise ValueError("duplicate filenames across the 800-image dataset") classes = sorted({label for row in all_rows for label in row["characters"].split("|")}) class_index = {name: index for index, name in enumerate(classes)} y = np.zeros((len(new_rows), len(classes) + 3), dtype=np.uint8) for i, row in enumerate(new_rows): labels = row["characters"].split("|") for label in labels: y[i, class_index[label]] = 1 y[i, len(classes) + len(labels) - 1] = 1 splitter = MultilabelStratifiedShuffleSplit(n_splits=1, test_size=32, random_state=args.seed) train_idx, test_idx = next(splitter.split(np.zeros(len(new_rows)), y)) new_train = [new_rows[i] for i in sorted(train_idx)] new_holdout = [new_rows[i] for i in sorted(test_idx)] # The previous fresh 20-image holdout is explicitly promoted into training. development = prior_development + prior_holdout20 + new_train if tuple(map(len, (development, new_holdout, historical_holdout))) != (736, 32, 32): raise ValueError("unexpected development/new holdout/historical sizes") development_names = {row["file_name"] for row in development} holdout_names = {row["file_name"] for row in new_holdout} historical_names = {row["file_name"] for row in historical_holdout} if development_names & (holdout_names | historical_names) or holdout_names & historical_names: raise ValueError("split leakage detected") if development_names | holdout_names | historical_names != {row["file_name"] for row in all_rows}: raise ValueError("split union does not cover the 800-image dataset") args.combined.mkdir(parents=True, exist_ok=True) for row in all_rows: copy_verified(Path(row["source_path"]), args.combined / row["file_name"]) with (args.combined / "human-tagged.txt.txt").open("w", newline="", encoding="utf-8") as handle: writer = csv.DictWriter(handle, fieldnames=["file_name", "characters", "index"]) writer.writeheader() for index, row in enumerate(all_rows): writer.writerow({"file_name": row["file_name"], "characters": row["characters"], "index": index}) write_manifest(args.splits / "all800.csv", all_rows) write_manifest(args.splits / "development736.csv", development) write_manifest(args.splits / "historical_holdout32.csv", historical_holdout) write_manifest(args.splits / "new_holdout32.csv", new_holdout) write_manifest(args.splits / "new_holdout32_public.csv", new_holdout, labels=False) paths = {row["file_name"]: Path(row["source_path"]) for row in all_rows} sha_groups: dict[str, list[str]] = {} file_hashes: dict[str, str] = {} hashes: dict[str, int] = {} for name, path in paths.items(): digest = hashlib.sha256(path.read_bytes()).hexdigest() file_hashes[name] = digest sha_groups.setdefault(digest, []).append(name) hashes[name] = phash(path) exact_duplicates = [group for group in sha_groups.values() if len(group) > 1] (args.splits / "image_sha256.json").write_text( json.dumps(file_hashes, sort_keys=True, indent=2) + "\n", encoding="utf-8" ) nearest = [] for row in new_holdout: name = row["file_name"] neighbor, distance = min( ((candidate, (hashes[name] ^ hashes[candidate]).bit_count()) for candidate in development_names), key=lambda item: item[1], ) nearest.append({"holdout": name, "nearest_development": neighbor, "phash_hamming": distance}) report = { "seed": args.seed, "classes": classes, "all": count_rows(all_rows, classes), "development": count_rows(development, classes), "promoted_prior_holdout": count_rows(prior_holdout20, classes), "incremental_train": count_rows(new_train, classes), "new_holdout": count_rows(new_holdout, classes), "historical_holdout": count_rows(historical_holdout, classes), "exact_duplicate_groups": exact_duplicates, "minimum_new_holdout_to_development_phash_distance": min(item["phash_hamming"] for item in nearest), "nearest_new_holdout_pairs": sorted(nearest, key=lambda item: item["phash_hamming"]), "policy": ( "Train/select only on development736 (prior development648 + promoted prior holdout20 + " "68 incremental images). Evaluate new_holdout32 once. Keep historical_holdout32 excluded " "as a non-tuning reference." ), } args.splits.mkdir(parents=True, exist_ok=True) (args.splits / "split_report.json").write_text( json.dumps(report, ensure_ascii=False, indent=2) + "\n", encoding="utf-8" ) print(json.dumps(report, ensure_ascii=False, indent=2)) if __name__ == "__main__": main()