File size: 7,685 Bytes
59839e3 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 151 152 153 154 155 156 157 | #!/usr/bin/env python3
"""Extend the audited 700-image dataset and lock 32 of 100 new images."""
from __future__ import annotations
import argparse
import csv
import hashlib
import json
import os
import shutil
from pathlib import Path
import numpy as np
from iterstrat.ml_stratifiers import MultilabelStratifiedShuffleSplit
from dataset_utils import read_rows
from prepare_data import count_rows, phash, write_manifest
def copy_verified(source: Path, destination: Path) -> None:
if destination.exists():
if hashlib.sha256(destination.read_bytes()).digest() != hashlib.sha256(source.read_bytes()).digest():
raise ValueError(f"stale combined image: {destination}")
return
shutil.copy2(source, destination)
def main() -> None:
parser = argparse.ArgumentParser()
parser.add_argument("--prior-dataset", type=Path, required=True)
parser.add_argument("--prior-splits", type=Path, required=True)
parser.add_argument("--new", type=Path, required=True)
parser.add_argument("--combined", type=Path, required=True)
parser.add_argument("--splits", type=Path, required=True)
# iterative-stratification may round the requested size for some seeds;
# this audited seed yields exactly 68/32 while preserving both label and
# label-cardinality balance. The explicit size assertion below is binding.
parser.add_argument("--seed", type=int, default=800001)
args = parser.parse_args()
prior_labels = {
row["file_name"]: row["characters"]
for row in read_rows(args.prior_dataset / "human-tagged.txt.txt")
}
prior_rows = read_rows(args.prior_splits / "all700.csv")
prior_development = read_rows(args.prior_splits / "development648.csv")
prior_holdout20 = read_rows(args.prior_splits / "new_holdout20.csv")
historical_holdout = read_rows(args.prior_splits / "historical_holdout32.csv")
if tuple(map(len, (prior_rows, prior_development, prior_holdout20, historical_holdout))) != (700, 648, 20, 32):
raise ValueError("expected prior all/development/new holdout/historical sizes 700/648/20/32")
for row in prior_rows:
if prior_labels.get(row["file_name"]) != row["characters"]:
raise ValueError(f"prior label mismatch: {row['file_name']}")
row["source_path"] = str((args.prior_dataset / row["file_name"]).resolve())
new_rows = read_rows(args.new / "human-tagged.txt.txt")
if len(new_rows) != 100:
raise ValueError(f"expected 100 new images, got {len(new_rows)}")
for row in new_rows:
row["origin"] = "incremental100"
row["source_path"] = str((args.new / row["file_name"]).resolve())
if not Path(row["source_path"]).is_file():
raise FileNotFoundError(row["source_path"])
all_rows = prior_rows + new_rows
if len({row["file_name"] for row in all_rows}) != 800:
raise ValueError("duplicate filenames across the 800-image dataset")
classes = sorted({label for row in all_rows for label in row["characters"].split("|")})
class_index = {name: index for index, name in enumerate(classes)}
y = np.zeros((len(new_rows), len(classes) + 3), dtype=np.uint8)
for i, row in enumerate(new_rows):
labels = row["characters"].split("|")
for label in labels:
y[i, class_index[label]] = 1
y[i, len(classes) + len(labels) - 1] = 1
splitter = MultilabelStratifiedShuffleSplit(n_splits=1, test_size=32, random_state=args.seed)
train_idx, test_idx = next(splitter.split(np.zeros(len(new_rows)), y))
new_train = [new_rows[i] for i in sorted(train_idx)]
new_holdout = [new_rows[i] for i in sorted(test_idx)]
# The previous fresh 20-image holdout is explicitly promoted into training.
development = prior_development + prior_holdout20 + new_train
if tuple(map(len, (development, new_holdout, historical_holdout))) != (736, 32, 32):
raise ValueError("unexpected development/new holdout/historical sizes")
development_names = {row["file_name"] for row in development}
holdout_names = {row["file_name"] for row in new_holdout}
historical_names = {row["file_name"] for row in historical_holdout}
if development_names & (holdout_names | historical_names) or holdout_names & historical_names:
raise ValueError("split leakage detected")
if development_names | holdout_names | historical_names != {row["file_name"] for row in all_rows}:
raise ValueError("split union does not cover the 800-image dataset")
args.combined.mkdir(parents=True, exist_ok=True)
for row in all_rows:
copy_verified(Path(row["source_path"]), args.combined / row["file_name"])
with (args.combined / "human-tagged.txt.txt").open("w", newline="", encoding="utf-8") as handle:
writer = csv.DictWriter(handle, fieldnames=["file_name", "characters", "index"])
writer.writeheader()
for index, row in enumerate(all_rows):
writer.writerow({"file_name": row["file_name"], "characters": row["characters"], "index": index})
write_manifest(args.splits / "all800.csv", all_rows)
write_manifest(args.splits / "development736.csv", development)
write_manifest(args.splits / "historical_holdout32.csv", historical_holdout)
write_manifest(args.splits / "new_holdout32.csv", new_holdout)
write_manifest(args.splits / "new_holdout32_public.csv", new_holdout, labels=False)
paths = {row["file_name"]: Path(row["source_path"]) for row in all_rows}
sha_groups: dict[str, list[str]] = {}
file_hashes: dict[str, str] = {}
hashes: dict[str, int] = {}
for name, path in paths.items():
digest = hashlib.sha256(path.read_bytes()).hexdigest()
file_hashes[name] = digest
sha_groups.setdefault(digest, []).append(name)
hashes[name] = phash(path)
exact_duplicates = [group for group in sha_groups.values() if len(group) > 1]
(args.splits / "image_sha256.json").write_text(
json.dumps(file_hashes, sort_keys=True, indent=2) + "\n", encoding="utf-8"
)
nearest = []
for row in new_holdout:
name = row["file_name"]
neighbor, distance = min(
((candidate, (hashes[name] ^ hashes[candidate]).bit_count()) for candidate in development_names),
key=lambda item: item[1],
)
nearest.append({"holdout": name, "nearest_development": neighbor, "phash_hamming": distance})
report = {
"seed": args.seed,
"classes": classes,
"all": count_rows(all_rows, classes),
"development": count_rows(development, classes),
"promoted_prior_holdout": count_rows(prior_holdout20, classes),
"incremental_train": count_rows(new_train, classes),
"new_holdout": count_rows(new_holdout, classes),
"historical_holdout": count_rows(historical_holdout, classes),
"exact_duplicate_groups": exact_duplicates,
"minimum_new_holdout_to_development_phash_distance": min(item["phash_hamming"] for item in nearest),
"nearest_new_holdout_pairs": sorted(nearest, key=lambda item: item["phash_hamming"]),
"policy": (
"Train/select only on development736 (prior development648 + promoted prior holdout20 + "
"68 incremental images). Evaluate new_holdout32 once. Keep historical_holdout32 excluded "
"as a non-tuning reference."
),
}
args.splits.mkdir(parents=True, exist_ok=True)
(args.splits / "split_report.json").write_text(
json.dumps(report, ensure_ascii=False, indent=2) + "\n", encoding="utf-8"
)
print(json.dumps(report, ensure_ascii=False, indent=2))
if __name__ == "__main__":
main()
|