v802-classifier / src /prepare_incremental_800.py
sneo's picture
Add v802 classifier head and source code
59839e3 verified
Raw History Blame Contribute Delete
7.69 kB
#!/usr/bin/env python3
"""Extend the audited 700-image dataset and lock 32 of 100 new images."""
from __future__ import annotations
import argparse
import csv
import hashlib
import json
import os
import shutil
from pathlib import Path
import numpy as np
from iterstrat.ml_stratifiers import MultilabelStratifiedShuffleSplit
from dataset_utils import read_rows
from prepare_data import count_rows, phash, write_manifest
def copy_verified(source: Path, destination: Path) -> None:
if destination.exists():
if hashlib.sha256(destination.read_bytes()).digest() != hashlib.sha256(source.read_bytes()).digest():
raise ValueError(f"stale combined image: {destination}")
return
shutil.copy2(source, destination)
def main() -> None:
parser = argparse.ArgumentParser()
parser.add_argument("--prior-dataset", type=Path, required=True)
parser.add_argument("--prior-splits", type=Path, required=True)
parser.add_argument("--new", type=Path, required=True)
parser.add_argument("--combined", type=Path, required=True)
parser.add_argument("--splits", type=Path, required=True)
# iterative-stratification may round the requested size for some seeds;
# this audited seed yields exactly 68/32 while preserving both label and
# label-cardinality balance. The explicit size assertion below is binding.
parser.add_argument("--seed", type=int, default=800001)
args = parser.parse_args()
prior_labels = {
row["file_name"]: row["characters"]
for row in read_rows(args.prior_dataset / "human-tagged.txt.txt")
}
prior_rows = read_rows(args.prior_splits / "all700.csv")
prior_development = read_rows(args.prior_splits / "development648.csv")
prior_holdout20 = read_rows(args.prior_splits / "new_holdout20.csv")
historical_holdout = read_rows(args.prior_splits / "historical_holdout32.csv")
if tuple(map(len, (prior_rows, prior_development, prior_holdout20, historical_holdout))) != (700, 648, 20, 32):
raise ValueError("expected prior all/development/new holdout/historical sizes 700/648/20/32")
for row in prior_rows:
if prior_labels.get(row["file_name"]) != row["characters"]:
raise ValueError(f"prior label mismatch: {row['file_name']}")
row["source_path"] = str((args.prior_dataset / row["file_name"]).resolve())
new_rows = read_rows(args.new / "human-tagged.txt.txt")
if len(new_rows) != 100:
raise ValueError(f"expected 100 new images, got {len(new_rows)}")
for row in new_rows:
row["origin"] = "incremental100"
row["source_path"] = str((args.new / row["file_name"]).resolve())
if not Path(row["source_path"]).is_file():
raise FileNotFoundError(row["source_path"])
all_rows = prior_rows + new_rows
if len({row["file_name"] for row in all_rows}) != 800:
raise ValueError("duplicate filenames across the 800-image dataset")
classes = sorted({label for row in all_rows for label in row["characters"].split("|")})
class_index = {name: index for index, name in enumerate(classes)}
y = np.zeros((len(new_rows), len(classes) + 3), dtype=np.uint8)
for i, row in enumerate(new_rows):
labels = row["characters"].split("|")
for label in labels:
y[i, class_index[label]] = 1
y[i, len(classes) + len(labels) - 1] = 1
splitter = MultilabelStratifiedShuffleSplit(n_splits=1, test_size=32, random_state=args.seed)
train_idx, test_idx = next(splitter.split(np.zeros(len(new_rows)), y))
new_train = [new_rows[i] for i in sorted(train_idx)]
new_holdout = [new_rows[i] for i in sorted(test_idx)]
# The previous fresh 20-image holdout is explicitly promoted into training.
development = prior_development + prior_holdout20 + new_train
if tuple(map(len, (development, new_holdout, historical_holdout))) != (736, 32, 32):
raise ValueError("unexpected development/new holdout/historical sizes")
development_names = {row["file_name"] for row in development}
holdout_names = {row["file_name"] for row in new_holdout}
historical_names = {row["file_name"] for row in historical_holdout}
if development_names & (holdout_names | historical_names) or holdout_names & historical_names:
raise ValueError("split leakage detected")
if development_names | holdout_names | historical_names != {row["file_name"] for row in all_rows}:
raise ValueError("split union does not cover the 800-image dataset")
args.combined.mkdir(parents=True, exist_ok=True)
for row in all_rows:
copy_verified(Path(row["source_path"]), args.combined / row["file_name"])
with (args.combined / "human-tagged.txt.txt").open("w", newline="", encoding="utf-8") as handle:
writer = csv.DictWriter(handle, fieldnames=["file_name", "characters", "index"])
writer.writeheader()
for index, row in enumerate(all_rows):
writer.writerow({"file_name": row["file_name"], "characters": row["characters"], "index": index})
write_manifest(args.splits / "all800.csv", all_rows)
write_manifest(args.splits / "development736.csv", development)
write_manifest(args.splits / "historical_holdout32.csv", historical_holdout)
write_manifest(args.splits / "new_holdout32.csv", new_holdout)
write_manifest(args.splits / "new_holdout32_public.csv", new_holdout, labels=False)
paths = {row["file_name"]: Path(row["source_path"]) for row in all_rows}
sha_groups: dict[str, list[str]] = {}
file_hashes: dict[str, str] = {}
hashes: dict[str, int] = {}
for name, path in paths.items():
digest = hashlib.sha256(path.read_bytes()).hexdigest()
file_hashes[name] = digest
sha_groups.setdefault(digest, []).append(name)
hashes[name] = phash(path)
exact_duplicates = [group for group in sha_groups.values() if len(group) > 1]
(args.splits / "image_sha256.json").write_text(
json.dumps(file_hashes, sort_keys=True, indent=2) + "\n", encoding="utf-8"
)
nearest = []
for row in new_holdout:
name = row["file_name"]
neighbor, distance = min(
((candidate, (hashes[name] ^ hashes[candidate]).bit_count()) for candidate in development_names),
key=lambda item: item[1],
)
nearest.append({"holdout": name, "nearest_development": neighbor, "phash_hamming": distance})
report = {
"seed": args.seed,
"classes": classes,
"all": count_rows(all_rows, classes),
"development": count_rows(development, classes),
"promoted_prior_holdout": count_rows(prior_holdout20, classes),
"incremental_train": count_rows(new_train, classes),
"new_holdout": count_rows(new_holdout, classes),
"historical_holdout": count_rows(historical_holdout, classes),
"exact_duplicate_groups": exact_duplicates,
"minimum_new_holdout_to_development_phash_distance": min(item["phash_hamming"] for item in nearest),
"nearest_new_holdout_pairs": sorted(nearest, key=lambda item: item["phash_hamming"]),
"policy": (
"Train/select only on development736 (prior development648 + promoted prior holdout20 + "
"68 incremental images). Evaluate new_holdout32 once. Keep historical_holdout32 excluded "
"as a non-tuning reference."
),
}
args.splits.mkdir(parents=True, exist_ok=True)
(args.splits / "split_report.json").write_text(
json.dumps(report, ensure_ascii=False, indent=2) + "\n", encoding="utf-8"
)
print(json.dumps(report, ensure_ascii=False, indent=2))
if __name__ == "__main__":
main()