Download src/prepare_incremental_800.py from sneo/v802-classifier: direct link, hf CLI and curl.
- Browser
- Download file 7.69 kB
-
https://huggingface.co/sneo/v802-classifier/resolve/main/src/prepare_incremental_800.py
- Command line
-
hf download hf://sneo/v802-classifier/src/prepare_incremental_800.py
-
curl -L -o prepare_incremental_800.py https://huggingface.co/sneo/v802-classifier/resolve/main/src/prepare_incremental_800.py
7.69 kB
| #!/usr/bin/env python3 | |
| """Extend the audited 700-image dataset and lock 32 of 100 new images.""" | |
| from __future__ import annotations | |
| import argparse | |
| import csv | |
| import hashlib | |
| import json | |
| import os | |
| import shutil | |
| from pathlib import Path | |
| import numpy as np | |
| from iterstrat.ml_stratifiers import MultilabelStratifiedShuffleSplit | |
| from dataset_utils import read_rows | |
| from prepare_data import count_rows, phash, write_manifest | |
| def copy_verified(source: Path, destination: Path) -> None: | |
| if destination.exists(): | |
| if hashlib.sha256(destination.read_bytes()).digest() != hashlib.sha256(source.read_bytes()).digest(): | |
| raise ValueError(f"stale combined image: {destination}") | |
| return | |
| shutil.copy2(source, destination) | |
| def main() -> None: | |
| parser = argparse.ArgumentParser() | |
| parser.add_argument("--prior-dataset", type=Path, required=True) | |
| parser.add_argument("--prior-splits", type=Path, required=True) | |
| parser.add_argument("--new", type=Path, required=True) | |
| parser.add_argument("--combined", type=Path, required=True) | |
| parser.add_argument("--splits", type=Path, required=True) | |
| # iterative-stratification may round the requested size for some seeds; | |
| # this audited seed yields exactly 68/32 while preserving both label and | |
| # label-cardinality balance. The explicit size assertion below is binding. | |
| parser.add_argument("--seed", type=int, default=800001) | |
| args = parser.parse_args() | |
| prior_labels = { | |
| row["file_name"]: row["characters"] | |
| for row in read_rows(args.prior_dataset / "human-tagged.txt.txt") | |
| } | |
| prior_rows = read_rows(args.prior_splits / "all700.csv") | |
| prior_development = read_rows(args.prior_splits / "development648.csv") | |
| prior_holdout20 = read_rows(args.prior_splits / "new_holdout20.csv") | |
| historical_holdout = read_rows(args.prior_splits / "historical_holdout32.csv") | |
| if tuple(map(len, (prior_rows, prior_development, prior_holdout20, historical_holdout))) != (700, 648, 20, 32): | |
| raise ValueError("expected prior all/development/new holdout/historical sizes 700/648/20/32") | |
| for row in prior_rows: | |
| if prior_labels.get(row["file_name"]) != row["characters"]: | |
| raise ValueError(f"prior label mismatch: {row['file_name']}") | |
| row["source_path"] = str((args.prior_dataset / row["file_name"]).resolve()) | |
| new_rows = read_rows(args.new / "human-tagged.txt.txt") | |
| if len(new_rows) != 100: | |
| raise ValueError(f"expected 100 new images, got {len(new_rows)}") | |
| for row in new_rows: | |
| row["origin"] = "incremental100" | |
| row["source_path"] = str((args.new / row["file_name"]).resolve()) | |
| if not Path(row["source_path"]).is_file(): | |
| raise FileNotFoundError(row["source_path"]) | |
| all_rows = prior_rows + new_rows | |
| if len({row["file_name"] for row in all_rows}) != 800: | |
| raise ValueError("duplicate filenames across the 800-image dataset") | |
| classes = sorted({label for row in all_rows for label in row["characters"].split("|")}) | |
| class_index = {name: index for index, name in enumerate(classes)} | |
| y = np.zeros((len(new_rows), len(classes) + 3), dtype=np.uint8) | |
| for i, row in enumerate(new_rows): | |
| labels = row["characters"].split("|") | |
| for label in labels: | |
| y[i, class_index[label]] = 1 | |
| y[i, len(classes) + len(labels) - 1] = 1 | |
| splitter = MultilabelStratifiedShuffleSplit(n_splits=1, test_size=32, random_state=args.seed) | |
| train_idx, test_idx = next(splitter.split(np.zeros(len(new_rows)), y)) | |
| new_train = [new_rows[i] for i in sorted(train_idx)] | |
| new_holdout = [new_rows[i] for i in sorted(test_idx)] | |
| # The previous fresh 20-image holdout is explicitly promoted into training. | |
| development = prior_development + prior_holdout20 + new_train | |
| if tuple(map(len, (development, new_holdout, historical_holdout))) != (736, 32, 32): | |
| raise ValueError("unexpected development/new holdout/historical sizes") | |
| development_names = {row["file_name"] for row in development} | |
| holdout_names = {row["file_name"] for row in new_holdout} | |
| historical_names = {row["file_name"] for row in historical_holdout} | |
| if development_names & (holdout_names | historical_names) or holdout_names & historical_names: | |
| raise ValueError("split leakage detected") | |
| if development_names | holdout_names | historical_names != {row["file_name"] for row in all_rows}: | |
| raise ValueError("split union does not cover the 800-image dataset") | |
| args.combined.mkdir(parents=True, exist_ok=True) | |
| for row in all_rows: | |
| copy_verified(Path(row["source_path"]), args.combined / row["file_name"]) | |
| with (args.combined / "human-tagged.txt.txt").open("w", newline="", encoding="utf-8") as handle: | |
| writer = csv.DictWriter(handle, fieldnames=["file_name", "characters", "index"]) | |
| writer.writeheader() | |
| for index, row in enumerate(all_rows): | |
| writer.writerow({"file_name": row["file_name"], "characters": row["characters"], "index": index}) | |
| write_manifest(args.splits / "all800.csv", all_rows) | |
| write_manifest(args.splits / "development736.csv", development) | |
| write_manifest(args.splits / "historical_holdout32.csv", historical_holdout) | |
| write_manifest(args.splits / "new_holdout32.csv", new_holdout) | |
| write_manifest(args.splits / "new_holdout32_public.csv", new_holdout, labels=False) | |
| paths = {row["file_name"]: Path(row["source_path"]) for row in all_rows} | |
| sha_groups: dict[str, list[str]] = {} | |
| file_hashes: dict[str, str] = {} | |
| hashes: dict[str, int] = {} | |
| for name, path in paths.items(): | |
| digest = hashlib.sha256(path.read_bytes()).hexdigest() | |
| file_hashes[name] = digest | |
| sha_groups.setdefault(digest, []).append(name) | |
| hashes[name] = phash(path) | |
| exact_duplicates = [group for group in sha_groups.values() if len(group) > 1] | |
| (args.splits / "image_sha256.json").write_text( | |
| json.dumps(file_hashes, sort_keys=True, indent=2) + "\n", encoding="utf-8" | |
| ) | |
| nearest = [] | |
| for row in new_holdout: | |
| name = row["file_name"] | |
| neighbor, distance = min( | |
| ((candidate, (hashes[name] ^ hashes[candidate]).bit_count()) for candidate in development_names), | |
| key=lambda item: item[1], | |
| ) | |
| nearest.append({"holdout": name, "nearest_development": neighbor, "phash_hamming": distance}) | |
| report = { | |
| "seed": args.seed, | |
| "classes": classes, | |
| "all": count_rows(all_rows, classes), | |
| "development": count_rows(development, classes), | |
| "promoted_prior_holdout": count_rows(prior_holdout20, classes), | |
| "incremental_train": count_rows(new_train, classes), | |
| "new_holdout": count_rows(new_holdout, classes), | |
| "historical_holdout": count_rows(historical_holdout, classes), | |
| "exact_duplicate_groups": exact_duplicates, | |
| "minimum_new_holdout_to_development_phash_distance": min(item["phash_hamming"] for item in nearest), | |
| "nearest_new_holdout_pairs": sorted(nearest, key=lambda item: item["phash_hamming"]), | |
| "policy": ( | |
| "Train/select only on development736 (prior development648 + promoted prior holdout20 + " | |
| "68 incremental images). Evaluate new_holdout32 once. Keep historical_holdout32 excluded " | |
| "as a non-tuning reference." | |
| ), | |
| } | |
| args.splits.mkdir(parents=True, exist_ok=True) | |
| (args.splits / "split_report.json").write_text( | |
| json.dumps(report, ensure_ascii=False, indent=2) + "\n", encoding="utf-8" | |
| ) | |
| print(json.dumps(report, ensure_ascii=False, indent=2)) | |
| if __name__ == "__main__": | |
| main() | |