Download src/prepare_data.py from sneo/v802-classifier: direct link, hf CLI and curl.
- Browser
- Download file 9.09 kB
-
https://huggingface.co/sneo/v802-classifier/resolve/main/src/prepare_data.py
- Command line
-
hf download hf://sneo/v802-classifier/src/prepare_data.py
-
curl -L -o prepare_data.py https://huggingface.co/sneo/v802-classifier/resolve/main/src/prepare_data.py
9.09 kB
| #!/usr/bin/env python3 | |
| """Build an audited 700-image dataset and freeze a fresh 20-image holdout.""" | |
| from __future__ import annotations | |
| import argparse | |
| import csv | |
| import hashlib | |
| import json | |
| import os | |
| import shutil | |
| from collections import Counter | |
| from pathlib import Path | |
| a | |
| import numpy as np | |
| from PIL import Image | |
| from iterstrat.ml_stratifiers import MultilabelStratifiedShuffleSplit | |
| from dataset_utils import read_rows | |
| def phash(path: Path, size: int = 32, low: int = 8) -> int: | |
| with Image.open(path) as image: | |
| array = np.asarray( | |
| image.convert("L").resize((size, size), Image.Resampling.LANCZOS), dtype=np.float64 | |
| ) | |
| n = np.arange(size) | |
| k = n[:, None] | |
| basis = np.cos(np.pi * (n + 0.5) * k / size) | |
| basis[0] /= np.sqrt(2) | |
| dct = (2 / size) * basis @ array @ basis.T | |
| block = dct[:low, :low] | |
| median = np.median(block.ravel()[1:]) | |
| value = 0 | |
| for bit in block.ravel() >= median: | |
| value = (value << 1) | int(bit) | |
| return value | |
| def add_origin(rows: list[dict[str, str]], root: Path, origin: str) -> list[dict[str, str]]: | |
| result = [] | |
| for row in rows: | |
| path = root / row["file_name"] | |
| if not path.is_file(): | |
| raise FileNotFoundError(path) | |
| result.append({ | |
| "file_name": row["file_name"], | |
| "characters": row["characters"], | |
| "origin": origin, | |
| "source_path": str(path.resolve()), | |
| }) | |
| return result | |
| def write_manifest(path: Path, rows: list[dict[str, str]], labels: bool = True) -> None: | |
| fields = ["file_name", "origin"] + (["characters"] if labels else []) | |
| path.parent.mkdir(parents=True, exist_ok=True) | |
| with path.open("w", newline="", encoding="utf-8") as handle: | |
| writer = csv.DictWriter(handle, fieldnames=fields) | |
| writer.writeheader() | |
| for row in rows: | |
| writer.writerow({field: row[field] for field in fields}) | |
| def count_rows(rows: list[dict[str, str]], classes: list[str]) -> dict[str, object]: | |
| cardinality = Counter(len(row["characters"].split("|")) for row in rows) | |
| return { | |
| "samples": len(rows), | |
| "classes": { | |
| name: sum(name in row["characters"].split("|") for row in rows) for name in classes | |
| }, | |
| "cardinality": {str(k): cardinality[k] for k in (1, 2, 3)}, | |
| } | |
| def main() -> None: | |
| parser = argparse.ArgumentParser() | |
| parser.add_argument("--old", type=Path, required=True) | |
| parser.add_argument("--new", type=Path, required=True) | |
| parser.add_argument("--extra", type=Path, required=True) | |
| parser.add_argument("--v2-splits", type=Path, required=True) | |
| parser.add_argument("--combined", type=Path, required=True) | |
| parser.add_argument("--splits", type=Path, required=True) | |
| parser.add_argument("--seed", type=int, default=700288) | |
| args = parser.parse_args() | |
| old_rows = add_origin(read_rows(args.old / "human-tagged.txt.txt"), args.old, "old440") | |
| new_rows = add_origin(read_rows(args.new / "human-tagged.txt.txt"), args.new, "new160") | |
| extra_rows = add_origin(read_rows(args.extra / "human-tagged.txt.txt"), args.extra, "extra100") | |
| all_rows = old_rows + new_rows + extra_rows | |
| if (len(old_rows), len(new_rows), len(extra_rows)) != (440, 160, 100): | |
| raise ValueError("expected 440 + 160 + 100 images") | |
| if len({row["file_name"] for row in all_rows}) != 700: | |
| raise ValueError("duplicate filenames") | |
| labels_by_name = {row["file_name"]: row["characters"] for row in all_rows} | |
| origin_by_name = {row["file_name"]: row["origin"] for row in all_rows} | |
| prior_dev = read_rows(args.v2_splits / "development.csv") | |
| prior_holdout = read_rows(args.v2_splits / "holdout_locked_labels.csv") | |
| for row in prior_dev + prior_holdout: | |
| if labels_by_name.get(row["file_name"]) != row["characters"]: | |
| raise ValueError(f"v2 label mismatch: {row['file_name']}") | |
| classes = sorted({label for row in all_rows for label in row["characters"].split("|")}) | |
| class_index = {name: index for index, name in enumerate(classes)} | |
| y = np.zeros((len(extra_rows), len(classes) + 3), dtype=np.uint8) | |
| for i, row in enumerate(extra_rows): | |
| labels = row["characters"].split("|") | |
| for label in labels: | |
| y[i, class_index[label]] = 1 | |
| y[i, len(classes) + len(labels) - 1] = 1 | |
| splitter = MultilabelStratifiedShuffleSplit( | |
| n_splits=1, test_size=20, random_state=args.seed | |
| ) | |
| extra_dev_idx, extra_test_idx = next(splitter.split(np.zeros(len(extra_rows)), y)) | |
| if len(extra_test_idx) != 20: | |
| raise RuntimeError(f"expected 20 holdout rows, got {len(extra_test_idx)}") | |
| extra_dev = [extra_rows[i] for i in sorted(extra_dev_idx)] | |
| extra_holdout = [extra_rows[i] for i in sorted(extra_test_idx)] | |
| development = [ | |
| { | |
| "file_name": row["file_name"], | |
| "characters": row["characters"], | |
| "origin": origin_by_name[row["file_name"]], | |
| } | |
| for row in prior_dev | |
| ] + extra_dev | |
| historical_holdout = [ | |
| { | |
| "file_name": row["file_name"], | |
| "characters": row["characters"], | |
| "origin": origin_by_name[row["file_name"]], | |
| } | |
| for row in prior_holdout | |
| ] | |
| if len(development) != 648 or len(historical_holdout) != 32: | |
| raise ValueError("unexpected development or historical holdout size") | |
| args.combined.mkdir(parents=True, exist_ok=True) | |
| for row in all_rows: | |
| source = Path(row["source_path"]) | |
| destination = args.combined / row["file_name"] | |
| if destination.exists(): | |
| if hashlib.sha256(destination.read_bytes()).digest() != hashlib.sha256(source.read_bytes()).digest(): | |
| raise ValueError(f"stale combined image: {destination}") | |
| if os.path.samefile(source, destination): | |
| temporary = destination.with_suffix(".copying") | |
| shutil.copy2(source, temporary) | |
| os.replace(temporary, destination) | |
| else: | |
| shutil.copy2(source, destination) | |
| with (args.combined / "human-tagged.txt.txt").open("w", newline="", encoding="utf-8") as handle: | |
| writer = csv.DictWriter(handle, fieldnames=["file_name", "characters", "index"]) | |
| writer.writeheader() | |
| for index, row in enumerate(all_rows): | |
| writer.writerow({"file_name": row["file_name"], "characters": row["characters"], "index": index}) | |
| public_rows = [{"file_name": row["file_name"], "origin": row["origin"]} for row in extra_holdout] | |
| write_manifest(args.splits / "all700.csv", all_rows) | |
| write_manifest(args.splits / "development648.csv", development) | |
| write_manifest(args.splits / "historical_holdout32.csv", historical_holdout) | |
| write_manifest(args.splits / "new_holdout20.csv", extra_holdout) | |
| write_manifest(args.splits / "new_holdout20_public.csv", public_rows, labels=False) | |
| sha_groups: dict[str, list[str]] = {} | |
| hashes: dict[str, int] = {} | |
| path_by_name = {row["file_name"]: Path(row["source_path"]) for row in all_rows} | |
| for name, path in path_by_name.items(): | |
| sha_groups.setdefault(hashlib.sha256(path.read_bytes()).hexdigest(), []).append(name) | |
| hashes[name] = phash(path) | |
| exact_duplicates = [group for group in sha_groups.values() if len(group) > 1] | |
| file_hashes = { | |
| name: hashlib.sha256(path.read_bytes()).hexdigest() for name, path in path_by_name.items() | |
| } | |
| (args.splits / "image_sha256.json").write_text( | |
| json.dumps(file_hashes, sort_keys=True, indent=2) + "\n", encoding="utf-8" | |
| ) | |
| development_names = [row["file_name"] for row in development] | |
| nearest = [] | |
| for row in extra_holdout: | |
| name = row["file_name"] | |
| neighbor, distance = min( | |
| ((candidate, (hashes[name] ^ hashes[candidate]).bit_count()) for candidate in development_names), | |
| key=lambda item: item[1], | |
| ) | |
| nearest.append({"holdout": name, "nearest_development": neighbor, "phash_hamming": distance}) | |
| report = { | |
| "seed": args.seed, | |
| "classes": classes, | |
| "all": count_rows(all_rows, classes), | |
| "development": count_rows(development, classes), | |
| "new_holdout": count_rows(extra_holdout, classes), | |
| "historical_holdout": count_rows(historical_holdout, classes), | |
| "exact_duplicate_groups": exact_duplicates, | |
| "minimum_new_holdout_to_development_phash_distance": min( | |
| item["phash_hamming"] for item in nearest | |
| ), | |
| "nearest_new_holdout_pairs": sorted(nearest, key=lambda item: item["phash_hamming"]), | |
| "policy": ( | |
| "Tune only on development648. Evaluate new_holdout20 once after model selection. " | |
| "Use historical_holdout32 only as a non-tuning reference to the published v2 result." | |
| ), | |
| } | |
| (args.splits / "split_report.json").write_text( | |
| json.dumps(report, ensure_ascii=False, indent=2) + "\n", encoding="utf-8" | |
| ) | |
| print(json.dumps(report, ensure_ascii=False, indent=2)) | |
| if __name__ == "__main__": | |
| main() | |