v802-classifier / src /prepare_data.py
sneo's picture
Add v802 classifier head and source code
59839e3 verified
Raw History Blame Contribute Delete
9.09 kB
#!/usr/bin/env python3
"""Build an audited 700-image dataset and freeze a fresh 20-image holdout."""
from __future__ import annotations
import argparse
import csv
import hashlib
import json
import os
import shutil
from collections import Counter
from pathlib import Path
a
import numpy as np
from PIL import Image
from iterstrat.ml_stratifiers import MultilabelStratifiedShuffleSplit
from dataset_utils import read_rows
def phash(path: Path, size: int = 32, low: int = 8) -> int:
with Image.open(path) as image:
array = np.asarray(
image.convert("L").resize((size, size), Image.Resampling.LANCZOS), dtype=np.float64
)
n = np.arange(size)
k = n[:, None]
basis = np.cos(np.pi * (n + 0.5) * k / size)
basis[0] /= np.sqrt(2)
dct = (2 / size) * basis @ array @ basis.T
block = dct[:low, :low]
median = np.median(block.ravel()[1:])
value = 0
for bit in block.ravel() >= median:
value = (value << 1) | int(bit)
return value
def add_origin(rows: list[dict[str, str]], root: Path, origin: str) -> list[dict[str, str]]:
result = []
for row in rows:
path = root / row["file_name"]
if not path.is_file():
raise FileNotFoundError(path)
result.append({
"file_name": row["file_name"],
"characters": row["characters"],
"origin": origin,
"source_path": str(path.resolve()),
})
return result
def write_manifest(path: Path, rows: list[dict[str, str]], labels: bool = True) -> None:
fields = ["file_name", "origin"] + (["characters"] if labels else [])
path.parent.mkdir(parents=True, exist_ok=True)
with path.open("w", newline="", encoding="utf-8") as handle:
writer = csv.DictWriter(handle, fieldnames=fields)
writer.writeheader()
for row in rows:
writer.writerow({field: row[field] for field in fields})
def count_rows(rows: list[dict[str, str]], classes: list[str]) -> dict[str, object]:
cardinality = Counter(len(row["characters"].split("|")) for row in rows)
return {
"samples": len(rows),
"classes": {
name: sum(name in row["characters"].split("|") for row in rows) for name in classes
},
"cardinality": {str(k): cardinality[k] for k in (1, 2, 3)},
}
def main() -> None:
parser = argparse.ArgumentParser()
parser.add_argument("--old", type=Path, required=True)
parser.add_argument("--new", type=Path, required=True)
parser.add_argument("--extra", type=Path, required=True)
parser.add_argument("--v2-splits", type=Path, required=True)
parser.add_argument("--combined", type=Path, required=True)
parser.add_argument("--splits", type=Path, required=True)
parser.add_argument("--seed", type=int, default=700288)
args = parser.parse_args()
old_rows = add_origin(read_rows(args.old / "human-tagged.txt.txt"), args.old, "old440")
new_rows = add_origin(read_rows(args.new / "human-tagged.txt.txt"), args.new, "new160")
extra_rows = add_origin(read_rows(args.extra / "human-tagged.txt.txt"), args.extra, "extra100")
all_rows = old_rows + new_rows + extra_rows
if (len(old_rows), len(new_rows), len(extra_rows)) != (440, 160, 100):
raise ValueError("expected 440 + 160 + 100 images")
if len({row["file_name"] for row in all_rows}) != 700:
raise ValueError("duplicate filenames")
labels_by_name = {row["file_name"]: row["characters"] for row in all_rows}
origin_by_name = {row["file_name"]: row["origin"] for row in all_rows}
prior_dev = read_rows(args.v2_splits / "development.csv")
prior_holdout = read_rows(args.v2_splits / "holdout_locked_labels.csv")
for row in prior_dev + prior_holdout:
if labels_by_name.get(row["file_name"]) != row["characters"]:
raise ValueError(f"v2 label mismatch: {row['file_name']}")
classes = sorted({label for row in all_rows for label in row["characters"].split("|")})
class_index = {name: index for index, name in enumerate(classes)}
y = np.zeros((len(extra_rows), len(classes) + 3), dtype=np.uint8)
for i, row in enumerate(extra_rows):
labels = row["characters"].split("|")
for label in labels:
y[i, class_index[label]] = 1
y[i, len(classes) + len(labels) - 1] = 1
splitter = MultilabelStratifiedShuffleSplit(
n_splits=1, test_size=20, random_state=args.seed
)
extra_dev_idx, extra_test_idx = next(splitter.split(np.zeros(len(extra_rows)), y))
if len(extra_test_idx) != 20:
raise RuntimeError(f"expected 20 holdout rows, got {len(extra_test_idx)}")
extra_dev = [extra_rows[i] for i in sorted(extra_dev_idx)]
extra_holdout = [extra_rows[i] for i in sorted(extra_test_idx)]
development = [
{
"file_name": row["file_name"],
"characters": row["characters"],
"origin": origin_by_name[row["file_name"]],
}
for row in prior_dev
] + extra_dev
historical_holdout = [
{
"file_name": row["file_name"],
"characters": row["characters"],
"origin": origin_by_name[row["file_name"]],
}
for row in prior_holdout
]
if len(development) != 648 or len(historical_holdout) != 32:
raise ValueError("unexpected development or historical holdout size")
args.combined.mkdir(parents=True, exist_ok=True)
for row in all_rows:
source = Path(row["source_path"])
destination = args.combined / row["file_name"]
if destination.exists():
if hashlib.sha256(destination.read_bytes()).digest() != hashlib.sha256(source.read_bytes()).digest():
raise ValueError(f"stale combined image: {destination}")
if os.path.samefile(source, destination):
temporary = destination.with_suffix(".copying")
shutil.copy2(source, temporary)
os.replace(temporary, destination)
else:
shutil.copy2(source, destination)
with (args.combined / "human-tagged.txt.txt").open("w", newline="", encoding="utf-8") as handle:
writer = csv.DictWriter(handle, fieldnames=["file_name", "characters", "index"])
writer.writeheader()
for index, row in enumerate(all_rows):
writer.writerow({"file_name": row["file_name"], "characters": row["characters"], "index": index})
public_rows = [{"file_name": row["file_name"], "origin": row["origin"]} for row in extra_holdout]
write_manifest(args.splits / "all700.csv", all_rows)
write_manifest(args.splits / "development648.csv", development)
write_manifest(args.splits / "historical_holdout32.csv", historical_holdout)
write_manifest(args.splits / "new_holdout20.csv", extra_holdout)
write_manifest(args.splits / "new_holdout20_public.csv", public_rows, labels=False)
sha_groups: dict[str, list[str]] = {}
hashes: dict[str, int] = {}
path_by_name = {row["file_name"]: Path(row["source_path"]) for row in all_rows}
for name, path in path_by_name.items():
sha_groups.setdefault(hashlib.sha256(path.read_bytes()).hexdigest(), []).append(name)
hashes[name] = phash(path)
exact_duplicates = [group for group in sha_groups.values() if len(group) > 1]
file_hashes = {
name: hashlib.sha256(path.read_bytes()).hexdigest() for name, path in path_by_name.items()
}
(args.splits / "image_sha256.json").write_text(
json.dumps(file_hashes, sort_keys=True, indent=2) + "\n", encoding="utf-8"
)
development_names = [row["file_name"] for row in development]
nearest = []
for row in extra_holdout:
name = row["file_name"]
neighbor, distance = min(
((candidate, (hashes[name] ^ hashes[candidate]).bit_count()) for candidate in development_names),
key=lambda item: item[1],
)
nearest.append({"holdout": name, "nearest_development": neighbor, "phash_hamming": distance})
report = {
"seed": args.seed,
"classes": classes,
"all": count_rows(all_rows, classes),
"development": count_rows(development, classes),
"new_holdout": count_rows(extra_holdout, classes),
"historical_holdout": count_rows(historical_holdout, classes),
"exact_duplicate_groups": exact_duplicates,
"minimum_new_holdout_to_development_phash_distance": min(
item["phash_hamming"] for item in nearest
),
"nearest_new_holdout_pairs": sorted(nearest, key=lambda item: item["phash_hamming"]),
"policy": (
"Tune only on development648. Evaluate new_holdout20 once after model selection. "
"Use historical_holdout32 only as a non-tuning reference to the published v2 result."
),
}
(args.splits / "split_report.json").write_text(
json.dumps(report, ensure_ascii=False, indent=2) + "\n", encoding="utf-8"
)
print(json.dumps(report, ensure_ascii=False, indent=2))
if __name__ == "__main__":
main()