File size: 7,685 Bytes
59839e3
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
#!/usr/bin/env python3
"""Extend the audited 700-image dataset and lock 32 of 100 new images."""

from __future__ import annotations

import argparse
import csv
import hashlib
import json
import os
import shutil
from pathlib import Path

import numpy as np
from iterstrat.ml_stratifiers import MultilabelStratifiedShuffleSplit

from dataset_utils import read_rows
from prepare_data import count_rows, phash, write_manifest


def copy_verified(source: Path, destination: Path) -> None:
    if destination.exists():
        if hashlib.sha256(destination.read_bytes()).digest() != hashlib.sha256(source.read_bytes()).digest():
            raise ValueError(f"stale combined image: {destination}")
        return
    shutil.copy2(source, destination)


def main() -> None:
    parser = argparse.ArgumentParser()
    parser.add_argument("--prior-dataset", type=Path, required=True)
    parser.add_argument("--prior-splits", type=Path, required=True)
    parser.add_argument("--new", type=Path, required=True)
    parser.add_argument("--combined", type=Path, required=True)
    parser.add_argument("--splits", type=Path, required=True)
    # iterative-stratification may round the requested size for some seeds;
    # this audited seed yields exactly 68/32 while preserving both label and
    # label-cardinality balance. The explicit size assertion below is binding.
    parser.add_argument("--seed", type=int, default=800001)
    args = parser.parse_args()

    prior_labels = {
        row["file_name"]: row["characters"]
        for row in read_rows(args.prior_dataset / "human-tagged.txt.txt")
    }
    prior_rows = read_rows(args.prior_splits / "all700.csv")
    prior_development = read_rows(args.prior_splits / "development648.csv")
    prior_holdout20 = read_rows(args.prior_splits / "new_holdout20.csv")
    historical_holdout = read_rows(args.prior_splits / "historical_holdout32.csv")
    if tuple(map(len, (prior_rows, prior_development, prior_holdout20, historical_holdout))) != (700, 648, 20, 32):
        raise ValueError("expected prior all/development/new holdout/historical sizes 700/648/20/32")
    for row in prior_rows:
        if prior_labels.get(row["file_name"]) != row["characters"]:
            raise ValueError(f"prior label mismatch: {row['file_name']}")
        row["source_path"] = str((args.prior_dataset / row["file_name"]).resolve())

    new_rows = read_rows(args.new / "human-tagged.txt.txt")
    if len(new_rows) != 100:
        raise ValueError(f"expected 100 new images, got {len(new_rows)}")
    for row in new_rows:
        row["origin"] = "incremental100"
        row["source_path"] = str((args.new / row["file_name"]).resolve())
        if not Path(row["source_path"]).is_file():
            raise FileNotFoundError(row["source_path"])

    all_rows = prior_rows + new_rows
    if len({row["file_name"] for row in all_rows}) != 800:
        raise ValueError("duplicate filenames across the 800-image dataset")
    classes = sorted({label for row in all_rows for label in row["characters"].split("|")})
    class_index = {name: index for index, name in enumerate(classes)}
    y = np.zeros((len(new_rows), len(classes) + 3), dtype=np.uint8)
    for i, row in enumerate(new_rows):
        labels = row["characters"].split("|")
        for label in labels:
            y[i, class_index[label]] = 1
        y[i, len(classes) + len(labels) - 1] = 1
    splitter = MultilabelStratifiedShuffleSplit(n_splits=1, test_size=32, random_state=args.seed)
    train_idx, test_idx = next(splitter.split(np.zeros(len(new_rows)), y))
    new_train = [new_rows[i] for i in sorted(train_idx)]
    new_holdout = [new_rows[i] for i in sorted(test_idx)]

    # The previous fresh 20-image holdout is explicitly promoted into training.
    development = prior_development + prior_holdout20 + new_train
    if tuple(map(len, (development, new_holdout, historical_holdout))) != (736, 32, 32):
        raise ValueError("unexpected development/new holdout/historical sizes")
    development_names = {row["file_name"] for row in development}
    holdout_names = {row["file_name"] for row in new_holdout}
    historical_names = {row["file_name"] for row in historical_holdout}
    if development_names & (holdout_names | historical_names) or holdout_names & historical_names:
        raise ValueError("split leakage detected")
    if development_names | holdout_names | historical_names != {row["file_name"] for row in all_rows}:
        raise ValueError("split union does not cover the 800-image dataset")

    args.combined.mkdir(parents=True, exist_ok=True)
    for row in all_rows:
        copy_verified(Path(row["source_path"]), args.combined / row["file_name"])
    with (args.combined / "human-tagged.txt.txt").open("w", newline="", encoding="utf-8") as handle:
        writer = csv.DictWriter(handle, fieldnames=["file_name", "characters", "index"])
        writer.writeheader()
        for index, row in enumerate(all_rows):
            writer.writerow({"file_name": row["file_name"], "characters": row["characters"], "index": index})

    write_manifest(args.splits / "all800.csv", all_rows)
    write_manifest(args.splits / "development736.csv", development)
    write_manifest(args.splits / "historical_holdout32.csv", historical_holdout)
    write_manifest(args.splits / "new_holdout32.csv", new_holdout)
    write_manifest(args.splits / "new_holdout32_public.csv", new_holdout, labels=False)

    paths = {row["file_name"]: Path(row["source_path"]) for row in all_rows}
    sha_groups: dict[str, list[str]] = {}
    file_hashes: dict[str, str] = {}
    hashes: dict[str, int] = {}
    for name, path in paths.items():
        digest = hashlib.sha256(path.read_bytes()).hexdigest()
        file_hashes[name] = digest
        sha_groups.setdefault(digest, []).append(name)
        hashes[name] = phash(path)
    exact_duplicates = [group for group in sha_groups.values() if len(group) > 1]
    (args.splits / "image_sha256.json").write_text(
        json.dumps(file_hashes, sort_keys=True, indent=2) + "\n", encoding="utf-8"
    )
    nearest = []
    for row in new_holdout:
        name = row["file_name"]
        neighbor, distance = min(
            ((candidate, (hashes[name] ^ hashes[candidate]).bit_count()) for candidate in development_names),
            key=lambda item: item[1],
        )
        nearest.append({"holdout": name, "nearest_development": neighbor, "phash_hamming": distance})
    report = {
        "seed": args.seed,
        "classes": classes,
        "all": count_rows(all_rows, classes),
        "development": count_rows(development, classes),
        "promoted_prior_holdout": count_rows(prior_holdout20, classes),
        "incremental_train": count_rows(new_train, classes),
        "new_holdout": count_rows(new_holdout, classes),
        "historical_holdout": count_rows(historical_holdout, classes),
        "exact_duplicate_groups": exact_duplicates,
        "minimum_new_holdout_to_development_phash_distance": min(item["phash_hamming"] for item in nearest),
        "nearest_new_holdout_pairs": sorted(nearest, key=lambda item: item["phash_hamming"]),
        "policy": (
            "Train/select only on development736 (prior development648 + promoted prior holdout20 + "
            "68 incremental images). Evaluate new_holdout32 once. Keep historical_holdout32 excluded "
            "as a non-tuning reference."
        ),
    }
    args.splits.mkdir(parents=True, exist_ok=True)
    (args.splits / "split_report.json").write_text(
        json.dumps(report, ensure_ascii=False, indent=2) + "\n", encoding="utf-8"
    )
    print(json.dumps(report, ensure_ascii=False, indent=2))


if __name__ == "__main__":
    main()