File size: 8,952 Bytes
9fc9873
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
#!/usr/bin/env python3
"""Build a resumable, image-backed GQA expansion candidate set.

This is deliberately independent from the old 12-row pilot and from the
formal HFD source plan.  It scans the complete balanced instruction splits,
keeps one deterministic conservative visible-state question per image, then
streams only the selected image records into an external output directory.
The resulting manifest is *pending* visual/audio/frontend admission; it is
not a training source plan.
"""

from __future__ import annotations

import argparse
from collections import Counter
import hashlib
import json
from pathlib import Path
from typing import Any

from PIL import Image

from build_hfd_vqa_candidate_pilot import conservative_gqa_row


def atomic_json(path: Path, value: Any) -> None:
    path.parent.mkdir(parents=True, exist_ok=True)
    partial = path.with_name(path.name + ".partial")
    partial.write_text(json.dumps(value, ensure_ascii=False, indent=2) + "\n", encoding="utf-8")
    partial.replace(path)


def score(row: dict[str, Any]) -> str:
    # Stable, source-order-independent tie break.  Include the question id so
    # repeated questions for one image do not silently depend on parquet order.
    return hashlib.sha256((str(row.get("imageId")) + "\0" + str(row.get("id"))).encode()).hexdigest()


def scan_instructions(split: str, limit: int, excluded_images: set[str]) -> tuple[list[dict[str, Any]], dict[str, Any]]:
    from datasets import load_dataset

    config = f"{split}_balanced_instructions"
    ds = load_dataset("lmms-lab-encoder/GQA", config, split=split, streaming=True)
    best: dict[str, dict[str, Any]] = {}
    rejected = Counter()
    scanned = 0
    for raw in ds:
        scanned += 1
        row = dict(raw)
        image_id = str(row.get("imageId", ""))
        if image_id in excluded_images:
            rejected["old_pilot_image"] += 1
            continue
        accepted, reason = conservative_gqa_row(row)
        if not accepted:
            rejected[reason] += 1
            continue
        item = {key: row.get(key) for key in ("id", "imageId", "question", "answer", "fullAnswer", "types", "semanticStr")}
        item.update({"split": split, "source_dataset": "GQA", "source_config": config,
                     "status": "pending_blind_image_review_audio_frontend", "text_gate": "conservative_gqa_row_v1",
                     "selection_score": score(row)})
        old = best.get(image_id)
        if old is None or item["selection_score"] < old["selection_score"]:
            best[image_id] = item
    ordered = sorted(best.values(), key=lambda r: (str(r["selection_score"]), str(r["id"])))
    selected = ordered[:limit]
    audit = {"split": split, "scanned": scanned, "accepted_unique_images": len(best),
             "selected": len(selected), "requested": limit, "rejected": dict(rejected),
             "question_ids_unique": len({str(r["id"]) for r in selected}),
             "image_ids_unique": len({str(r["imageId"]) for r in selected})}
    return selected, audit


def fetch_images(root: Path, rows: list[dict[str, Any]]) -> tuple[list[dict[str, Any]], dict[str, Any]]:
    """Stream balanced image shards, materializing only wanted image ids.

    The routine is resumable: existing files are verified and reused.  We do
    not hold parquet/image payloads in memory after each row.
    """
    from datasets import load_dataset

    wanted = {(str(row["split"]), str(row["imageId"])): row for row in rows}
    found: dict[tuple[str, str], dict[str, Any]] = {}
    counters = Counter()
    by_split = {split: [key for key in wanted if key[0] == split] for split in ("train", "val")}
    for split in ("train", "val"):
        if not by_split[split]:
            continue
        ds = load_dataset("lmms-lab-encoder/GQA", f"{split}_balanced_images", split=split, streaming=True)
        for number, raw in enumerate(ds, 1):
            image_id = str(raw.get("id", ""))
            key = (split, image_id)
            if key not in wanted:
                continue
            row = wanted[key]
            path = root / "images" / split / f"{image_id}.jpg"
            path.parent.mkdir(parents=True, exist_ok=True)
            try:
                if path.is_file():
                    with Image.open(path) as image:
                        image.load()
                        width, height = image.size
                else:
                    image = raw.get("image")
                    if image is None:
                        counters["missing_image"] += 1
                        continue
                    image = image.convert("RGB")
                    width, height = image.size
                    if min(width, height) < 224:
                        counters["image_too_small"] += 1
                        continue
                    temporary = path.with_name(path.name + ".partial")
                    image.save(temporary, format="JPEG", quality=95)
                    temporary.replace(path)
                if min(width, height) < 224:
                    counters["image_too_small"] += 1
                    continue
                digest = hashlib.sha256(path.read_bytes()).hexdigest()
                found[key] = {**row, "image_path": str(path.resolve()), "image_sha256": digest,
                              "width": int(width), "height": int(height),
                              "status": "pending_blind_image_review_audio_frontend"}
                counters["saved_or_reused"] += 1
            except Exception as exc:  # leave row pending for a later resume
                counters[f"image_error:{type(exc).__name__}"] += 1
            if len(found) == len(wanted):
                break
            if number % 5000 == 0:
                print(f"images_scan split={split} rows={number} found={len(found)}/{len(wanted)}", flush=True)
        counters[f"{split}_image_rows_scanned"] = number
    admitted = [found[key] for key in wanted if key in found]
    # A hash collision here would indicate source corruption or a split leak.
    train_hashes = {r["image_sha256"] for r in admitted if r["split"] == "train"}
    val_hashes = {r["image_sha256"] for r in admitted if r["split"] == "val"}
    if train_hashes & val_hashes:
        raise RuntimeError("GQA candidate image content overlaps train and val")
    audit = {"requested": len(rows), "selected_with_images": len(admitted),
             "pending_image_rows": len(rows) - len(admitted), "image_gate": dict(counters),
             "cross_split_image_sha256_overlap": 0,
             "training_ready": False,
             "reason_not_training_ready": "blind_visual_review_and_question_audio_and_causal_frontend_pending"}
    return admitted, audit


def main() -> None:
    parser = argparse.ArgumentParser(description=__doc__)
    parser.add_argument("--output-root", type=Path, required=True)
    parser.add_argument("--train", type=int, default=10000)
    parser.add_argument("--val", type=int, default=1000)
    parser.add_argument("--old-pilot", type=Path, default=None)
    args = parser.parse_args()
    root = args.output_root.expanduser().resolve()
    old_images: set[str] = set()
    if args.old_pilot and args.old_pilot.is_file():
        old_images = {str(r.get("imageId")) for r in json.loads(args.old_pilot.read_text())}
    rows_train, audit_train = scan_instructions("train", args.train, old_images)
    rows_val, audit_val = scan_instructions("val", args.val, old_images)
    train_images = {str(r["imageId"]) for r in rows_train}
    rows_val = [r for r in rows_val if str(r["imageId"]) not in train_images]
    if len(rows_val) < args.val:
        raise RuntimeError(f"cross-split image disjointness reduced val below request: {len(rows_val)}/{args.val}")
    rows = rows_train + rows_val[:args.val]
    atomic_json(root / "gqa_text_candidates.json", rows)
    atomic_json(root / "gqa_text_selection_audit.json", {"dataset": "lmms-lab-encoder/GQA",
        "train": audit_train, "val": {**audit_val, "selected_after_cross_split": len(rows_val[:args.val])},
        "old_pilot_images_excluded": len(old_images), "new_candidate_rows": len(rows),
        "new_image_ids": len({str(r["imageId"]) for r in rows}), "training_ready": False})
    admitted, image_audit = fetch_images(root, rows)
    atomic_json(root / "gqa_candidate_manifest.json", admitted)
    atomic_json(root / "gqa_image_selection_audit.json", {"text": {"train": audit_train, "val": audit_val}, **image_audit,
        "status": "pending_blind_image_review_audio_frontend", "source_plan_modified": False})
    print(json.dumps({"text_candidates": len(rows), "with_images": len(admitted),
                      "train_with_images": sum(r["split"] == "train" for r in admitted),
                      "val_with_images": sum(r["split"] == "val" for r in admitted),
                      "audit": image_audit}, ensure_ascii=False, indent=2), flush=True)


if __name__ == "__main__":
    main()