File size: 14,403 Bytes
9fc9873
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
#!/usr/bin/env python3
"""Build a small, auditable VQA admission pilot without touching formal data.

Two outputs are produced under an external directory:

* a training-ready *subset* of the already encoded HFD source plan.  It uses
  only official-human temporal Charades-Ego rows and high-agreement VQAv2
  rows, keeps the original body/frontend references, and is suitable for a
  tiny opt-in HFD run;
* an optional GQA screening manifest.  GQA rows are deliberately marked
  ``pending_audio_frontend``: downloading an image and accepting its text is
  not enough to pretend that it is a causal HFD source.

The formal ``source_plan_5x.json`` is never rewritten.  This is an admission
pilot and a reproducible quality gate, not a silent dataset replacement.
"""

from __future__ import annotations

import argparse
import hashlib
import json
import re
from collections import Counter
from pathlib import Path
from typing import Any, Iterable, Mapping


HFD_COHORT_QUALITY = {
    "charades": "official_human_temporal_action_annotation_derived_binary_QA",
    "vqav2_static": "human_agreement_>=9_and_independent_image_review",
}


def _stable_key(row: Mapping[str, Any]) -> str:
    value = "|".join(
        str(row.get(key, ""))
        for key in ("id", "gt_question_id", "path", "motion_name", "split")
    )
    return hashlib.sha256(value.encode("utf-8")).hexdigest()


def _nested_question(row: Mapping[str, Any]) -> Mapping[str, Any]:
    value = row.get("question")
    return value if isinstance(value, Mapping) else {}


def _source_cohort(row: Mapping[str, Any]) -> str:
    return str(row.get("cohort") or row.get("data_domain") or "")


def _quality_gate(row: Mapping[str, Any]) -> tuple[bool, str]:
    """Reject rows whose provenance or causal frontend is not auditable."""

    cohort = _source_cohort(row)
    if cohort not in HFD_COHORT_QUALITY:
        return False, "cohort_not_in_conservative_pilot"
    question = _nested_question(row)
    if question.get("swa_has_question") is False:
        return False, "explicit_non_question_row"
    if not str(question.get("question", "")).strip() or not str(question.get("answer", "")).strip():
        return False, "empty_question_or_answer"
    path = Path(str(row.get("path", "")))
    if not path.is_file():
        return False, "body_path_missing"
    frontend_root = Path(str(row.get("frontend_root", "")))
    frontend = frontend_root / "data" / "frontend" / f"{row.get('id')}.pt"
    # The static VQAv2 frontend can be factorized/inline; the row's timeline
    # remains the authoritative existence check in that case.
    timeline = row.get("timeline")
    question_voice = timeline.get("question_voice") if isinstance(timeline, Mapping) else None
    if question_voice and not Path(str(question_voice)).is_file():
        return False, "question_audio_missing"
    if not question_voice and not frontend.is_file():
        return False, "frontend_missing"
    if cohort == "charades":
        provenance = str(question.get("gt_provenance", ""))
        if provenance != HFD_COHORT_QUALITY[cohort]:
            return False, "charades_provenance_not_human_temporal"
    if cohort == "vqav2_static":
        votes = int(question.get("human_votes", 0) or 0)
        # The recovered source-plan schema stores the independent blind image
        # review at row level.  Older pilot rows put it inside ``question``;
        # accept both locations, but never weaken the gate when neither exists.
        review = row.get("visual_review") or question.get("visual_review")
        if votes < 9 or not isinstance(review, Mapping) or review.get("accepted") is not True:
            return False, "vqav2_agreement_or_visual_review_failed"
    return True, "accepted"


def select_hfd_rows(
    rows: Iterable[Mapping[str, Any]], *, train_limit: int, val_limit: int, seed: int = 260922
) -> tuple[list[dict[str, Any]], dict[str, Any]]:
    """Select disjoint, deterministic, source-plan-backed pilot rows."""

    requested = {"train": int(train_limit), "val": int(val_limit)}
    candidates: dict[str, list[dict[str, Any]]] = {"train": [], "val": []}
    rejected = Counter()
    for raw in rows:
        row = dict(raw)
        split = str(row.get("split", ""))
        if split not in candidates:
            rejected["split_not_train_or_val"] += 1
            continue
        accepted, reason = _quality_gate(row)
        if not accepted:
            rejected[reason] += 1
            continue
        row["pilot_quality"] = HFD_COHORT_QUALITY[_source_cohort(row)]
        row["pilot_seed"] = int(seed)
        candidates[split].append(row)

    selected: list[dict[str, Any]] = []
    counts = Counter()
    # Hash ordering avoids source-plan ordering or motion-family clustering.
    for split in ("train", "val"):
        # Round-robin the accepted cohorts.  A plain global hash sort can
        # accidentally make a tiny pilot all VQAv2 (or all Charades), which
        # tests only one upstream source while appearing to test two.
        pools = {
            cohort: sorted(
                (row for row in candidates[split] if _source_cohort(row) == cohort),
                key=_stable_key,
            )
            for cohort in sorted({_source_cohort(row) for row in candidates[split]})
        }
        used_body: set[str] = set()
        cursors = {cohort: 0 for cohort in pools}
        while sum(1 for item in selected if item.get("split") == split) < requested[split]:
            made_progress = False
            for cohort, pool in pools.items():
                while cursors[cohort] < len(pool):
                    row = pool[cursors[cohort]]
                    cursors[cohort] += 1
                    body = str(row.get("path", ""))
                    if body in used_body:
                        continue
                    selected.append(row)
                    used_body.add(body)
                    counts[f"{split}:{cohort}"] += 1
                    made_progress = True
                    break
                if sum(1 for item in selected if item.get("split") == split) >= requested[split]:
                    break
            if not made_progress:
                break
        if sum(1 for item in selected if item.get("split") == split) < requested[split]:
            raise ValueError(
                f"not enough disjoint accepted rows for {split}: "
                f"{sum(1 for item in selected if item.get('split') == split)}/{requested[split]}"
            )
    selected.sort(key=lambda row: (str(row.get("split")), _stable_key(row)))
    return selected, {
        "requested": requested,
        "accepted_pool": {split: len(value) for split, value in candidates.items()},
        "selected": len(selected),
        "selected_by_split": dict(Counter(str(row["split"]) for row in selected)),
        "selected_by_cohort": dict(counts),
        "rejected": dict(rejected),
        "formal_source_plan_changed": False,
    }


_GQA_BAD = re.compile(
    r"\b(why|might|could|would|should|probably|before|after|future|written|text|"
    r"says|read|brand|license|country|city|name|feel|taste|famous|occupation|"
    r"dangerous|safe|injured|how old|time of day)\b",
    re.IGNORECASE,
)


def conservative_gqa_row(row: Mapping[str, Any]) -> tuple[bool, str]:
    """Conservative text-side gate for streamed GQA, before image review."""

    question = " ".join(str(row.get("question", "")).split())
    answer = " ".join(str(row.get("answer", "")).split())
    words = re.findall(r"[A-Za-z0-9']+", question)
    answer_words = re.findall(r"[A-Za-z0-9']+", answer)
    if row.get("isBalanced") is not True:
        return False, "not_balanced"
    if not (3 <= len(words) <= 18):
        return False, "question_length"
    if not (1 <= len(answer_words) <= 6):
        return False, "answer_length"
    if _GQA_BAD.search(question):
        return False, "speculative_or_text_question"
    if answer.lower() in {"unknown", "unanswerable", "not sure", "can't tell", "cant tell"}:
        return False, "unknown_answer"
    types = row.get("types")
    if isinstance(types, Mapping) and str(types.get("structural", "")) not in {"verify", "query"}:
        return False, "unsupported_gqa_type"
    return True, "accepted_text_gate_pending_image_review"


def _write_json(path: Path, value: Any) -> None:
    path.parent.mkdir(parents=True, exist_ok=True)
    temporary = path.with_name(path.name + ".partial")
    temporary.write_text(json.dumps(value, ensure_ascii=False, indent=2) + "\n", encoding="utf-8")
    temporary.replace(path)


def _stream_gqa(output: Path, *, train_limit: int, val_limit: int, scan_limit: int) -> dict[str, Any]:
    try:
        from datasets import load_dataset
    except ImportError as exc:  # pragma: no cover - optional data-side dependency
        raise RuntimeError("GQA pilot requires the optional `datasets` package") from exc

    selected: list[dict[str, Any]] = []
    audit: dict[str, Any] = {"dataset": "lmms-lab-encoder/GQA", "splits": {}, "status": "text_screened"}
    for split, limit in (("train", train_limit), ("val", val_limit)):
        config = f"{split}_balanced_instructions"
        ds = load_dataset("lmms-lab-encoder/GQA", config, split=split, streaming=True)
        pool: list[dict[str, Any]] = []
        rejected = Counter()
        for index, row in enumerate(ds):
            if index >= scan_limit:
                break
            ok, reason = conservative_gqa_row(row)
            if not ok:
                rejected[reason] += 1
                continue
            item = {key: row.get(key) for key in ("id", "imageId", "question", "answer", "fullAnswer", "types", "semanticStr")}
            item.update({"split": split, "source_dataset": "GQA", "status": "pending_audio_frontend_and_blind_image_review"})
            pool.append(item)
        # The image stream is ordered by image id.  Selecting the earliest
        # accepted unique ids makes this small pilot finish after reading only
        # a tiny prefix; the full admission job can use a hash-ordered sample
        # once an offline image index exists.
        pool.sort(key=lambda item: (int(item["imageId"]), str(item["id"])))
        unique_images: set[str] = set()
        for item in pool:
            image_id = str(item["imageId"])
            if image_id in unique_images:
                continue
            selected.append(item)
            unique_images.add(image_id)
            if sum(1 for x in selected if x["split"] == split) >= limit:
                break
        audit["splits"][split] = {"scanned": min(index + 1, scan_limit), "accepted_text_pool": len(pool), "selected": sum(1 for x in selected if x["split"] == split), "rejected": dict(rejected)}
    if {str(x["imageId"]) for x in selected if x["split"] == "train"} & {str(x["imageId"]) for x in selected if x["split"] == "val"}:
        raise RuntimeError("GQA pilot has a cross-split image collision")

    image_root = output / "gqa_images"
    image_root.mkdir(parents=True, exist_ok=True)
    wanted = {str(x["imageId"]): x for x in selected}
    image_audit = Counter()
    for split in ("train", "val"):
        split_wanted = {key: value for key, value in wanted.items() if value["split"] == split}
        if not split_wanted:
            continue
        ds = load_dataset("lmms-lab-encoder/GQA", f"{split}_balanced_images", split=split, streaming=True)
        for row in ds:
            image_id = str(row.get("id"))
            if image_id not in split_wanted:
                continue
            image = row.get("image")
            if image is None:
                image_audit["missing_image"] += 1
                continue
            image = image.convert("RGB")
            if min(image.size) < 224:
                image_audit["image_too_small"] += 1
                continue
            path = image_root / split / f"{image_id}.jpg"
            path.parent.mkdir(parents=True, exist_ok=True)
            image.save(path, format="JPEG", quality=95)
            digest = hashlib.sha256(path.read_bytes()).hexdigest()
            split_wanted[image_id].update({"image_path": str(path.resolve()), "image_sha256": digest, "width": image.width, "height": image.height, "status": "screened_text_and_image_geometry_pending_semantic_review"})
            image_audit["saved"] += 1
            if all("image_path" in value for value in split_wanted.values()):
                break
    admitted = [item for item in selected if "image_path" in item]
    audit["image_gate"] = dict(image_audit)
    audit["selected_with_images"] = len(admitted)
    audit["training_ready"] = False
    audit["reason_not_training_ready"] = "GQA has no HFD question WAV, causal packet frontend, or motion/body binding in this pilot"
    _write_json(output / "gqa_candidate_manifest.json", admitted)
    _write_json(output / "gqa_selection_audit.json", audit)
    return audit


def main() -> None:
    parser = argparse.ArgumentParser(description=__doc__)
    parser.add_argument("--source-plan", type=Path, required=True)
    parser.add_argument("--output-root", type=Path, required=True)
    parser.add_argument("--train", type=int, default=64, help="accepted HFD pilot rows")
    parser.add_argument("--val", type=int, default=16, help="accepted HFD pilot rows")
    parser.add_argument("--include-gqa", action="store_true")
    parser.add_argument("--gqa-train", type=int, default=32)
    parser.add_argument("--gqa-val", type=int, default=8)
    parser.add_argument("--gqa-scan-limit", type=int, default=50000)
    args = parser.parse_args()
    source_plan = args.source_plan.expanduser().resolve()
    output = args.output_root.expanduser().resolve()
    rows = json.loads(source_plan.read_text(encoding="utf-8"))
    selected, audit = select_hfd_rows(rows, train_limit=args.train, val_limit=args.val)
    _write_json(output / "hfd_candidate_pilot_source_plan.json", selected)
    _write_json(output / "hfd_candidate_pilot_audit.json", {**audit, "source_plan": str(source_plan), "schema": "hfd_vqa_candidate_pilot_v1"})
    print(json.dumps(audit, ensure_ascii=False, indent=2), flush=True)
    if args.include_gqa:
        print(json.dumps(_stream_gqa(output, train_limit=args.gqa_train, val_limit=args.gqa_val, scan_limit=args.gqa_scan_limit), ensure_ascii=False, indent=2), flush=True)
    print(f"[ready] {output}", flush=True)


if __name__ == "__main__":
    main()