#!/usr/bin/env python3 """Build a small, auditable VQA admission pilot without touching formal data. Two outputs are produced under an external directory: * a training-ready *subset* of the already encoded HFD source plan. It uses only official-human temporal Charades-Ego rows and high-agreement VQAv2 rows, keeps the original body/frontend references, and is suitable for a tiny opt-in HFD run; * an optional GQA screening manifest. GQA rows are deliberately marked ``pending_audio_frontend``: downloading an image and accepting its text is not enough to pretend that it is a causal HFD source. The formal ``source_plan_5x.json`` is never rewritten. This is an admission pilot and a reproducible quality gate, not a silent dataset replacement. """ from __future__ import annotations import argparse import hashlib import json import re from collections import Counter from pathlib import Path from typing import Any, Iterable, Mapping HFD_COHORT_QUALITY = { "charades": "official_human_temporal_action_annotation_derived_binary_QA", "vqav2_static": "human_agreement_>=9_and_independent_image_review", } def _stable_key(row: Mapping[str, Any]) -> str: value = "|".join( str(row.get(key, "")) for key in ("id", "gt_question_id", "path", "motion_name", "split") ) return hashlib.sha256(value.encode("utf-8")).hexdigest() def _nested_question(row: Mapping[str, Any]) -> Mapping[str, Any]: value = row.get("question") return value if isinstance(value, Mapping) else {} def _source_cohort(row: Mapping[str, Any]) -> str: return str(row.get("cohort") or row.get("data_domain") or "") def _quality_gate(row: Mapping[str, Any]) -> tuple[bool, str]: """Reject rows whose provenance or causal frontend is not auditable.""" cohort = _source_cohort(row) if cohort not in HFD_COHORT_QUALITY: return False, "cohort_not_in_conservative_pilot" question = _nested_question(row) if question.get("swa_has_question") is False: return False, "explicit_non_question_row" if not str(question.get("question", "")).strip() or not str(question.get("answer", "")).strip(): return False, "empty_question_or_answer" path = Path(str(row.get("path", ""))) if not path.is_file(): return False, "body_path_missing" frontend_root = Path(str(row.get("frontend_root", ""))) frontend = frontend_root / "data" / "frontend" / f"{row.get('id')}.pt" # The static VQAv2 frontend can be factorized/inline; the row's timeline # remains the authoritative existence check in that case. timeline = row.get("timeline") question_voice = timeline.get("question_voice") if isinstance(timeline, Mapping) else None if question_voice and not Path(str(question_voice)).is_file(): return False, "question_audio_missing" if not question_voice and not frontend.is_file(): return False, "frontend_missing" if cohort == "charades": provenance = str(question.get("gt_provenance", "")) if provenance != HFD_COHORT_QUALITY[cohort]: return False, "charades_provenance_not_human_temporal" if cohort == "vqav2_static": votes = int(question.get("human_votes", 0) or 0) # The recovered source-plan schema stores the independent blind image # review at row level. Older pilot rows put it inside ``question``; # accept both locations, but never weaken the gate when neither exists. review = row.get("visual_review") or question.get("visual_review") if votes < 9 or not isinstance(review, Mapping) or review.get("accepted") is not True: return False, "vqav2_agreement_or_visual_review_failed" return True, "accepted" def select_hfd_rows( rows: Iterable[Mapping[str, Any]], *, train_limit: int, val_limit: int, seed: int = 260922 ) -> tuple[list[dict[str, Any]], dict[str, Any]]: """Select disjoint, deterministic, source-plan-backed pilot rows.""" requested = {"train": int(train_limit), "val": int(val_limit)} candidates: dict[str, list[dict[str, Any]]] = {"train": [], "val": []} rejected = Counter() for raw in rows: row = dict(raw) split = str(row.get("split", "")) if split not in candidates: rejected["split_not_train_or_val"] += 1 continue accepted, reason = _quality_gate(row) if not accepted: rejected[reason] += 1 continue row["pilot_quality"] = HFD_COHORT_QUALITY[_source_cohort(row)] row["pilot_seed"] = int(seed) candidates[split].append(row) selected: list[dict[str, Any]] = [] counts = Counter() # Hash ordering avoids source-plan ordering or motion-family clustering. for split in ("train", "val"): # Round-robin the accepted cohorts. A plain global hash sort can # accidentally make a tiny pilot all VQAv2 (or all Charades), which # tests only one upstream source while appearing to test two. pools = { cohort: sorted( (row for row in candidates[split] if _source_cohort(row) == cohort), key=_stable_key, ) for cohort in sorted({_source_cohort(row) for row in candidates[split]}) } used_body: set[str] = set() cursors = {cohort: 0 for cohort in pools} while sum(1 for item in selected if item.get("split") == split) < requested[split]: made_progress = False for cohort, pool in pools.items(): while cursors[cohort] < len(pool): row = pool[cursors[cohort]] cursors[cohort] += 1 body = str(row.get("path", "")) if body in used_body: continue selected.append(row) used_body.add(body) counts[f"{split}:{cohort}"] += 1 made_progress = True break if sum(1 for item in selected if item.get("split") == split) >= requested[split]: break if not made_progress: break if sum(1 for item in selected if item.get("split") == split) < requested[split]: raise ValueError( f"not enough disjoint accepted rows for {split}: " f"{sum(1 for item in selected if item.get('split') == split)}/{requested[split]}" ) selected.sort(key=lambda row: (str(row.get("split")), _stable_key(row))) return selected, { "requested": requested, "accepted_pool": {split: len(value) for split, value in candidates.items()}, "selected": len(selected), "selected_by_split": dict(Counter(str(row["split"]) for row in selected)), "selected_by_cohort": dict(counts), "rejected": dict(rejected), "formal_source_plan_changed": False, } _GQA_BAD = re.compile( r"\b(why|might|could|would|should|probably|before|after|future|written|text|" r"says|read|brand|license|country|city|name|feel|taste|famous|occupation|" r"dangerous|safe|injured|how old|time of day)\b", re.IGNORECASE, ) def conservative_gqa_row(row: Mapping[str, Any]) -> tuple[bool, str]: """Conservative text-side gate for streamed GQA, before image review.""" question = " ".join(str(row.get("question", "")).split()) answer = " ".join(str(row.get("answer", "")).split()) words = re.findall(r"[A-Za-z0-9']+", question) answer_words = re.findall(r"[A-Za-z0-9']+", answer) if row.get("isBalanced") is not True: return False, "not_balanced" if not (3 <= len(words) <= 18): return False, "question_length" if not (1 <= len(answer_words) <= 6): return False, "answer_length" if _GQA_BAD.search(question): return False, "speculative_or_text_question" if answer.lower() in {"unknown", "unanswerable", "not sure", "can't tell", "cant tell"}: return False, "unknown_answer" types = row.get("types") if isinstance(types, Mapping) and str(types.get("structural", "")) not in {"verify", "query"}: return False, "unsupported_gqa_type" return True, "accepted_text_gate_pending_image_review" def _write_json(path: Path, value: Any) -> None: path.parent.mkdir(parents=True, exist_ok=True) temporary = path.with_name(path.name + ".partial") temporary.write_text(json.dumps(value, ensure_ascii=False, indent=2) + "\n", encoding="utf-8") temporary.replace(path) def _stream_gqa(output: Path, *, train_limit: int, val_limit: int, scan_limit: int) -> dict[str, Any]: try: from datasets import load_dataset except ImportError as exc: # pragma: no cover - optional data-side dependency raise RuntimeError("GQA pilot requires the optional `datasets` package") from exc selected: list[dict[str, Any]] = [] audit: dict[str, Any] = {"dataset": "lmms-lab-encoder/GQA", "splits": {}, "status": "text_screened"} for split, limit in (("train", train_limit), ("val", val_limit)): config = f"{split}_balanced_instructions" ds = load_dataset("lmms-lab-encoder/GQA", config, split=split, streaming=True) pool: list[dict[str, Any]] = [] rejected = Counter() for index, row in enumerate(ds): if index >= scan_limit: break ok, reason = conservative_gqa_row(row) if not ok: rejected[reason] += 1 continue item = {key: row.get(key) for key in ("id", "imageId", "question", "answer", "fullAnswer", "types", "semanticStr")} item.update({"split": split, "source_dataset": "GQA", "status": "pending_audio_frontend_and_blind_image_review"}) pool.append(item) # The image stream is ordered by image id. Selecting the earliest # accepted unique ids makes this small pilot finish after reading only # a tiny prefix; the full admission job can use a hash-ordered sample # once an offline image index exists. pool.sort(key=lambda item: (int(item["imageId"]), str(item["id"]))) unique_images: set[str] = set() for item in pool: image_id = str(item["imageId"]) if image_id in unique_images: continue selected.append(item) unique_images.add(image_id) if sum(1 for x in selected if x["split"] == split) >= limit: break audit["splits"][split] = {"scanned": min(index + 1, scan_limit), "accepted_text_pool": len(pool), "selected": sum(1 for x in selected if x["split"] == split), "rejected": dict(rejected)} if {str(x["imageId"]) for x in selected if x["split"] == "train"} & {str(x["imageId"]) for x in selected if x["split"] == "val"}: raise RuntimeError("GQA pilot has a cross-split image collision") image_root = output / "gqa_images" image_root.mkdir(parents=True, exist_ok=True) wanted = {str(x["imageId"]): x for x in selected} image_audit = Counter() for split in ("train", "val"): split_wanted = {key: value for key, value in wanted.items() if value["split"] == split} if not split_wanted: continue ds = load_dataset("lmms-lab-encoder/GQA", f"{split}_balanced_images", split=split, streaming=True) for row in ds: image_id = str(row.get("id")) if image_id not in split_wanted: continue image = row.get("image") if image is None: image_audit["missing_image"] += 1 continue image = image.convert("RGB") if min(image.size) < 224: image_audit["image_too_small"] += 1 continue path = image_root / split / f"{image_id}.jpg" path.parent.mkdir(parents=True, exist_ok=True) image.save(path, format="JPEG", quality=95) digest = hashlib.sha256(path.read_bytes()).hexdigest() split_wanted[image_id].update({"image_path": str(path.resolve()), "image_sha256": digest, "width": image.width, "height": image.height, "status": "screened_text_and_image_geometry_pending_semantic_review"}) image_audit["saved"] += 1 if all("image_path" in value for value in split_wanted.values()): break admitted = [item for item in selected if "image_path" in item] audit["image_gate"] = dict(image_audit) audit["selected_with_images"] = len(admitted) audit["training_ready"] = False audit["reason_not_training_ready"] = "GQA has no HFD question WAV, causal packet frontend, or motion/body binding in this pilot" _write_json(output / "gqa_candidate_manifest.json", admitted) _write_json(output / "gqa_selection_audit.json", audit) return audit def main() -> None: parser = argparse.ArgumentParser(description=__doc__) parser.add_argument("--source-plan", type=Path, required=True) parser.add_argument("--output-root", type=Path, required=True) parser.add_argument("--train", type=int, default=64, help="accepted HFD pilot rows") parser.add_argument("--val", type=int, default=16, help="accepted HFD pilot rows") parser.add_argument("--include-gqa", action="store_true") parser.add_argument("--gqa-train", type=int, default=32) parser.add_argument("--gqa-val", type=int, default=8) parser.add_argument("--gqa-scan-limit", type=int, default=50000) args = parser.parse_args() source_plan = args.source_plan.expanduser().resolve() output = args.output_root.expanduser().resolve() rows = json.loads(source_plan.read_text(encoding="utf-8")) selected, audit = select_hfd_rows(rows, train_limit=args.train, val_limit=args.val) _write_json(output / "hfd_candidate_pilot_source_plan.json", selected) _write_json(output / "hfd_candidate_pilot_audit.json", {**audit, "source_plan": str(source_plan), "schema": "hfd_vqa_candidate_pilot_v1"}) print(json.dumps(audit, ensure_ascii=False, indent=2), flush=True) if args.include_gqa: print(json.dumps(_stream_gqa(output, train_limit=args.gqa_train, val_limit=args.gqa_val, scan_limit=args.gqa_scan_limit), ensure_ascii=False, indent=2), flush=True) print(f"[ready] {output}", flush=True) if __name__ == "__main__": main()