Download scripts/data/build_hfd_vqa_candidate_pilot.py from Tinghong-Ye22/code-HFD: direct link, hf CLI and curl.
- Browser
- Download file 14.4 kB
-
https://huggingface.co/Tinghong-Ye22/code-HFD/resolve/main/scripts/data/build_hfd_vqa_candidate_pilot.py
- Command line
-
hf download hf://Tinghong-Ye22/code-HFD/scripts/data/build_hfd_vqa_candidate_pilot.py
-
curl -L -o build_hfd_vqa_candidate_pilot.py https://huggingface.co/Tinghong-Ye22/code-HFD/resolve/main/scripts/data/build_hfd_vqa_candidate_pilot.py
14.4 kB
| #!/usr/bin/env python3 | |
| """Build a small, auditable VQA admission pilot without touching formal data. | |
| Two outputs are produced under an external directory: | |
| * a training-ready *subset* of the already encoded HFD source plan. It uses | |
| only official-human temporal Charades-Ego rows and high-agreement VQAv2 | |
| rows, keeps the original body/frontend references, and is suitable for a | |
| tiny opt-in HFD run; | |
| * an optional GQA screening manifest. GQA rows are deliberately marked | |
| ``pending_audio_frontend``: downloading an image and accepting its text is | |
| not enough to pretend that it is a causal HFD source. | |
| The formal ``source_plan_5x.json`` is never rewritten. This is an admission | |
| pilot and a reproducible quality gate, not a silent dataset replacement. | |
| """ | |
| from __future__ import annotations | |
| import argparse | |
| import hashlib | |
| import json | |
| import re | |
| from collections import Counter | |
| from pathlib import Path | |
| from typing import Any, Iterable, Mapping | |
| HFD_COHORT_QUALITY = { | |
| "charades": "official_human_temporal_action_annotation_derived_binary_QA", | |
| "vqav2_static": "human_agreement_>=9_and_independent_image_review", | |
| } | |
| def _stable_key(row: Mapping[str, Any]) -> str: | |
| value = "|".join( | |
| str(row.get(key, "")) | |
| for key in ("id", "gt_question_id", "path", "motion_name", "split") | |
| ) | |
| return hashlib.sha256(value.encode("utf-8")).hexdigest() | |
| def _nested_question(row: Mapping[str, Any]) -> Mapping[str, Any]: | |
| value = row.get("question") | |
| return value if isinstance(value, Mapping) else {} | |
| def _source_cohort(row: Mapping[str, Any]) -> str: | |
| return str(row.get("cohort") or row.get("data_domain") or "") | |
| def _quality_gate(row: Mapping[str, Any]) -> tuple[bool, str]: | |
| """Reject rows whose provenance or causal frontend is not auditable.""" | |
| cohort = _source_cohort(row) | |
| if cohort not in HFD_COHORT_QUALITY: | |
| return False, "cohort_not_in_conservative_pilot" | |
| question = _nested_question(row) | |
| if question.get("swa_has_question") is False: | |
| return False, "explicit_non_question_row" | |
| if not str(question.get("question", "")).strip() or not str(question.get("answer", "")).strip(): | |
| return False, "empty_question_or_answer" | |
| path = Path(str(row.get("path", ""))) | |
| if not path.is_file(): | |
| return False, "body_path_missing" | |
| frontend_root = Path(str(row.get("frontend_root", ""))) | |
| frontend = frontend_root / "data" / "frontend" / f"{row.get('id')}.pt" | |
| # The static VQAv2 frontend can be factorized/inline; the row's timeline | |
| # remains the authoritative existence check in that case. | |
| timeline = row.get("timeline") | |
| question_voice = timeline.get("question_voice") if isinstance(timeline, Mapping) else None | |
| if question_voice and not Path(str(question_voice)).is_file(): | |
| return False, "question_audio_missing" | |
| if not question_voice and not frontend.is_file(): | |
| return False, "frontend_missing" | |
| if cohort == "charades": | |
| provenance = str(question.get("gt_provenance", "")) | |
| if provenance != HFD_COHORT_QUALITY[cohort]: | |
| return False, "charades_provenance_not_human_temporal" | |
| if cohort == "vqav2_static": | |
| votes = int(question.get("human_votes", 0) or 0) | |
| # The recovered source-plan schema stores the independent blind image | |
| # review at row level. Older pilot rows put it inside ``question``; | |
| # accept both locations, but never weaken the gate when neither exists. | |
| review = row.get("visual_review") or question.get("visual_review") | |
| if votes < 9 or not isinstance(review, Mapping) or review.get("accepted") is not True: | |
| return False, "vqav2_agreement_or_visual_review_failed" | |
| return True, "accepted" | |
| def select_hfd_rows( | |
| rows: Iterable[Mapping[str, Any]], *, train_limit: int, val_limit: int, seed: int = 260922 | |
| ) -> tuple[list[dict[str, Any]], dict[str, Any]]: | |
| """Select disjoint, deterministic, source-plan-backed pilot rows.""" | |
| requested = {"train": int(train_limit), "val": int(val_limit)} | |
| candidates: dict[str, list[dict[str, Any]]] = {"train": [], "val": []} | |
| rejected = Counter() | |
| for raw in rows: | |
| row = dict(raw) | |
| split = str(row.get("split", "")) | |
| if split not in candidates: | |
| rejected["split_not_train_or_val"] += 1 | |
| continue | |
| accepted, reason = _quality_gate(row) | |
| if not accepted: | |
| rejected[reason] += 1 | |
| continue | |
| row["pilot_quality"] = HFD_COHORT_QUALITY[_source_cohort(row)] | |
| row["pilot_seed"] = int(seed) | |
| candidates[split].append(row) | |
| selected: list[dict[str, Any]] = [] | |
| counts = Counter() | |
| # Hash ordering avoids source-plan ordering or motion-family clustering. | |
| for split in ("train", "val"): | |
| # Round-robin the accepted cohorts. A plain global hash sort can | |
| # accidentally make a tiny pilot all VQAv2 (or all Charades), which | |
| # tests only one upstream source while appearing to test two. | |
| pools = { | |
| cohort: sorted( | |
| (row for row in candidates[split] if _source_cohort(row) == cohort), | |
| key=_stable_key, | |
| ) | |
| for cohort in sorted({_source_cohort(row) for row in candidates[split]}) | |
| } | |
| used_body: set[str] = set() | |
| cursors = {cohort: 0 for cohort in pools} | |
| while sum(1 for item in selected if item.get("split") == split) < requested[split]: | |
| made_progress = False | |
| for cohort, pool in pools.items(): | |
| while cursors[cohort] < len(pool): | |
| row = pool[cursors[cohort]] | |
| cursors[cohort] += 1 | |
| body = str(row.get("path", "")) | |
| if body in used_body: | |
| continue | |
| selected.append(row) | |
| used_body.add(body) | |
| counts[f"{split}:{cohort}"] += 1 | |
| made_progress = True | |
| break | |
| if sum(1 for item in selected if item.get("split") == split) >= requested[split]: | |
| break | |
| if not made_progress: | |
| break | |
| if sum(1 for item in selected if item.get("split") == split) < requested[split]: | |
| raise ValueError( | |
| f"not enough disjoint accepted rows for {split}: " | |
| f"{sum(1 for item in selected if item.get('split') == split)}/{requested[split]}" | |
| ) | |
| selected.sort(key=lambda row: (str(row.get("split")), _stable_key(row))) | |
| return selected, { | |
| "requested": requested, | |
| "accepted_pool": {split: len(value) for split, value in candidates.items()}, | |
| "selected": len(selected), | |
| "selected_by_split": dict(Counter(str(row["split"]) for row in selected)), | |
| "selected_by_cohort": dict(counts), | |
| "rejected": dict(rejected), | |
| "formal_source_plan_changed": False, | |
| } | |
| _GQA_BAD = re.compile( | |
| r"\b(why|might|could|would|should|probably|before|after|future|written|text|" | |
| r"says|read|brand|license|country|city|name|feel|taste|famous|occupation|" | |
| r"dangerous|safe|injured|how old|time of day)\b", | |
| re.IGNORECASE, | |
| ) | |
| def conservative_gqa_row(row: Mapping[str, Any]) -> tuple[bool, str]: | |
| """Conservative text-side gate for streamed GQA, before image review.""" | |
| question = " ".join(str(row.get("question", "")).split()) | |
| answer = " ".join(str(row.get("answer", "")).split()) | |
| words = re.findall(r"[A-Za-z0-9']+", question) | |
| answer_words = re.findall(r"[A-Za-z0-9']+", answer) | |
| if row.get("isBalanced") is not True: | |
| return False, "not_balanced" | |
| if not (3 <= len(words) <= 18): | |
| return False, "question_length" | |
| if not (1 <= len(answer_words) <= 6): | |
| return False, "answer_length" | |
| if _GQA_BAD.search(question): | |
| return False, "speculative_or_text_question" | |
| if answer.lower() in {"unknown", "unanswerable", "not sure", "can't tell", "cant tell"}: | |
| return False, "unknown_answer" | |
| types = row.get("types") | |
| if isinstance(types, Mapping) and str(types.get("structural", "")) not in {"verify", "query"}: | |
| return False, "unsupported_gqa_type" | |
| return True, "accepted_text_gate_pending_image_review" | |
| def _write_json(path: Path, value: Any) -> None: | |
| path.parent.mkdir(parents=True, exist_ok=True) | |
| temporary = path.with_name(path.name + ".partial") | |
| temporary.write_text(json.dumps(value, ensure_ascii=False, indent=2) + "\n", encoding="utf-8") | |
| temporary.replace(path) | |
| def _stream_gqa(output: Path, *, train_limit: int, val_limit: int, scan_limit: int) -> dict[str, Any]: | |
| try: | |
| from datasets import load_dataset | |
| except ImportError as exc: # pragma: no cover - optional data-side dependency | |
| raise RuntimeError("GQA pilot requires the optional `datasets` package") from exc | |
| selected: list[dict[str, Any]] = [] | |
| audit: dict[str, Any] = {"dataset": "lmms-lab-encoder/GQA", "splits": {}, "status": "text_screened"} | |
| for split, limit in (("train", train_limit), ("val", val_limit)): | |
| config = f"{split}_balanced_instructions" | |
| ds = load_dataset("lmms-lab-encoder/GQA", config, split=split, streaming=True) | |
| pool: list[dict[str, Any]] = [] | |
| rejected = Counter() | |
| for index, row in enumerate(ds): | |
| if index >= scan_limit: | |
| break | |
| ok, reason = conservative_gqa_row(row) | |
| if not ok: | |
| rejected[reason] += 1 | |
| continue | |
| item = {key: row.get(key) for key in ("id", "imageId", "question", "answer", "fullAnswer", "types", "semanticStr")} | |
| item.update({"split": split, "source_dataset": "GQA", "status": "pending_audio_frontend_and_blind_image_review"}) | |
| pool.append(item) | |
| # The image stream is ordered by image id. Selecting the earliest | |
| # accepted unique ids makes this small pilot finish after reading only | |
| # a tiny prefix; the full admission job can use a hash-ordered sample | |
| # once an offline image index exists. | |
| pool.sort(key=lambda item: (int(item["imageId"]), str(item["id"]))) | |
| unique_images: set[str] = set() | |
| for item in pool: | |
| image_id = str(item["imageId"]) | |
| if image_id in unique_images: | |
| continue | |
| selected.append(item) | |
| unique_images.add(image_id) | |
| if sum(1 for x in selected if x["split"] == split) >= limit: | |
| break | |
| audit["splits"][split] = {"scanned": min(index + 1, scan_limit), "accepted_text_pool": len(pool), "selected": sum(1 for x in selected if x["split"] == split), "rejected": dict(rejected)} | |
| if {str(x["imageId"]) for x in selected if x["split"] == "train"} & {str(x["imageId"]) for x in selected if x["split"] == "val"}: | |
| raise RuntimeError("GQA pilot has a cross-split image collision") | |
| image_root = output / "gqa_images" | |
| image_root.mkdir(parents=True, exist_ok=True) | |
| wanted = {str(x["imageId"]): x for x in selected} | |
| image_audit = Counter() | |
| for split in ("train", "val"): | |
| split_wanted = {key: value for key, value in wanted.items() if value["split"] == split} | |
| if not split_wanted: | |
| continue | |
| ds = load_dataset("lmms-lab-encoder/GQA", f"{split}_balanced_images", split=split, streaming=True) | |
| for row in ds: | |
| image_id = str(row.get("id")) | |
| if image_id not in split_wanted: | |
| continue | |
| image = row.get("image") | |
| if image is None: | |
| image_audit["missing_image"] += 1 | |
| continue | |
| image = image.convert("RGB") | |
| if min(image.size) < 224: | |
| image_audit["image_too_small"] += 1 | |
| continue | |
| path = image_root / split / f"{image_id}.jpg" | |
| path.parent.mkdir(parents=True, exist_ok=True) | |
| image.save(path, format="JPEG", quality=95) | |
| digest = hashlib.sha256(path.read_bytes()).hexdigest() | |
| split_wanted[image_id].update({"image_path": str(path.resolve()), "image_sha256": digest, "width": image.width, "height": image.height, "status": "screened_text_and_image_geometry_pending_semantic_review"}) | |
| image_audit["saved"] += 1 | |
| if all("image_path" in value for value in split_wanted.values()): | |
| break | |
| admitted = [item for item in selected if "image_path" in item] | |
| audit["image_gate"] = dict(image_audit) | |
| audit["selected_with_images"] = len(admitted) | |
| audit["training_ready"] = False | |
| audit["reason_not_training_ready"] = "GQA has no HFD question WAV, causal packet frontend, or motion/body binding in this pilot" | |
| _write_json(output / "gqa_candidate_manifest.json", admitted) | |
| _write_json(output / "gqa_selection_audit.json", audit) | |
| return audit | |
| def main() -> None: | |
| parser = argparse.ArgumentParser(description=__doc__) | |
| parser.add_argument("--source-plan", type=Path, required=True) | |
| parser.add_argument("--output-root", type=Path, required=True) | |
| parser.add_argument("--train", type=int, default=64, help="accepted HFD pilot rows") | |
| parser.add_argument("--val", type=int, default=16, help="accepted HFD pilot rows") | |
| parser.add_argument("--include-gqa", action="store_true") | |
| parser.add_argument("--gqa-train", type=int, default=32) | |
| parser.add_argument("--gqa-val", type=int, default=8) | |
| parser.add_argument("--gqa-scan-limit", type=int, default=50000) | |
| args = parser.parse_args() | |
| source_plan = args.source_plan.expanduser().resolve() | |
| output = args.output_root.expanduser().resolve() | |
| rows = json.loads(source_plan.read_text(encoding="utf-8")) | |
| selected, audit = select_hfd_rows(rows, train_limit=args.train, val_limit=args.val) | |
| _write_json(output / "hfd_candidate_pilot_source_plan.json", selected) | |
| _write_json(output / "hfd_candidate_pilot_audit.json", {**audit, "source_plan": str(source_plan), "schema": "hfd_vqa_candidate_pilot_v1"}) | |
| print(json.dumps(audit, ensure_ascii=False, indent=2), flush=True) | |
| if args.include_gqa: | |
| print(json.dumps(_stream_gqa(output, train_limit=args.gqa_train, val_limit=args.gqa_val, scan_limit=args.gqa_scan_limit), ensure_ascii=False, indent=2), flush=True) | |
| print(f"[ready] {output}", flush=True) | |
| if __name__ == "__main__": | |
| main() | |