code-HFD / scripts /data /build_hfd_vqa_candidate_pilot.py
Tinghong-Ye22's picture
Add files using upload-large-folder tool
9fc9873 verified
Raw History Blame Contribute Delete
14.4 kB
#!/usr/bin/env python3
"""Build a small, auditable VQA admission pilot without touching formal data.
Two outputs are produced under an external directory:
* a training-ready *subset* of the already encoded HFD source plan. It uses
only official-human temporal Charades-Ego rows and high-agreement VQAv2
rows, keeps the original body/frontend references, and is suitable for a
tiny opt-in HFD run;
* an optional GQA screening manifest. GQA rows are deliberately marked
``pending_audio_frontend``: downloading an image and accepting its text is
not enough to pretend that it is a causal HFD source.
The formal ``source_plan_5x.json`` is never rewritten. This is an admission
pilot and a reproducible quality gate, not a silent dataset replacement.
"""
from __future__ import annotations
import argparse
import hashlib
import json
import re
from collections import Counter
from pathlib import Path
from typing import Any, Iterable, Mapping
HFD_COHORT_QUALITY = {
"charades": "official_human_temporal_action_annotation_derived_binary_QA",
"vqav2_static": "human_agreement_>=9_and_independent_image_review",
}
def _stable_key(row: Mapping[str, Any]) -> str:
value = "|".join(
str(row.get(key, ""))
for key in ("id", "gt_question_id", "path", "motion_name", "split")
)
return hashlib.sha256(value.encode("utf-8")).hexdigest()
def _nested_question(row: Mapping[str, Any]) -> Mapping[str, Any]:
value = row.get("question")
return value if isinstance(value, Mapping) else {}
def _source_cohort(row: Mapping[str, Any]) -> str:
return str(row.get("cohort") or row.get("data_domain") or "")
def _quality_gate(row: Mapping[str, Any]) -> tuple[bool, str]:
"""Reject rows whose provenance or causal frontend is not auditable."""
cohort = _source_cohort(row)
if cohort not in HFD_COHORT_QUALITY:
return False, "cohort_not_in_conservative_pilot"
question = _nested_question(row)
if question.get("swa_has_question") is False:
return False, "explicit_non_question_row"
if not str(question.get("question", "")).strip() or not str(question.get("answer", "")).strip():
return False, "empty_question_or_answer"
path = Path(str(row.get("path", "")))
if not path.is_file():
return False, "body_path_missing"
frontend_root = Path(str(row.get("frontend_root", "")))
frontend = frontend_root / "data" / "frontend" / f"{row.get('id')}.pt"
# The static VQAv2 frontend can be factorized/inline; the row's timeline
# remains the authoritative existence check in that case.
timeline = row.get("timeline")
question_voice = timeline.get("question_voice") if isinstance(timeline, Mapping) else None
if question_voice and not Path(str(question_voice)).is_file():
return False, "question_audio_missing"
if not question_voice and not frontend.is_file():
return False, "frontend_missing"
if cohort == "charades":
provenance = str(question.get("gt_provenance", ""))
if provenance != HFD_COHORT_QUALITY[cohort]:
return False, "charades_provenance_not_human_temporal"
if cohort == "vqav2_static":
votes = int(question.get("human_votes", 0) or 0)
# The recovered source-plan schema stores the independent blind image
# review at row level. Older pilot rows put it inside ``question``;
# accept both locations, but never weaken the gate when neither exists.
review = row.get("visual_review") or question.get("visual_review")
if votes < 9 or not isinstance(review, Mapping) or review.get("accepted") is not True:
return False, "vqav2_agreement_or_visual_review_failed"
return True, "accepted"
def select_hfd_rows(
rows: Iterable[Mapping[str, Any]], *, train_limit: int, val_limit: int, seed: int = 260922
) -> tuple[list[dict[str, Any]], dict[str, Any]]:
"""Select disjoint, deterministic, source-plan-backed pilot rows."""
requested = {"train": int(train_limit), "val": int(val_limit)}
candidates: dict[str, list[dict[str, Any]]] = {"train": [], "val": []}
rejected = Counter()
for raw in rows:
row = dict(raw)
split = str(row.get("split", ""))
if split not in candidates:
rejected["split_not_train_or_val"] += 1
continue
accepted, reason = _quality_gate(row)
if not accepted:
rejected[reason] += 1
continue
row["pilot_quality"] = HFD_COHORT_QUALITY[_source_cohort(row)]
row["pilot_seed"] = int(seed)
candidates[split].append(row)
selected: list[dict[str, Any]] = []
counts = Counter()
# Hash ordering avoids source-plan ordering or motion-family clustering.
for split in ("train", "val"):
# Round-robin the accepted cohorts. A plain global hash sort can
# accidentally make a tiny pilot all VQAv2 (or all Charades), which
# tests only one upstream source while appearing to test two.
pools = {
cohort: sorted(
(row for row in candidates[split] if _source_cohort(row) == cohort),
key=_stable_key,
)
for cohort in sorted({_source_cohort(row) for row in candidates[split]})
}
used_body: set[str] = set()
cursors = {cohort: 0 for cohort in pools}
while sum(1 for item in selected if item.get("split") == split) < requested[split]:
made_progress = False
for cohort, pool in pools.items():
while cursors[cohort] < len(pool):
row = pool[cursors[cohort]]
cursors[cohort] += 1
body = str(row.get("path", ""))
if body in used_body:
continue
selected.append(row)
used_body.add(body)
counts[f"{split}:{cohort}"] += 1
made_progress = True
break
if sum(1 for item in selected if item.get("split") == split) >= requested[split]:
break
if not made_progress:
break
if sum(1 for item in selected if item.get("split") == split) < requested[split]:
raise ValueError(
f"not enough disjoint accepted rows for {split}: "
f"{sum(1 for item in selected if item.get('split') == split)}/{requested[split]}"
)
selected.sort(key=lambda row: (str(row.get("split")), _stable_key(row)))
return selected, {
"requested": requested,
"accepted_pool": {split: len(value) for split, value in candidates.items()},
"selected": len(selected),
"selected_by_split": dict(Counter(str(row["split"]) for row in selected)),
"selected_by_cohort": dict(counts),
"rejected": dict(rejected),
"formal_source_plan_changed": False,
}
_GQA_BAD = re.compile(
r"\b(why|might|could|would|should|probably|before|after|future|written|text|"
r"says|read|brand|license|country|city|name|feel|taste|famous|occupation|"
r"dangerous|safe|injured|how old|time of day)\b",
re.IGNORECASE,
)
def conservative_gqa_row(row: Mapping[str, Any]) -> tuple[bool, str]:
"""Conservative text-side gate for streamed GQA, before image review."""
question = " ".join(str(row.get("question", "")).split())
answer = " ".join(str(row.get("answer", "")).split())
words = re.findall(r"[A-Za-z0-9']+", question)
answer_words = re.findall(r"[A-Za-z0-9']+", answer)
if row.get("isBalanced") is not True:
return False, "not_balanced"
if not (3 <= len(words) <= 18):
return False, "question_length"
if not (1 <= len(answer_words) <= 6):
return False, "answer_length"
if _GQA_BAD.search(question):
return False, "speculative_or_text_question"
if answer.lower() in {"unknown", "unanswerable", "not sure", "can't tell", "cant tell"}:
return False, "unknown_answer"
types = row.get("types")
if isinstance(types, Mapping) and str(types.get("structural", "")) not in {"verify", "query"}:
return False, "unsupported_gqa_type"
return True, "accepted_text_gate_pending_image_review"
def _write_json(path: Path, value: Any) -> None:
path.parent.mkdir(parents=True, exist_ok=True)
temporary = path.with_name(path.name + ".partial")
temporary.write_text(json.dumps(value, ensure_ascii=False, indent=2) + "\n", encoding="utf-8")
temporary.replace(path)
def _stream_gqa(output: Path, *, train_limit: int, val_limit: int, scan_limit: int) -> dict[str, Any]:
try:
from datasets import load_dataset
except ImportError as exc: # pragma: no cover - optional data-side dependency
raise RuntimeError("GQA pilot requires the optional `datasets` package") from exc
selected: list[dict[str, Any]] = []
audit: dict[str, Any] = {"dataset": "lmms-lab-encoder/GQA", "splits": {}, "status": "text_screened"}
for split, limit in (("train", train_limit), ("val", val_limit)):
config = f"{split}_balanced_instructions"
ds = load_dataset("lmms-lab-encoder/GQA", config, split=split, streaming=True)
pool: list[dict[str, Any]] = []
rejected = Counter()
for index, row in enumerate(ds):
if index >= scan_limit:
break
ok, reason = conservative_gqa_row(row)
if not ok:
rejected[reason] += 1
continue
item = {key: row.get(key) for key in ("id", "imageId", "question", "answer", "fullAnswer", "types", "semanticStr")}
item.update({"split": split, "source_dataset": "GQA", "status": "pending_audio_frontend_and_blind_image_review"})
pool.append(item)
# The image stream is ordered by image id. Selecting the earliest
# accepted unique ids makes this small pilot finish after reading only
# a tiny prefix; the full admission job can use a hash-ordered sample
# once an offline image index exists.
pool.sort(key=lambda item: (int(item["imageId"]), str(item["id"])))
unique_images: set[str] = set()
for item in pool:
image_id = str(item["imageId"])
if image_id in unique_images:
continue
selected.append(item)
unique_images.add(image_id)
if sum(1 for x in selected if x["split"] == split) >= limit:
break
audit["splits"][split] = {"scanned": min(index + 1, scan_limit), "accepted_text_pool": len(pool), "selected": sum(1 for x in selected if x["split"] == split), "rejected": dict(rejected)}
if {str(x["imageId"]) for x in selected if x["split"] == "train"} & {str(x["imageId"]) for x in selected if x["split"] == "val"}:
raise RuntimeError("GQA pilot has a cross-split image collision")
image_root = output / "gqa_images"
image_root.mkdir(parents=True, exist_ok=True)
wanted = {str(x["imageId"]): x for x in selected}
image_audit = Counter()
for split in ("train", "val"):
split_wanted = {key: value for key, value in wanted.items() if value["split"] == split}
if not split_wanted:
continue
ds = load_dataset("lmms-lab-encoder/GQA", f"{split}_balanced_images", split=split, streaming=True)
for row in ds:
image_id = str(row.get("id"))
if image_id not in split_wanted:
continue
image = row.get("image")
if image is None:
image_audit["missing_image"] += 1
continue
image = image.convert("RGB")
if min(image.size) < 224:
image_audit["image_too_small"] += 1
continue
path = image_root / split / f"{image_id}.jpg"
path.parent.mkdir(parents=True, exist_ok=True)
image.save(path, format="JPEG", quality=95)
digest = hashlib.sha256(path.read_bytes()).hexdigest()
split_wanted[image_id].update({"image_path": str(path.resolve()), "image_sha256": digest, "width": image.width, "height": image.height, "status": "screened_text_and_image_geometry_pending_semantic_review"})
image_audit["saved"] += 1
if all("image_path" in value for value in split_wanted.values()):
break
admitted = [item for item in selected if "image_path" in item]
audit["image_gate"] = dict(image_audit)
audit["selected_with_images"] = len(admitted)
audit["training_ready"] = False
audit["reason_not_training_ready"] = "GQA has no HFD question WAV, causal packet frontend, or motion/body binding in this pilot"
_write_json(output / "gqa_candidate_manifest.json", admitted)
_write_json(output / "gqa_selection_audit.json", audit)
return audit
def main() -> None:
parser = argparse.ArgumentParser(description=__doc__)
parser.add_argument("--source-plan", type=Path, required=True)
parser.add_argument("--output-root", type=Path, required=True)
parser.add_argument("--train", type=int, default=64, help="accepted HFD pilot rows")
parser.add_argument("--val", type=int, default=16, help="accepted HFD pilot rows")
parser.add_argument("--include-gqa", action="store_true")
parser.add_argument("--gqa-train", type=int, default=32)
parser.add_argument("--gqa-val", type=int, default=8)
parser.add_argument("--gqa-scan-limit", type=int, default=50000)
args = parser.parse_args()
source_plan = args.source_plan.expanduser().resolve()
output = args.output_root.expanduser().resolve()
rows = json.loads(source_plan.read_text(encoding="utf-8"))
selected, audit = select_hfd_rows(rows, train_limit=args.train, val_limit=args.val)
_write_json(output / "hfd_candidate_pilot_source_plan.json", selected)
_write_json(output / "hfd_candidate_pilot_audit.json", {**audit, "source_plan": str(source_plan), "schema": "hfd_vqa_candidate_pilot_v1"})
print(json.dumps(audit, ensure_ascii=False, indent=2), flush=True)
if args.include_gqa:
print(json.dumps(_stream_gqa(output, train_limit=args.gqa_train, val_limit=args.gqa_val, scan_limit=args.gqa_scan_limit), ensure_ascii=False, indent=2), flush=True)
print(f"[ready] {output}", flush=True)
if __name__ == "__main__":
main()