code-HFD / scripts /data /build_hfd_gqa_scale.py
Tinghong-Ye22's picture
Add files using upload-large-folder tool
9fc9873 verified
Raw History Blame Contribute Delete
8.95 kB
#!/usr/bin/env python3
"""Build a resumable, image-backed GQA expansion candidate set.
This is deliberately independent from the old 12-row pilot and from the
formal HFD source plan. It scans the complete balanced instruction splits,
keeps one deterministic conservative visible-state question per image, then
streams only the selected image records into an external output directory.
The resulting manifest is *pending* visual/audio/frontend admission; it is
not a training source plan.
"""
from __future__ import annotations
import argparse
from collections import Counter
import hashlib
import json
from pathlib import Path
from typing import Any
from PIL import Image
from build_hfd_vqa_candidate_pilot import conservative_gqa_row
def atomic_json(path: Path, value: Any) -> None:
path.parent.mkdir(parents=True, exist_ok=True)
partial = path.with_name(path.name + ".partial")
partial.write_text(json.dumps(value, ensure_ascii=False, indent=2) + "\n", encoding="utf-8")
partial.replace(path)
def score(row: dict[str, Any]) -> str:
# Stable, source-order-independent tie break. Include the question id so
# repeated questions for one image do not silently depend on parquet order.
return hashlib.sha256((str(row.get("imageId")) + "\0" + str(row.get("id"))).encode()).hexdigest()
def scan_instructions(split: str, limit: int, excluded_images: set[str]) -> tuple[list[dict[str, Any]], dict[str, Any]]:
from datasets import load_dataset
config = f"{split}_balanced_instructions"
ds = load_dataset("lmms-lab-encoder/GQA", config, split=split, streaming=True)
best: dict[str, dict[str, Any]] = {}
rejected = Counter()
scanned = 0
for raw in ds:
scanned += 1
row = dict(raw)
image_id = str(row.get("imageId", ""))
if image_id in excluded_images:
rejected["old_pilot_image"] += 1
continue
accepted, reason = conservative_gqa_row(row)
if not accepted:
rejected[reason] += 1
continue
item = {key: row.get(key) for key in ("id", "imageId", "question", "answer", "fullAnswer", "types", "semanticStr")}
item.update({"split": split, "source_dataset": "GQA", "source_config": config,
"status": "pending_blind_image_review_audio_frontend", "text_gate": "conservative_gqa_row_v1",
"selection_score": score(row)})
old = best.get(image_id)
if old is None or item["selection_score"] < old["selection_score"]:
best[image_id] = item
ordered = sorted(best.values(), key=lambda r: (str(r["selection_score"]), str(r["id"])))
selected = ordered[:limit]
audit = {"split": split, "scanned": scanned, "accepted_unique_images": len(best),
"selected": len(selected), "requested": limit, "rejected": dict(rejected),
"question_ids_unique": len({str(r["id"]) for r in selected}),
"image_ids_unique": len({str(r["imageId"]) for r in selected})}
return selected, audit
def fetch_images(root: Path, rows: list[dict[str, Any]]) -> tuple[list[dict[str, Any]], dict[str, Any]]:
"""Stream balanced image shards, materializing only wanted image ids.
The routine is resumable: existing files are verified and reused. We do
not hold parquet/image payloads in memory after each row.
"""
from datasets import load_dataset
wanted = {(str(row["split"]), str(row["imageId"])): row for row in rows}
found: dict[tuple[str, str], dict[str, Any]] = {}
counters = Counter()
by_split = {split: [key for key in wanted if key[0] == split] for split in ("train", "val")}
for split in ("train", "val"):
if not by_split[split]:
continue
ds = load_dataset("lmms-lab-encoder/GQA", f"{split}_balanced_images", split=split, streaming=True)
for number, raw in enumerate(ds, 1):
image_id = str(raw.get("id", ""))
key = (split, image_id)
if key not in wanted:
continue
row = wanted[key]
path = root / "images" / split / f"{image_id}.jpg"
path.parent.mkdir(parents=True, exist_ok=True)
try:
if path.is_file():
with Image.open(path) as image:
image.load()
width, height = image.size
else:
image = raw.get("image")
if image is None:
counters["missing_image"] += 1
continue
image = image.convert("RGB")
width, height = image.size
if min(width, height) < 224:
counters["image_too_small"] += 1
continue
temporary = path.with_name(path.name + ".partial")
image.save(temporary, format="JPEG", quality=95)
temporary.replace(path)
if min(width, height) < 224:
counters["image_too_small"] += 1
continue
digest = hashlib.sha256(path.read_bytes()).hexdigest()
found[key] = {**row, "image_path": str(path.resolve()), "image_sha256": digest,
"width": int(width), "height": int(height),
"status": "pending_blind_image_review_audio_frontend"}
counters["saved_or_reused"] += 1
except Exception as exc: # leave row pending for a later resume
counters[f"image_error:{type(exc).__name__}"] += 1
if len(found) == len(wanted):
break
if number % 5000 == 0:
print(f"images_scan split={split} rows={number} found={len(found)}/{len(wanted)}", flush=True)
counters[f"{split}_image_rows_scanned"] = number
admitted = [found[key] for key in wanted if key in found]
# A hash collision here would indicate source corruption or a split leak.
train_hashes = {r["image_sha256"] for r in admitted if r["split"] == "train"}
val_hashes = {r["image_sha256"] for r in admitted if r["split"] == "val"}
if train_hashes & val_hashes:
raise RuntimeError("GQA candidate image content overlaps train and val")
audit = {"requested": len(rows), "selected_with_images": len(admitted),
"pending_image_rows": len(rows) - len(admitted), "image_gate": dict(counters),
"cross_split_image_sha256_overlap": 0,
"training_ready": False,
"reason_not_training_ready": "blind_visual_review_and_question_audio_and_causal_frontend_pending"}
return admitted, audit
def main() -> None:
parser = argparse.ArgumentParser(description=__doc__)
parser.add_argument("--output-root", type=Path, required=True)
parser.add_argument("--train", type=int, default=10000)
parser.add_argument("--val", type=int, default=1000)
parser.add_argument("--old-pilot", type=Path, default=None)
args = parser.parse_args()
root = args.output_root.expanduser().resolve()
old_images: set[str] = set()
if args.old_pilot and args.old_pilot.is_file():
old_images = {str(r.get("imageId")) for r in json.loads(args.old_pilot.read_text())}
rows_train, audit_train = scan_instructions("train", args.train, old_images)
rows_val, audit_val = scan_instructions("val", args.val, old_images)
train_images = {str(r["imageId"]) for r in rows_train}
rows_val = [r for r in rows_val if str(r["imageId"]) not in train_images]
if len(rows_val) < args.val:
raise RuntimeError(f"cross-split image disjointness reduced val below request: {len(rows_val)}/{args.val}")
rows = rows_train + rows_val[:args.val]
atomic_json(root / "gqa_text_candidates.json", rows)
atomic_json(root / "gqa_text_selection_audit.json", {"dataset": "lmms-lab-encoder/GQA",
"train": audit_train, "val": {**audit_val, "selected_after_cross_split": len(rows_val[:args.val])},
"old_pilot_images_excluded": len(old_images), "new_candidate_rows": len(rows),
"new_image_ids": len({str(r["imageId"]) for r in rows}), "training_ready": False})
admitted, image_audit = fetch_images(root, rows)
atomic_json(root / "gqa_candidate_manifest.json", admitted)
atomic_json(root / "gqa_image_selection_audit.json", {"text": {"train": audit_train, "val": audit_val}, **image_audit,
"status": "pending_blind_image_review_audio_frontend", "source_plan_modified": False})
print(json.dumps({"text_candidates": len(rows), "with_images": len(admitted),
"train_with_images": sum(r["split"] == "train" for r in admitted),
"val_with_images": sum(r["split"] == "val" for r in admitted),
"audit": image_audit}, ensure_ascii=False, indent=2), flush=True)
if __name__ == "__main__":
main()