#!/usr/bin/env python3 """Build a resumable, image-backed GQA expansion candidate set. This is deliberately independent from the old 12-row pilot and from the formal HFD source plan. It scans the complete balanced instruction splits, keeps one deterministic conservative visible-state question per image, then streams only the selected image records into an external output directory. The resulting manifest is *pending* visual/audio/frontend admission; it is not a training source plan. """ from __future__ import annotations import argparse from collections import Counter import hashlib import json from pathlib import Path from typing import Any from PIL import Image from build_hfd_vqa_candidate_pilot import conservative_gqa_row def atomic_json(path: Path, value: Any) -> None: path.parent.mkdir(parents=True, exist_ok=True) partial = path.with_name(path.name + ".partial") partial.write_text(json.dumps(value, ensure_ascii=False, indent=2) + "\n", encoding="utf-8") partial.replace(path) def score(row: dict[str, Any]) -> str: # Stable, source-order-independent tie break. Include the question id so # repeated questions for one image do not silently depend on parquet order. return hashlib.sha256((str(row.get("imageId")) + "\0" + str(row.get("id"))).encode()).hexdigest() def scan_instructions(split: str, limit: int, excluded_images: set[str]) -> tuple[list[dict[str, Any]], dict[str, Any]]: from datasets import load_dataset config = f"{split}_balanced_instructions" ds = load_dataset("lmms-lab-encoder/GQA", config, split=split, streaming=True) best: dict[str, dict[str, Any]] = {} rejected = Counter() scanned = 0 for raw in ds: scanned += 1 row = dict(raw) image_id = str(row.get("imageId", "")) if image_id in excluded_images: rejected["old_pilot_image"] += 1 continue accepted, reason = conservative_gqa_row(row) if not accepted: rejected[reason] += 1 continue item = {key: row.get(key) for key in ("id", "imageId", "question", "answer", "fullAnswer", "types", "semanticStr")} item.update({"split": split, "source_dataset": "GQA", "source_config": config, "status": "pending_blind_image_review_audio_frontend", "text_gate": "conservative_gqa_row_v1", "selection_score": score(row)}) old = best.get(image_id) if old is None or item["selection_score"] < old["selection_score"]: best[image_id] = item ordered = sorted(best.values(), key=lambda r: (str(r["selection_score"]), str(r["id"]))) selected = ordered[:limit] audit = {"split": split, "scanned": scanned, "accepted_unique_images": len(best), "selected": len(selected), "requested": limit, "rejected": dict(rejected), "question_ids_unique": len({str(r["id"]) for r in selected}), "image_ids_unique": len({str(r["imageId"]) for r in selected})} return selected, audit def fetch_images(root: Path, rows: list[dict[str, Any]]) -> tuple[list[dict[str, Any]], dict[str, Any]]: """Stream balanced image shards, materializing only wanted image ids. The routine is resumable: existing files are verified and reused. We do not hold parquet/image payloads in memory after each row. """ from datasets import load_dataset wanted = {(str(row["split"]), str(row["imageId"])): row for row in rows} found: dict[tuple[str, str], dict[str, Any]] = {} counters = Counter() by_split = {split: [key for key in wanted if key[0] == split] for split in ("train", "val")} for split in ("train", "val"): if not by_split[split]: continue ds = load_dataset("lmms-lab-encoder/GQA", f"{split}_balanced_images", split=split, streaming=True) for number, raw in enumerate(ds, 1): image_id = str(raw.get("id", "")) key = (split, image_id) if key not in wanted: continue row = wanted[key] path = root / "images" / split / f"{image_id}.jpg" path.parent.mkdir(parents=True, exist_ok=True) try: if path.is_file(): with Image.open(path) as image: image.load() width, height = image.size else: image = raw.get("image") if image is None: counters["missing_image"] += 1 continue image = image.convert("RGB") width, height = image.size if min(width, height) < 224: counters["image_too_small"] += 1 continue temporary = path.with_name(path.name + ".partial") image.save(temporary, format="JPEG", quality=95) temporary.replace(path) if min(width, height) < 224: counters["image_too_small"] += 1 continue digest = hashlib.sha256(path.read_bytes()).hexdigest() found[key] = {**row, "image_path": str(path.resolve()), "image_sha256": digest, "width": int(width), "height": int(height), "status": "pending_blind_image_review_audio_frontend"} counters["saved_or_reused"] += 1 except Exception as exc: # leave row pending for a later resume counters[f"image_error:{type(exc).__name__}"] += 1 if len(found) == len(wanted): break if number % 5000 == 0: print(f"images_scan split={split} rows={number} found={len(found)}/{len(wanted)}", flush=True) counters[f"{split}_image_rows_scanned"] = number admitted = [found[key] for key in wanted if key in found] # A hash collision here would indicate source corruption or a split leak. train_hashes = {r["image_sha256"] for r in admitted if r["split"] == "train"} val_hashes = {r["image_sha256"] for r in admitted if r["split"] == "val"} if train_hashes & val_hashes: raise RuntimeError("GQA candidate image content overlaps train and val") audit = {"requested": len(rows), "selected_with_images": len(admitted), "pending_image_rows": len(rows) - len(admitted), "image_gate": dict(counters), "cross_split_image_sha256_overlap": 0, "training_ready": False, "reason_not_training_ready": "blind_visual_review_and_question_audio_and_causal_frontend_pending"} return admitted, audit def main() -> None: parser = argparse.ArgumentParser(description=__doc__) parser.add_argument("--output-root", type=Path, required=True) parser.add_argument("--train", type=int, default=10000) parser.add_argument("--val", type=int, default=1000) parser.add_argument("--old-pilot", type=Path, default=None) args = parser.parse_args() root = args.output_root.expanduser().resolve() old_images: set[str] = set() if args.old_pilot and args.old_pilot.is_file(): old_images = {str(r.get("imageId")) for r in json.loads(args.old_pilot.read_text())} rows_train, audit_train = scan_instructions("train", args.train, old_images) rows_val, audit_val = scan_instructions("val", args.val, old_images) train_images = {str(r["imageId"]) for r in rows_train} rows_val = [r for r in rows_val if str(r["imageId"]) not in train_images] if len(rows_val) < args.val: raise RuntimeError(f"cross-split image disjointness reduced val below request: {len(rows_val)}/{args.val}") rows = rows_train + rows_val[:args.val] atomic_json(root / "gqa_text_candidates.json", rows) atomic_json(root / "gqa_text_selection_audit.json", {"dataset": "lmms-lab-encoder/GQA", "train": audit_train, "val": {**audit_val, "selected_after_cross_split": len(rows_val[:args.val])}, "old_pilot_images_excluded": len(old_images), "new_candidate_rows": len(rows), "new_image_ids": len({str(r["imageId"]) for r in rows}), "training_ready": False}) admitted, image_audit = fetch_images(root, rows) atomic_json(root / "gqa_candidate_manifest.json", admitted) atomic_json(root / "gqa_image_selection_audit.json", {"text": {"train": audit_train, "val": audit_val}, **image_audit, "status": "pending_blind_image_review_audio_frontend", "source_plan_modified": False}) print(json.dumps({"text_candidates": len(rows), "with_images": len(admitted), "train_with_images": sum(r["split"] == "train" for r in admitted), "val_with_images": sum(r["split"] == "val" for r in admitted), "audit": image_audit}, ensure_ascii=False, indent=2), flush=True) if __name__ == "__main__": main()