File size: 8,952 Bytes
9fc9873 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 151 152 153 154 155 156 157 158 159 160 161 162 163 164 165 166 167 168 169 170 171 172 173 174 175 176 177 178 179 180 | #!/usr/bin/env python3
"""Build a resumable, image-backed GQA expansion candidate set.
This is deliberately independent from the old 12-row pilot and from the
formal HFD source plan. It scans the complete balanced instruction splits,
keeps one deterministic conservative visible-state question per image, then
streams only the selected image records into an external output directory.
The resulting manifest is *pending* visual/audio/frontend admission; it is
not a training source plan.
"""
from __future__ import annotations
import argparse
from collections import Counter
import hashlib
import json
from pathlib import Path
from typing import Any
from PIL import Image
from build_hfd_vqa_candidate_pilot import conservative_gqa_row
def atomic_json(path: Path, value: Any) -> None:
path.parent.mkdir(parents=True, exist_ok=True)
partial = path.with_name(path.name + ".partial")
partial.write_text(json.dumps(value, ensure_ascii=False, indent=2) + "\n", encoding="utf-8")
partial.replace(path)
def score(row: dict[str, Any]) -> str:
# Stable, source-order-independent tie break. Include the question id so
# repeated questions for one image do not silently depend on parquet order.
return hashlib.sha256((str(row.get("imageId")) + "\0" + str(row.get("id"))).encode()).hexdigest()
def scan_instructions(split: str, limit: int, excluded_images: set[str]) -> tuple[list[dict[str, Any]], dict[str, Any]]:
from datasets import load_dataset
config = f"{split}_balanced_instructions"
ds = load_dataset("lmms-lab-encoder/GQA", config, split=split, streaming=True)
best: dict[str, dict[str, Any]] = {}
rejected = Counter()
scanned = 0
for raw in ds:
scanned += 1
row = dict(raw)
image_id = str(row.get("imageId", ""))
if image_id in excluded_images:
rejected["old_pilot_image"] += 1
continue
accepted, reason = conservative_gqa_row(row)
if not accepted:
rejected[reason] += 1
continue
item = {key: row.get(key) for key in ("id", "imageId", "question", "answer", "fullAnswer", "types", "semanticStr")}
item.update({"split": split, "source_dataset": "GQA", "source_config": config,
"status": "pending_blind_image_review_audio_frontend", "text_gate": "conservative_gqa_row_v1",
"selection_score": score(row)})
old = best.get(image_id)
if old is None or item["selection_score"] < old["selection_score"]:
best[image_id] = item
ordered = sorted(best.values(), key=lambda r: (str(r["selection_score"]), str(r["id"])))
selected = ordered[:limit]
audit = {"split": split, "scanned": scanned, "accepted_unique_images": len(best),
"selected": len(selected), "requested": limit, "rejected": dict(rejected),
"question_ids_unique": len({str(r["id"]) for r in selected}),
"image_ids_unique": len({str(r["imageId"]) for r in selected})}
return selected, audit
def fetch_images(root: Path, rows: list[dict[str, Any]]) -> tuple[list[dict[str, Any]], dict[str, Any]]:
"""Stream balanced image shards, materializing only wanted image ids.
The routine is resumable: existing files are verified and reused. We do
not hold parquet/image payloads in memory after each row.
"""
from datasets import load_dataset
wanted = {(str(row["split"]), str(row["imageId"])): row for row in rows}
found: dict[tuple[str, str], dict[str, Any]] = {}
counters = Counter()
by_split = {split: [key for key in wanted if key[0] == split] for split in ("train", "val")}
for split in ("train", "val"):
if not by_split[split]:
continue
ds = load_dataset("lmms-lab-encoder/GQA", f"{split}_balanced_images", split=split, streaming=True)
for number, raw in enumerate(ds, 1):
image_id = str(raw.get("id", ""))
key = (split, image_id)
if key not in wanted:
continue
row = wanted[key]
path = root / "images" / split / f"{image_id}.jpg"
path.parent.mkdir(parents=True, exist_ok=True)
try:
if path.is_file():
with Image.open(path) as image:
image.load()
width, height = image.size
else:
image = raw.get("image")
if image is None:
counters["missing_image"] += 1
continue
image = image.convert("RGB")
width, height = image.size
if min(width, height) < 224:
counters["image_too_small"] += 1
continue
temporary = path.with_name(path.name + ".partial")
image.save(temporary, format="JPEG", quality=95)
temporary.replace(path)
if min(width, height) < 224:
counters["image_too_small"] += 1
continue
digest = hashlib.sha256(path.read_bytes()).hexdigest()
found[key] = {**row, "image_path": str(path.resolve()), "image_sha256": digest,
"width": int(width), "height": int(height),
"status": "pending_blind_image_review_audio_frontend"}
counters["saved_or_reused"] += 1
except Exception as exc: # leave row pending for a later resume
counters[f"image_error:{type(exc).__name__}"] += 1
if len(found) == len(wanted):
break
if number % 5000 == 0:
print(f"images_scan split={split} rows={number} found={len(found)}/{len(wanted)}", flush=True)
counters[f"{split}_image_rows_scanned"] = number
admitted = [found[key] for key in wanted if key in found]
# A hash collision here would indicate source corruption or a split leak.
train_hashes = {r["image_sha256"] for r in admitted if r["split"] == "train"}
val_hashes = {r["image_sha256"] for r in admitted if r["split"] == "val"}
if train_hashes & val_hashes:
raise RuntimeError("GQA candidate image content overlaps train and val")
audit = {"requested": len(rows), "selected_with_images": len(admitted),
"pending_image_rows": len(rows) - len(admitted), "image_gate": dict(counters),
"cross_split_image_sha256_overlap": 0,
"training_ready": False,
"reason_not_training_ready": "blind_visual_review_and_question_audio_and_causal_frontend_pending"}
return admitted, audit
def main() -> None:
parser = argparse.ArgumentParser(description=__doc__)
parser.add_argument("--output-root", type=Path, required=True)
parser.add_argument("--train", type=int, default=10000)
parser.add_argument("--val", type=int, default=1000)
parser.add_argument("--old-pilot", type=Path, default=None)
args = parser.parse_args()
root = args.output_root.expanduser().resolve()
old_images: set[str] = set()
if args.old_pilot and args.old_pilot.is_file():
old_images = {str(r.get("imageId")) for r in json.loads(args.old_pilot.read_text())}
rows_train, audit_train = scan_instructions("train", args.train, old_images)
rows_val, audit_val = scan_instructions("val", args.val, old_images)
train_images = {str(r["imageId"]) for r in rows_train}
rows_val = [r for r in rows_val if str(r["imageId"]) not in train_images]
if len(rows_val) < args.val:
raise RuntimeError(f"cross-split image disjointness reduced val below request: {len(rows_val)}/{args.val}")
rows = rows_train + rows_val[:args.val]
atomic_json(root / "gqa_text_candidates.json", rows)
atomic_json(root / "gqa_text_selection_audit.json", {"dataset": "lmms-lab-encoder/GQA",
"train": audit_train, "val": {**audit_val, "selected_after_cross_split": len(rows_val[:args.val])},
"old_pilot_images_excluded": len(old_images), "new_candidate_rows": len(rows),
"new_image_ids": len({str(r["imageId"]) for r in rows}), "training_ready": False})
admitted, image_audit = fetch_images(root, rows)
atomic_json(root / "gqa_candidate_manifest.json", admitted)
atomic_json(root / "gqa_image_selection_audit.json", {"text": {"train": audit_train, "val": audit_val}, **image_audit,
"status": "pending_blind_image_review_audio_frontend", "source_plan_modified": False})
print(json.dumps({"text_candidates": len(rows), "with_images": len(admitted),
"train_with_images": sum(r["split"] == "train" for r in admitted),
"val_with_images": sum(r["split"] == "val" for r in admitted),
"audit": image_audit}, ensure_ascii=False, indent=2), flush=True)
if __name__ == "__main__":
main()
|