Download scripts/data/build_hfd_gqa_scale.py from Tinghong-Ye22/code-HFD: direct link, hf CLI and curl.
- Browser
- Download file 8.95 kB
-
https://huggingface.co/Tinghong-Ye22/code-HFD/resolve/main/scripts/data/build_hfd_gqa_scale.py
- Command line
-
hf download hf://Tinghong-Ye22/code-HFD/scripts/data/build_hfd_gqa_scale.py
-
curl -L -o build_hfd_gqa_scale.py https://huggingface.co/Tinghong-Ye22/code-HFD/resolve/main/scripts/data/build_hfd_gqa_scale.py
8.95 kB
| #!/usr/bin/env python3 | |
| """Build a resumable, image-backed GQA expansion candidate set. | |
| This is deliberately independent from the old 12-row pilot and from the | |
| formal HFD source plan. It scans the complete balanced instruction splits, | |
| keeps one deterministic conservative visible-state question per image, then | |
| streams only the selected image records into an external output directory. | |
| The resulting manifest is *pending* visual/audio/frontend admission; it is | |
| not a training source plan. | |
| """ | |
| from __future__ import annotations | |
| import argparse | |
| from collections import Counter | |
| import hashlib | |
| import json | |
| from pathlib import Path | |
| from typing import Any | |
| from PIL import Image | |
| from build_hfd_vqa_candidate_pilot import conservative_gqa_row | |
| def atomic_json(path: Path, value: Any) -> None: | |
| path.parent.mkdir(parents=True, exist_ok=True) | |
| partial = path.with_name(path.name + ".partial") | |
| partial.write_text(json.dumps(value, ensure_ascii=False, indent=2) + "\n", encoding="utf-8") | |
| partial.replace(path) | |
| def score(row: dict[str, Any]) -> str: | |
| # Stable, source-order-independent tie break. Include the question id so | |
| # repeated questions for one image do not silently depend on parquet order. | |
| return hashlib.sha256((str(row.get("imageId")) + "\0" + str(row.get("id"))).encode()).hexdigest() | |
| def scan_instructions(split: str, limit: int, excluded_images: set[str]) -> tuple[list[dict[str, Any]], dict[str, Any]]: | |
| from datasets import load_dataset | |
| config = f"{split}_balanced_instructions" | |
| ds = load_dataset("lmms-lab-encoder/GQA", config, split=split, streaming=True) | |
| best: dict[str, dict[str, Any]] = {} | |
| rejected = Counter() | |
| scanned = 0 | |
| for raw in ds: | |
| scanned += 1 | |
| row = dict(raw) | |
| image_id = str(row.get("imageId", "")) | |
| if image_id in excluded_images: | |
| rejected["old_pilot_image"] += 1 | |
| continue | |
| accepted, reason = conservative_gqa_row(row) | |
| if not accepted: | |
| rejected[reason] += 1 | |
| continue | |
| item = {key: row.get(key) for key in ("id", "imageId", "question", "answer", "fullAnswer", "types", "semanticStr")} | |
| item.update({"split": split, "source_dataset": "GQA", "source_config": config, | |
| "status": "pending_blind_image_review_audio_frontend", "text_gate": "conservative_gqa_row_v1", | |
| "selection_score": score(row)}) | |
| old = best.get(image_id) | |
| if old is None or item["selection_score"] < old["selection_score"]: | |
| best[image_id] = item | |
| ordered = sorted(best.values(), key=lambda r: (str(r["selection_score"]), str(r["id"]))) | |
| selected = ordered[:limit] | |
| audit = {"split": split, "scanned": scanned, "accepted_unique_images": len(best), | |
| "selected": len(selected), "requested": limit, "rejected": dict(rejected), | |
| "question_ids_unique": len({str(r["id"]) for r in selected}), | |
| "image_ids_unique": len({str(r["imageId"]) for r in selected})} | |
| return selected, audit | |
| def fetch_images(root: Path, rows: list[dict[str, Any]]) -> tuple[list[dict[str, Any]], dict[str, Any]]: | |
| """Stream balanced image shards, materializing only wanted image ids. | |
| The routine is resumable: existing files are verified and reused. We do | |
| not hold parquet/image payloads in memory after each row. | |
| """ | |
| from datasets import load_dataset | |
| wanted = {(str(row["split"]), str(row["imageId"])): row for row in rows} | |
| found: dict[tuple[str, str], dict[str, Any]] = {} | |
| counters = Counter() | |
| by_split = {split: [key for key in wanted if key[0] == split] for split in ("train", "val")} | |
| for split in ("train", "val"): | |
| if not by_split[split]: | |
| continue | |
| ds = load_dataset("lmms-lab-encoder/GQA", f"{split}_balanced_images", split=split, streaming=True) | |
| for number, raw in enumerate(ds, 1): | |
| image_id = str(raw.get("id", "")) | |
| key = (split, image_id) | |
| if key not in wanted: | |
| continue | |
| row = wanted[key] | |
| path = root / "images" / split / f"{image_id}.jpg" | |
| path.parent.mkdir(parents=True, exist_ok=True) | |
| try: | |
| if path.is_file(): | |
| with Image.open(path) as image: | |
| image.load() | |
| width, height = image.size | |
| else: | |
| image = raw.get("image") | |
| if image is None: | |
| counters["missing_image"] += 1 | |
| continue | |
| image = image.convert("RGB") | |
| width, height = image.size | |
| if min(width, height) < 224: | |
| counters["image_too_small"] += 1 | |
| continue | |
| temporary = path.with_name(path.name + ".partial") | |
| image.save(temporary, format="JPEG", quality=95) | |
| temporary.replace(path) | |
| if min(width, height) < 224: | |
| counters["image_too_small"] += 1 | |
| continue | |
| digest = hashlib.sha256(path.read_bytes()).hexdigest() | |
| found[key] = {**row, "image_path": str(path.resolve()), "image_sha256": digest, | |
| "width": int(width), "height": int(height), | |
| "status": "pending_blind_image_review_audio_frontend"} | |
| counters["saved_or_reused"] += 1 | |
| except Exception as exc: # leave row pending for a later resume | |
| counters[f"image_error:{type(exc).__name__}"] += 1 | |
| if len(found) == len(wanted): | |
| break | |
| if number % 5000 == 0: | |
| print(f"images_scan split={split} rows={number} found={len(found)}/{len(wanted)}", flush=True) | |
| counters[f"{split}_image_rows_scanned"] = number | |
| admitted = [found[key] for key in wanted if key in found] | |
| # A hash collision here would indicate source corruption or a split leak. | |
| train_hashes = {r["image_sha256"] for r in admitted if r["split"] == "train"} | |
| val_hashes = {r["image_sha256"] for r in admitted if r["split"] == "val"} | |
| if train_hashes & val_hashes: | |
| raise RuntimeError("GQA candidate image content overlaps train and val") | |
| audit = {"requested": len(rows), "selected_with_images": len(admitted), | |
| "pending_image_rows": len(rows) - len(admitted), "image_gate": dict(counters), | |
| "cross_split_image_sha256_overlap": 0, | |
| "training_ready": False, | |
| "reason_not_training_ready": "blind_visual_review_and_question_audio_and_causal_frontend_pending"} | |
| return admitted, audit | |
| def main() -> None: | |
| parser = argparse.ArgumentParser(description=__doc__) | |
| parser.add_argument("--output-root", type=Path, required=True) | |
| parser.add_argument("--train", type=int, default=10000) | |
| parser.add_argument("--val", type=int, default=1000) | |
| parser.add_argument("--old-pilot", type=Path, default=None) | |
| args = parser.parse_args() | |
| root = args.output_root.expanduser().resolve() | |
| old_images: set[str] = set() | |
| if args.old_pilot and args.old_pilot.is_file(): | |
| old_images = {str(r.get("imageId")) for r in json.loads(args.old_pilot.read_text())} | |
| rows_train, audit_train = scan_instructions("train", args.train, old_images) | |
| rows_val, audit_val = scan_instructions("val", args.val, old_images) | |
| train_images = {str(r["imageId"]) for r in rows_train} | |
| rows_val = [r for r in rows_val if str(r["imageId"]) not in train_images] | |
| if len(rows_val) < args.val: | |
| raise RuntimeError(f"cross-split image disjointness reduced val below request: {len(rows_val)}/{args.val}") | |
| rows = rows_train + rows_val[:args.val] | |
| atomic_json(root / "gqa_text_candidates.json", rows) | |
| atomic_json(root / "gqa_text_selection_audit.json", {"dataset": "lmms-lab-encoder/GQA", | |
| "train": audit_train, "val": {**audit_val, "selected_after_cross_split": len(rows_val[:args.val])}, | |
| "old_pilot_images_excluded": len(old_images), "new_candidate_rows": len(rows), | |
| "new_image_ids": len({str(r["imageId"]) for r in rows}), "training_ready": False}) | |
| admitted, image_audit = fetch_images(root, rows) | |
| atomic_json(root / "gqa_candidate_manifest.json", admitted) | |
| atomic_json(root / "gqa_image_selection_audit.json", {"text": {"train": audit_train, "val": audit_val}, **image_audit, | |
| "status": "pending_blind_image_review_audio_frontend", "source_plan_modified": False}) | |
| print(json.dumps({"text_candidates": len(rows), "with_images": len(admitted), | |
| "train_with_images": sum(r["split"] == "train" for r in admitted), | |
| "val_with_images": sum(r["split"] == "val" for r in admitted), | |
| "audit": image_audit}, ensure_ascii=False, indent=2), flush=True) | |
| if __name__ == "__main__": | |
| main() | |