Download scripts/data/build_hfd_multimodal_overlay_catalog.py from Tinghong-Ye22/code-HFD: direct link, hf CLI and curl.
- Browser
- Download file 10.3 kB
-
https://huggingface.co/Tinghong-Ye22/code-HFD/resolve/main/scripts/data/build_hfd_multimodal_overlay_catalog.py
- Command line
-
hf download hf://Tinghong-Ye22/code-HFD/scripts/data/build_hfd_multimodal_overlay_catalog.py
-
curl -L -o build_hfd_multimodal_overlay_catalog.py https://huggingface.co/Tinghong-Ye22/code-HFD/resolve/main/scripts/data/build_hfd_multimodal_overlay_catalog.py
10.3 kB
| #!/usr/bin/env python3 | |
| """Build a tiny relation catalog for visual overlays on all HFD tasks. | |
| Only paths, text, split and timing metadata are stored. Motion/audio/Qwen | |
| tensors are not copied. Rows with missing or malformed audited timelines are | |
| rejected, and a visual/question identity occurring in multiple splits is | |
| rejected to prevent train/validation leakage. | |
| """ | |
| from __future__ import annotations | |
| import argparse | |
| import hashlib | |
| import json | |
| from collections import Counter, defaultdict | |
| from pathlib import Path | |
| from typing import Any | |
| from psi.data.seed.hfd_multimodal_overlay import ( | |
| HFD_INTERACTION_TASKS, | |
| MultimodalOverlayPolicy, | |
| validate_catalog_payload, | |
| ) | |
| def _read_json(path: Path) -> dict[str, Any] | None: | |
| try: | |
| value = json.loads(path.read_text(encoding="utf-8")) | |
| except (OSError, UnicodeDecodeError, json.JSONDecodeError): | |
| return None | |
| return value if isinstance(value, dict) else None | |
| def _question(row: dict[str, Any]) -> dict[str, Any]: | |
| value = row.get("question") | |
| return value if isinstance(value, dict) else {} | |
| def _timeline_path(row: dict[str, Any]) -> Path: | |
| return Path(row["frontend_root"]) / "data" / "timeline" / f"{row['id']}.json" | |
| def _inline_timeline(row: dict[str, Any]) -> dict[str, Any] | None: | |
| value = row.get("timeline") | |
| return value if isinstance(value, dict) else None | |
| def _timeline_payload(row: dict[str, Any]) -> dict[str, Any] | None: | |
| """Load either the normal JSON timeline or the inline VQAv2 timeline. | |
| The VQAv2 encoder intentionally keeps its small timing record in the | |
| source-plan row and stores the large frontend tensor separately. The old | |
| catalog builder only looked for ``data/timeline/*.json`` and therefore | |
| silently dropped every static-image row. | |
| """ | |
| inline = _inline_timeline(row) | |
| return inline if inline is not None else _read_json(_timeline_path(row)) | |
| def _timeline_wav(row: dict[str, Any], timeline: dict[str, Any]) -> Path: | |
| inline_voice = timeline.get("question_voice") | |
| if inline_voice: | |
| return Path(str(inline_voice)) | |
| return Path(row["frontend_root"]) / "data" / "timeline" / f"{row['id']}.wav" | |
| def _answer_wav(row: dict[str, Any]) -> Path: | |
| question = _question(row) | |
| question_id = str(row.get("gt_question_id") or question.get("id") or "") | |
| return Path(row["frontend_root"]) / "data" / "voices" / f"{question_id}_answer.wav" | |
| def _key(row: dict[str, Any], timeline: dict[str, Any]) -> str: | |
| question = _question(row) | |
| visual_path = question.get("video_path") or question.get("image_path") | |
| domain = row.get("data_domain") or row.get("cohort") or question.get("source_dataset") or "" | |
| values = ( | |
| str(domain), | |
| str(row.get("gt_question_id", "")), | |
| str(visual_path or ""), | |
| str(question.get("question", "")), | |
| str(question.get("answer", "")), | |
| str(timeline.get("question_start", "")), | |
| str(timeline.get("question_end", "")), | |
| ) | |
| return "\x1f".join(values) | |
| def _asset_id(key: str) -> str: | |
| return "hfd-mm-" + hashlib.sha256(key.encode("utf-8")).hexdigest()[:20] | |
| def build_catalog(plan_path: Path, policy: MultimodalOverlayPolicy) -> dict[str, Any]: | |
| plan = json.loads(plan_path.read_text(encoding="utf-8")) | |
| if not isinstance(plan, list): | |
| raise ValueError("source plan must be a JSON list") | |
| stats: dict[str, Any] = defaultdict(int) | |
| candidates: dict[str, list[tuple[dict[str, Any], dict[str, Any]]]] = defaultdict(list) | |
| for row in plan: | |
| stats["plan_rows"] += 1 | |
| question = _question(row) | |
| # Charades-Ego also contains audited negative/ordinary rows that are | |
| # deliberately *not* questions. They belong to a visual-only | |
| # negative pool, not to this question/answer asset catalog. | |
| if question.get("swa_has_question") is False: | |
| stats["non_question_rows"] += 1 | |
| continue | |
| timeline_path = _timeline_path(row) | |
| timeline = _timeline_payload(row) | |
| if timeline is None: | |
| stats["missing_or_invalid_timeline"] += 1 | |
| continue | |
| required = ( | |
| str(question.get("question", "")).strip(), | |
| str(question.get("answer", "")).strip(), | |
| str(question.get("video_path") or question.get("image_path") or "").strip(), | |
| ) | |
| if not all(required): | |
| stats["missing_question_fields"] += 1 | |
| continue | |
| q_start = float(timeline.get("question_start", -1.0)) | |
| q_end = float(timeline.get("question_end", -1.0)) | |
| source_seconds = float(timeline.get("source_seconds", -1.0)) | |
| if not (0.0 <= q_start < q_end <= source_seconds): | |
| stats["invalid_question_interval"] += 1 | |
| continue | |
| timeline_wav = _timeline_wav(row, timeline) | |
| if not timeline_wav.is_file(): | |
| stats["missing_timeline_audio"] += 1 | |
| continue | |
| video_path = Path(question.get("video_path") or question.get("image_path") or "") | |
| if not video_path.is_file(): | |
| stats["missing_video"] += 1 | |
| continue | |
| candidates[_key(row, timeline)].append((row, timeline)) | |
| stats["dedup_keys"] = len(candidates) | |
| stats["dedup_rows_removed"] = sum(max(len(rows) - 1, 0) for rows in candidates.values()) | |
| conflicts = { | |
| key: rows | |
| for key, rows in candidates.items() | |
| if len({str(row.get("split", "")) for row, _ in rows}) > 1 | |
| } | |
| stats["cross_split_conflict_keys"] = len(conflicts) | |
| assets: list[dict[str, Any]] = [] | |
| for key, rows in sorted(candidates.items()): | |
| if key in conflicts: | |
| continue | |
| row, timeline = rows[0] | |
| question = _question(row) | |
| # Recompute every path for the selected row. These values must not | |
| # leak from the final iteration of the validation loop above. | |
| timeline_path = _timeline_path(row) | |
| timeline_wav = _timeline_wav(row, timeline) | |
| video_path = Path(question.get("video_path") or question.get("image_path") or "") | |
| answer_wav = _answer_wav(row) | |
| assets.append( | |
| { | |
| "asset_id": _asset_id(key), | |
| "split": str(row.get("split", "")), | |
| "question_id": str(row.get("gt_question_id") or question.get("id") or ""), | |
| "question": str(question["question"]), | |
| "answer": str(question["answer"]), | |
| "video_path": str(video_path), | |
| "timeline_path": str( | |
| timeline_path | |
| if timeline_path.is_file() | |
| else Path(row["frontend_root"]) / "data" / "frontend" / f"{row['id']}.pt" | |
| ), | |
| "timeline_inline": _inline_timeline(row) is not None, | |
| "question_audio_path": str(timeline_wav), | |
| "answer_audio_path": str(answer_wav) if answer_wav.is_file() else "", | |
| "motion_names": sorted({str(item[0].get("motion_name", "")) for item in rows} - {""}), | |
| "question_start": float(timeline["question_start"]), | |
| "question_end": float(timeline["question_end"]), | |
| "video_offset": float(timeline.get("video_offset", 0.0)), | |
| "source_seconds": float(timeline["source_seconds"]), | |
| "data_domain": str( | |
| row.get("data_domain") | |
| or row.get("cohort") | |
| or question.get("source_dataset") | |
| or "unknown" | |
| ), | |
| } | |
| ) | |
| stats["assets"] = len(assets) | |
| stats["assets_with_answer_audio"] = sum(bool(row["answer_audio_path"]) for row in assets) | |
| stats["assets_by_split_train"] = sum(row["split"] == "train" for row in assets) | |
| stats["assets_by_split_val"] = sum(row["split"] == "val" for row in assets) | |
| stats["assets_by_split_test"] = sum(row["split"] == "test" for row in assets) | |
| stats["assets_missing_answer_audio"] = sum(not row["answer_audio_path"] for row in assets) | |
| stats["assets_by_domain"] = dict(Counter(row["data_domain"] for row in assets)) | |
| payload: dict[str, Any] = { | |
| "schema": "hfd_unified_multimodal_overlay_catalog_v1", | |
| "tasks": list(HFD_INTERACTION_TASKS), | |
| "stop_is_subtask_of": "interruption", | |
| "policy": policy.as_dict(), | |
| "source_plan": str(plan_path.resolve()), | |
| "source_plan_sha256": hashlib.sha256(plan_path.read_bytes()).hexdigest(), | |
| "storage_contract": { | |
| "task_motion_audio": "owned by original HFD task dataset", | |
| "visual_question_assets": "referenced here, never copied", | |
| "runtime_relation": "deterministic source_id + task + split mapping", | |
| "qwen_hidden_or_kv": "never stored in this catalog", | |
| }, | |
| "timing_contract": { | |
| "audio_semantic_stride_seconds": 0.4, | |
| "visual_reference_fps": 8.0, | |
| "qa_question_audio": "timeline WAV [question_start, question_end]", | |
| "qa_answer_audio": "separate target, never fed back as input", | |
| "overlay_interval": "inside active motion with lead/tail guard", | |
| }, | |
| "stats": dict(stats), | |
| "assets": assets, | |
| } | |
| validate_catalog_payload(payload) | |
| return payload | |
| def main() -> None: | |
| parser = argparse.ArgumentParser(description=__doc__) | |
| parser.add_argument("--source-plan", type=Path, required=True) | |
| parser.add_argument("--output", type=Path, required=True) | |
| parser.add_argument("--qa-ratio", type=float, default=0.75) | |
| parser.add_argument("--visual-only-ratio", type=float, default=0.10) | |
| parser.add_argument("--audio-only-ratio", type=float, default=0.15) | |
| args = parser.parse_args() | |
| policy = MultimodalOverlayPolicy( | |
| qa_ratio=args.qa_ratio, | |
| visual_only_ratio=args.visual_only_ratio, | |
| audio_only_ratio=args.audio_only_ratio, | |
| ) | |
| payload = build_catalog(args.source_plan.expanduser().resolve(), policy) | |
| output = args.output.expanduser().resolve() | |
| output.parent.mkdir(parents=True, exist_ok=True) | |
| output.write_text(json.dumps(payload, ensure_ascii=False, indent=2) + "\n", encoding="utf-8") | |
| print(json.dumps(payload["stats"], ensure_ascii=False, indent=2), flush=True) | |
| print(f"[ready] {output}", flush=True) | |
| if __name__ == "__main__": | |
| main() | |