#!/usr/bin/env python3 """Build a tiny relation catalog for visual overlays on all HFD tasks. Only paths, text, split and timing metadata are stored. Motion/audio/Qwen tensors are not copied. Rows with missing or malformed audited timelines are rejected, and a visual/question identity occurring in multiple splits is rejected to prevent train/validation leakage. """ from __future__ import annotations import argparse import hashlib import json from collections import Counter, defaultdict from pathlib import Path from typing import Any from psi.data.seed.hfd_multimodal_overlay import ( HFD_INTERACTION_TASKS, MultimodalOverlayPolicy, validate_catalog_payload, ) def _read_json(path: Path) -> dict[str, Any] | None: try: value = json.loads(path.read_text(encoding="utf-8")) except (OSError, UnicodeDecodeError, json.JSONDecodeError): return None return value if isinstance(value, dict) else None def _question(row: dict[str, Any]) -> dict[str, Any]: value = row.get("question") return value if isinstance(value, dict) else {} def _timeline_path(row: dict[str, Any]) -> Path: return Path(row["frontend_root"]) / "data" / "timeline" / f"{row['id']}.json" def _inline_timeline(row: dict[str, Any]) -> dict[str, Any] | None: value = row.get("timeline") return value if isinstance(value, dict) else None def _timeline_payload(row: dict[str, Any]) -> dict[str, Any] | None: """Load either the normal JSON timeline or the inline VQAv2 timeline. The VQAv2 encoder intentionally keeps its small timing record in the source-plan row and stores the large frontend tensor separately. The old catalog builder only looked for ``data/timeline/*.json`` and therefore silently dropped every static-image row. """ inline = _inline_timeline(row) return inline if inline is not None else _read_json(_timeline_path(row)) def _timeline_wav(row: dict[str, Any], timeline: dict[str, Any]) -> Path: inline_voice = timeline.get("question_voice") if inline_voice: return Path(str(inline_voice)) return Path(row["frontend_root"]) / "data" / "timeline" / f"{row['id']}.wav" def _answer_wav(row: dict[str, Any]) -> Path: question = _question(row) question_id = str(row.get("gt_question_id") or question.get("id") or "") return Path(row["frontend_root"]) / "data" / "voices" / f"{question_id}_answer.wav" def _key(row: dict[str, Any], timeline: dict[str, Any]) -> str: question = _question(row) visual_path = question.get("video_path") or question.get("image_path") domain = row.get("data_domain") or row.get("cohort") or question.get("source_dataset") or "" values = ( str(domain), str(row.get("gt_question_id", "")), str(visual_path or ""), str(question.get("question", "")), str(question.get("answer", "")), str(timeline.get("question_start", "")), str(timeline.get("question_end", "")), ) return "\x1f".join(values) def _asset_id(key: str) -> str: return "hfd-mm-" + hashlib.sha256(key.encode("utf-8")).hexdigest()[:20] def build_catalog(plan_path: Path, policy: MultimodalOverlayPolicy) -> dict[str, Any]: plan = json.loads(plan_path.read_text(encoding="utf-8")) if not isinstance(plan, list): raise ValueError("source plan must be a JSON list") stats: dict[str, Any] = defaultdict(int) candidates: dict[str, list[tuple[dict[str, Any], dict[str, Any]]]] = defaultdict(list) for row in plan: stats["plan_rows"] += 1 question = _question(row) # Charades-Ego also contains audited negative/ordinary rows that are # deliberately *not* questions. They belong to a visual-only # negative pool, not to this question/answer asset catalog. if question.get("swa_has_question") is False: stats["non_question_rows"] += 1 continue timeline_path = _timeline_path(row) timeline = _timeline_payload(row) if timeline is None: stats["missing_or_invalid_timeline"] += 1 continue required = ( str(question.get("question", "")).strip(), str(question.get("answer", "")).strip(), str(question.get("video_path") or question.get("image_path") or "").strip(), ) if not all(required): stats["missing_question_fields"] += 1 continue q_start = float(timeline.get("question_start", -1.0)) q_end = float(timeline.get("question_end", -1.0)) source_seconds = float(timeline.get("source_seconds", -1.0)) if not (0.0 <= q_start < q_end <= source_seconds): stats["invalid_question_interval"] += 1 continue timeline_wav = _timeline_wav(row, timeline) if not timeline_wav.is_file(): stats["missing_timeline_audio"] += 1 continue video_path = Path(question.get("video_path") or question.get("image_path") or "") if not video_path.is_file(): stats["missing_video"] += 1 continue candidates[_key(row, timeline)].append((row, timeline)) stats["dedup_keys"] = len(candidates) stats["dedup_rows_removed"] = sum(max(len(rows) - 1, 0) for rows in candidates.values()) conflicts = { key: rows for key, rows in candidates.items() if len({str(row.get("split", "")) for row, _ in rows}) > 1 } stats["cross_split_conflict_keys"] = len(conflicts) assets: list[dict[str, Any]] = [] for key, rows in sorted(candidates.items()): if key in conflicts: continue row, timeline = rows[0] question = _question(row) # Recompute every path for the selected row. These values must not # leak from the final iteration of the validation loop above. timeline_path = _timeline_path(row) timeline_wav = _timeline_wav(row, timeline) video_path = Path(question.get("video_path") or question.get("image_path") or "") answer_wav = _answer_wav(row) assets.append( { "asset_id": _asset_id(key), "split": str(row.get("split", "")), "question_id": str(row.get("gt_question_id") or question.get("id") or ""), "question": str(question["question"]), "answer": str(question["answer"]), "video_path": str(video_path), "timeline_path": str( timeline_path if timeline_path.is_file() else Path(row["frontend_root"]) / "data" / "frontend" / f"{row['id']}.pt" ), "timeline_inline": _inline_timeline(row) is not None, "question_audio_path": str(timeline_wav), "answer_audio_path": str(answer_wav) if answer_wav.is_file() else "", "motion_names": sorted({str(item[0].get("motion_name", "")) for item in rows} - {""}), "question_start": float(timeline["question_start"]), "question_end": float(timeline["question_end"]), "video_offset": float(timeline.get("video_offset", 0.0)), "source_seconds": float(timeline["source_seconds"]), "data_domain": str( row.get("data_domain") or row.get("cohort") or question.get("source_dataset") or "unknown" ), } ) stats["assets"] = len(assets) stats["assets_with_answer_audio"] = sum(bool(row["answer_audio_path"]) for row in assets) stats["assets_by_split_train"] = sum(row["split"] == "train" for row in assets) stats["assets_by_split_val"] = sum(row["split"] == "val" for row in assets) stats["assets_by_split_test"] = sum(row["split"] == "test" for row in assets) stats["assets_missing_answer_audio"] = sum(not row["answer_audio_path"] for row in assets) stats["assets_by_domain"] = dict(Counter(row["data_domain"] for row in assets)) payload: dict[str, Any] = { "schema": "hfd_unified_multimodal_overlay_catalog_v1", "tasks": list(HFD_INTERACTION_TASKS), "stop_is_subtask_of": "interruption", "policy": policy.as_dict(), "source_plan": str(plan_path.resolve()), "source_plan_sha256": hashlib.sha256(plan_path.read_bytes()).hexdigest(), "storage_contract": { "task_motion_audio": "owned by original HFD task dataset", "visual_question_assets": "referenced here, never copied", "runtime_relation": "deterministic source_id + task + split mapping", "qwen_hidden_or_kv": "never stored in this catalog", }, "timing_contract": { "audio_semantic_stride_seconds": 0.4, "visual_reference_fps": 8.0, "qa_question_audio": "timeline WAV [question_start, question_end]", "qa_answer_audio": "separate target, never fed back as input", "overlay_interval": "inside active motion with lead/tail guard", }, "stats": dict(stats), "assets": assets, } validate_catalog_payload(payload) return payload def main() -> None: parser = argparse.ArgumentParser(description=__doc__) parser.add_argument("--source-plan", type=Path, required=True) parser.add_argument("--output", type=Path, required=True) parser.add_argument("--qa-ratio", type=float, default=0.75) parser.add_argument("--visual-only-ratio", type=float, default=0.10) parser.add_argument("--audio-only-ratio", type=float, default=0.15) args = parser.parse_args() policy = MultimodalOverlayPolicy( qa_ratio=args.qa_ratio, visual_only_ratio=args.visual_only_ratio, audio_only_ratio=args.audio_only_ratio, ) payload = build_catalog(args.source_plan.expanduser().resolve(), policy) output = args.output.expanduser().resolve() output.parent.mkdir(parents=True, exist_ok=True) output.write_text(json.dumps(payload, ensure_ascii=False, indent=2) + "\n", encoding="utf-8") print(json.dumps(payload["stats"], ensure_ascii=False, indent=2), flush=True) print(f"[ready] {output}", flush=True) if __name__ == "__main__": main()