code-HFD / scripts /data /build_hfd_multimodal_overlay_catalog.py
Tinghong-Ye22's picture
Add files using upload-large-folder tool
9fc9873 verified
Raw History Blame Contribute Delete
10.3 kB
#!/usr/bin/env python3
"""Build a tiny relation catalog for visual overlays on all HFD tasks.
Only paths, text, split and timing metadata are stored. Motion/audio/Qwen
tensors are not copied. Rows with missing or malformed audited timelines are
rejected, and a visual/question identity occurring in multiple splits is
rejected to prevent train/validation leakage.
"""
from __future__ import annotations
import argparse
import hashlib
import json
from collections import Counter, defaultdict
from pathlib import Path
from typing import Any
from psi.data.seed.hfd_multimodal_overlay import (
HFD_INTERACTION_TASKS,
MultimodalOverlayPolicy,
validate_catalog_payload,
)
def _read_json(path: Path) -> dict[str, Any] | None:
try:
value = json.loads(path.read_text(encoding="utf-8"))
except (OSError, UnicodeDecodeError, json.JSONDecodeError):
return None
return value if isinstance(value, dict) else None
def _question(row: dict[str, Any]) -> dict[str, Any]:
value = row.get("question")
return value if isinstance(value, dict) else {}
def _timeline_path(row: dict[str, Any]) -> Path:
return Path(row["frontend_root"]) / "data" / "timeline" / f"{row['id']}.json"
def _inline_timeline(row: dict[str, Any]) -> dict[str, Any] | None:
value = row.get("timeline")
return value if isinstance(value, dict) else None
def _timeline_payload(row: dict[str, Any]) -> dict[str, Any] | None:
"""Load either the normal JSON timeline or the inline VQAv2 timeline.
The VQAv2 encoder intentionally keeps its small timing record in the
source-plan row and stores the large frontend tensor separately. The old
catalog builder only looked for ``data/timeline/*.json`` and therefore
silently dropped every static-image row.
"""
inline = _inline_timeline(row)
return inline if inline is not None else _read_json(_timeline_path(row))
def _timeline_wav(row: dict[str, Any], timeline: dict[str, Any]) -> Path:
inline_voice = timeline.get("question_voice")
if inline_voice:
return Path(str(inline_voice))
return Path(row["frontend_root"]) / "data" / "timeline" / f"{row['id']}.wav"
def _answer_wav(row: dict[str, Any]) -> Path:
question = _question(row)
question_id = str(row.get("gt_question_id") or question.get("id") or "")
return Path(row["frontend_root"]) / "data" / "voices" / f"{question_id}_answer.wav"
def _key(row: dict[str, Any], timeline: dict[str, Any]) -> str:
question = _question(row)
visual_path = question.get("video_path") or question.get("image_path")
domain = row.get("data_domain") or row.get("cohort") or question.get("source_dataset") or ""
values = (
str(domain),
str(row.get("gt_question_id", "")),
str(visual_path or ""),
str(question.get("question", "")),
str(question.get("answer", "")),
str(timeline.get("question_start", "")),
str(timeline.get("question_end", "")),
)
return "\x1f".join(values)
def _asset_id(key: str) -> str:
return "hfd-mm-" + hashlib.sha256(key.encode("utf-8")).hexdigest()[:20]
def build_catalog(plan_path: Path, policy: MultimodalOverlayPolicy) -> dict[str, Any]:
plan = json.loads(plan_path.read_text(encoding="utf-8"))
if not isinstance(plan, list):
raise ValueError("source plan must be a JSON list")
stats: dict[str, Any] = defaultdict(int)
candidates: dict[str, list[tuple[dict[str, Any], dict[str, Any]]]] = defaultdict(list)
for row in plan:
stats["plan_rows"] += 1
question = _question(row)
# Charades-Ego also contains audited negative/ordinary rows that are
# deliberately *not* questions. They belong to a visual-only
# negative pool, not to this question/answer asset catalog.
if question.get("swa_has_question") is False:
stats["non_question_rows"] += 1
continue
timeline_path = _timeline_path(row)
timeline = _timeline_payload(row)
if timeline is None:
stats["missing_or_invalid_timeline"] += 1
continue
required = (
str(question.get("question", "")).strip(),
str(question.get("answer", "")).strip(),
str(question.get("video_path") or question.get("image_path") or "").strip(),
)
if not all(required):
stats["missing_question_fields"] += 1
continue
q_start = float(timeline.get("question_start", -1.0))
q_end = float(timeline.get("question_end", -1.0))
source_seconds = float(timeline.get("source_seconds", -1.0))
if not (0.0 <= q_start < q_end <= source_seconds):
stats["invalid_question_interval"] += 1
continue
timeline_wav = _timeline_wav(row, timeline)
if not timeline_wav.is_file():
stats["missing_timeline_audio"] += 1
continue
video_path = Path(question.get("video_path") or question.get("image_path") or "")
if not video_path.is_file():
stats["missing_video"] += 1
continue
candidates[_key(row, timeline)].append((row, timeline))
stats["dedup_keys"] = len(candidates)
stats["dedup_rows_removed"] = sum(max(len(rows) - 1, 0) for rows in candidates.values())
conflicts = {
key: rows
for key, rows in candidates.items()
if len({str(row.get("split", "")) for row, _ in rows}) > 1
}
stats["cross_split_conflict_keys"] = len(conflicts)
assets: list[dict[str, Any]] = []
for key, rows in sorted(candidates.items()):
if key in conflicts:
continue
row, timeline = rows[0]
question = _question(row)
# Recompute every path for the selected row. These values must not
# leak from the final iteration of the validation loop above.
timeline_path = _timeline_path(row)
timeline_wav = _timeline_wav(row, timeline)
video_path = Path(question.get("video_path") or question.get("image_path") or "")
answer_wav = _answer_wav(row)
assets.append(
{
"asset_id": _asset_id(key),
"split": str(row.get("split", "")),
"question_id": str(row.get("gt_question_id") or question.get("id") or ""),
"question": str(question["question"]),
"answer": str(question["answer"]),
"video_path": str(video_path),
"timeline_path": str(
timeline_path
if timeline_path.is_file()
else Path(row["frontend_root"]) / "data" / "frontend" / f"{row['id']}.pt"
),
"timeline_inline": _inline_timeline(row) is not None,
"question_audio_path": str(timeline_wav),
"answer_audio_path": str(answer_wav) if answer_wav.is_file() else "",
"motion_names": sorted({str(item[0].get("motion_name", "")) for item in rows} - {""}),
"question_start": float(timeline["question_start"]),
"question_end": float(timeline["question_end"]),
"video_offset": float(timeline.get("video_offset", 0.0)),
"source_seconds": float(timeline["source_seconds"]),
"data_domain": str(
row.get("data_domain")
or row.get("cohort")
or question.get("source_dataset")
or "unknown"
),
}
)
stats["assets"] = len(assets)
stats["assets_with_answer_audio"] = sum(bool(row["answer_audio_path"]) for row in assets)
stats["assets_by_split_train"] = sum(row["split"] == "train" for row in assets)
stats["assets_by_split_val"] = sum(row["split"] == "val" for row in assets)
stats["assets_by_split_test"] = sum(row["split"] == "test" for row in assets)
stats["assets_missing_answer_audio"] = sum(not row["answer_audio_path"] for row in assets)
stats["assets_by_domain"] = dict(Counter(row["data_domain"] for row in assets))
payload: dict[str, Any] = {
"schema": "hfd_unified_multimodal_overlay_catalog_v1",
"tasks": list(HFD_INTERACTION_TASKS),
"stop_is_subtask_of": "interruption",
"policy": policy.as_dict(),
"source_plan": str(plan_path.resolve()),
"source_plan_sha256": hashlib.sha256(plan_path.read_bytes()).hexdigest(),
"storage_contract": {
"task_motion_audio": "owned by original HFD task dataset",
"visual_question_assets": "referenced here, never copied",
"runtime_relation": "deterministic source_id + task + split mapping",
"qwen_hidden_or_kv": "never stored in this catalog",
},
"timing_contract": {
"audio_semantic_stride_seconds": 0.4,
"visual_reference_fps": 8.0,
"qa_question_audio": "timeline WAV [question_start, question_end]",
"qa_answer_audio": "separate target, never fed back as input",
"overlay_interval": "inside active motion with lead/tail guard",
},
"stats": dict(stats),
"assets": assets,
}
validate_catalog_payload(payload)
return payload
def main() -> None:
parser = argparse.ArgumentParser(description=__doc__)
parser.add_argument("--source-plan", type=Path, required=True)
parser.add_argument("--output", type=Path, required=True)
parser.add_argument("--qa-ratio", type=float, default=0.75)
parser.add_argument("--visual-only-ratio", type=float, default=0.10)
parser.add_argument("--audio-only-ratio", type=float, default=0.15)
args = parser.parse_args()
policy = MultimodalOverlayPolicy(
qa_ratio=args.qa_ratio,
visual_only_ratio=args.visual_only_ratio,
audio_only_ratio=args.audio_only_ratio,
)
payload = build_catalog(args.source_plan.expanduser().resolve(), policy)
output = args.output.expanduser().resolve()
output.parent.mkdir(parents=True, exist_ok=True)
output.write_text(json.dumps(payload, ensure_ascii=False, indent=2) + "\n", encoding="utf-8")
print(json.dumps(payload["stats"], ensure_ascii=False, indent=2), flush=True)
print(f"[ready] {output}", flush=True)
if __name__ == "__main__":
main()