code-Psi0-yth-v24 / scripts /data /prebuild_multimodal_records_cache.py
Tinghong-Ye22's picture
Add files using upload-large-folder tool
f1fda4a verified
Raw History Blame Contribute Delete
19.6 kB
from __future__ import annotations
import argparse
import json
import sys
from pathlib import Path
from typing import Any
REPO_ROOT = Path(__file__).resolve().parents[2]
SRC_ROOT = REPO_ROOT / "src"
if str(SRC_ROOT) not in sys.path:
sys.path.insert(0, str(SRC_ROOT))
from psi.data.seed.seed_audio_motion_dataset import SeedAudioMotionDataset
from psi.data.seed.seed_motion_dataset import SeedMotionDataset
from psi.data.seed.seed_video_motion_dataset import SeedVideoClipMotionDataset, SeedVideoMotionDataset
def _as_bool(value: Any) -> bool:
if isinstance(value, bool):
return value
if isinstance(value, (int, float)):
return bool(value)
if isinstance(value, str):
return value.strip().lower() in {"1", "true", "yes", "y", "on"}
return bool(value)
def _csv_items(value: str) -> list[str]:
return [item.strip() for item in str(value).split(",") if item.strip()]
def _load_sources(raw: str, *, arg_name: str) -> list[dict[str, Any]]:
raw = str(raw or "").strip()
if not raw:
return []
try:
data = json.loads(raw)
except json.JSONDecodeError as exc:
raise ValueError(f"Bad JSON in {arg_name}") from exc
if not isinstance(data, list):
raise ValueError(f"{arg_name} must be a JSON list")
return [dict(item) for item in data]
def _print_ready(prefix: str, **items: Any) -> None:
fields = [f"{key}={value}" for key, value in items.items() if value not in (None, "")]
print(f"[{prefix}] " + " ".join(fields), flush=True)
def parse_args() -> argparse.Namespace:
parser = argparse.ArgumentParser(
description="Prebuild persistent multimodal records caches outside training startup."
)
parser.add_argument("--tasks", default="text,audio,audio_command,video_clip")
parser.add_argument("--splits", default="train,val")
parser.add_argument("--action-chunk-size", type=int, default=30)
parser.add_argument("--num-past-frames", type=int, default=0)
parser.add_argument(
"--use-delta-actions",
action=argparse.BooleanOptionalAction,
default=False,
)
parser.add_argument("--text-root-dir", type=str, default="")
parser.add_argument("--text-metadata-csv", type=str, default="metadata/seed_metadata_v004.csv")
parser.add_argument("--text-records-cache-dir", type=str, default="")
parser.add_argument("--text-val-ratio", type=float, default=0.1)
parser.add_argument("--text-split-seed", type=int, default=42)
parser.add_argument("--text-upsample-rate", type=int, default=1)
parser.add_argument("--text-state-history-stride", type=int, default=30)
parser.add_argument("--audio-sources-json", type=str, default="")
parser.add_argument("--audio-records-cache-dir", type=str, default="")
parser.add_argument("--audio-val-ratio", type=float, default=0.1)
parser.add_argument("--audio-split-seed", type=int, default=42)
parser.add_argument("--audio-sample-stride", type=int, default=1)
parser.add_argument("--audio-motion-fps", type=int, default=50)
parser.add_argument("--audio-audio-sampling-rate", type=int, default=16000)
parser.add_argument("--audio-audio-window-seconds", type=float, default=1.0)
parser.add_argument("--audio-audio-left-context-seconds", type=float, default=0.0)
parser.add_argument(
"--audio-full-audio",
action=argparse.BooleanOptionalAction,
default=True,
)
parser.add_argument(
"--audio-align-start-hold",
action=argparse.BooleanOptionalAction,
default=True,
)
parser.add_argument("--audio-command-sources-json", type=str, default="")
parser.add_argument("--audio-command-records-cache-dir", type=str, default="")
parser.add_argument("--audio-command-val-ratio", type=float, default=0.1)
parser.add_argument("--audio-command-split-seed", type=int, default=42)
parser.add_argument("--audio-command-sample-stride", type=int, default=1)
parser.add_argument("--audio-command-motion-fps", type=int, default=50)
parser.add_argument("--audio-command-audio-sampling-rate", type=int, default=16000)
parser.add_argument("--audio-command-audio-window-seconds", type=float, default=1.0)
parser.add_argument("--audio-command-audio-left-context-seconds", type=float, default=0.0)
parser.add_argument(
"--audio-command-full-audio",
action=argparse.BooleanOptionalAction,
default=True,
)
parser.add_argument(
"--audio-command-align-start-hold",
action=argparse.BooleanOptionalAction,
default=True,
)
parser.add_argument("--video-sources-json", type=str, default="")
parser.add_argument("--video-records-cache-dir", type=str, default="")
parser.add_argument("--video-val-ratio", type=float, default=0.1)
parser.add_argument("--video-split-seed", type=int, default=42)
parser.add_argument("--video-sample-stride", type=int, default=1)
parser.add_argument("--video-motion-fps", type=int, default=50)
parser.add_argument("--video-video-sample-fps", type=float, default=1.0)
parser.add_argument("--video-video-size-pixels", type=int, default=50176)
parser.add_argument("--video-video-decoder", type=str, default="ffmpeg")
parser.add_argument("--video-record-build-workers", type=int, default=0)
parser.add_argument("--video-record-build-chunk-size", type=int, default=4096)
parser.add_argument("--video-clip-sources-json", type=str, default="")
parser.add_argument("--video-clip-records-cache-dir", type=str, default="")
parser.add_argument("--video-clip-val-ratio", type=float, default=0.1)
parser.add_argument("--video-clip-split-seed", type=int, default=42)
parser.add_argument("--video-clip-sample-stride", type=int, default=1)
parser.add_argument("--video-clip-motion-fps", type=int, default=50)
parser.add_argument("--video-clip-video-sample-fps", type=float, default=8.0)
parser.add_argument("--video-clip-video-size-pixels", type=int, default=50176)
parser.add_argument("--video-clip-video-decoder", type=str, default="ffmpeg")
parser.add_argument("--video-clip-record-build-workers", type=int, default=0)
parser.add_argument("--video-clip-record-build-chunk-size", type=int, default=4096)
parser.add_argument("--video-clip-action-exec-horizon", type=int, default=0)
parser.add_argument("--video-clip-clip-stride-frames", type=int, default=0)
parser.add_argument(
"--video-clip-trust-metadata",
action=argparse.BooleanOptionalAction,
default=False,
)
return parser.parse_args()
def prebuild_text(args: argparse.Namespace, split: str) -> None:
if not str(args.text_root_dir).strip():
_print_ready("SKIP", task="text", split=split, reason="empty_root_dir")
return
dataset = SeedMotionDataset(
root_dir=args.text_root_dir,
metadata_csv=args.text_metadata_csv,
split=split,
val_ratio=float(args.text_val_ratio),
split_seed=int(args.text_split_seed),
action_chunk_size=int(args.action_chunk_size),
upsample_rate=int(args.text_upsample_rate),
use_delta_actions=bool(args.use_delta_actions),
num_past_frames=int(args.num_past_frames),
state_history_stride=int(args.text_state_history_stride),
records_cache_dir=str(args.text_records_cache_dir),
)
_print_ready(
"READY",
task="text",
split=split,
records_cache=dataset._records_cache_path(),
motions=len(dataset.motion_records),
samples=dataset.num_samples,
)
def _prebuild_audio_family(
args: argparse.Namespace,
split: str,
*,
task_name: str,
raw_sources_json: str,
val_ratio: float,
split_seed: int,
sample_stride: int,
motion_fps: int,
audio_sampling_rate: int,
audio_window_seconds: float,
audio_left_context_seconds: float,
full_audio: bool,
align_start_hold: bool,
default_audio_task: str,
) -> None:
sources = _load_sources(raw_sources_json, arg_name=f"{task_name}-sources-json")
if not sources:
_print_ready("SKIP", task=task_name, split=split, reason="empty_sources")
return
for source in sources:
dataset = SeedAudioMotionDataset(
root_dir=str(source.get("root_dir", "")),
video_dir=str(source.get("video_dir", "")),
split=split,
val_ratio=float(source.get("val_ratio", val_ratio)),
split_seed=int(source.get("split_seed", split_seed)),
action_chunk_size=int(args.action_chunk_size),
sample_stride=int(source.get("sample_stride", sample_stride)),
use_delta_actions=bool(args.use_delta_actions),
num_past_frames=int(args.num_past_frames),
motion_fps=int(source.get("motion_fps", motion_fps)),
audio_sampling_rate=int(source.get("audio_sampling_rate", audio_sampling_rate)),
audio_window_seconds=float(source.get("audio_window_seconds", audio_window_seconds)),
audio_left_context_seconds=float(
source.get("audio_left_context_seconds", audio_left_context_seconds)
),
full_audio=_as_bool(source.get("full_audio", full_audio)),
align_start_hold=_as_bool(source.get("align_start_hold", align_start_hold)),
use_audio_instruction=True,
dataset_root=str(source.get("dataset_root", "")),
metadata_jsonl=str(source.get("metadata_jsonl", source.get("manifest_path", ""))),
motion_dir_key=str(source.get("motion_dir_key", "converted_path")),
audio_path_key=str(source.get("audio_path_key", source.get("video_path_key", "video_path"))),
instruction_key=str(source.get("instruction_key", "audio_caption")),
name_key=str(source.get("name_key", "name")),
id_key=str(source.get("id_key", "id")),
motion_metadata_csv=str(source.get("motion_metadata_csv", "")),
motion_metadata_key=str(source.get("motion_metadata_key", "filename")),
motion_metadata_path_key=str(source.get("motion_metadata_path_key", "move_g1_path")),
audio_file_extensions=str(source.get("audio_file_extensions", ".wav,.flac,.mp3,.m4a,.aac,.ogg,.npy")),
dataset_name=str(source.get("dataset_name", task_name)),
source_name=str(source.get("name", source.get("dataset_name", task_name))),
require_audio=_as_bool(source.get("require_audio", True)),
audio_task=str(source.get("audio_task", default_audio_task)),
records_cache_dir=str(source.get("records_cache_dir", getattr(args, f"{task_name}_records_cache_dir", ""))),
)
cache_path = (
dataset._records_cache_path() if dataset.metadata_jsonl else dataset._legacy_records_cache_path()
)
_print_ready(
"READY",
task=task_name,
split=split,
source=dataset.source_name,
records_cache=cache_path,
motions=len(dataset.motion_records),
samples=dataset.num_samples,
)
def prebuild_video(args: argparse.Namespace, split: str) -> None:
sources = _load_sources(args.video_sources_json, arg_name="video-sources-json")
if not sources:
_print_ready("SKIP", task="video", split=split, reason="empty_sources")
return
for source in sources:
dataset = SeedVideoMotionDataset(
root_dir=str(source.get("root_dir", "")),
video_dir=str(source.get("video_dir", "")),
split=split,
val_ratio=float(source.get("val_ratio", args.video_val_ratio)),
split_seed=int(source.get("split_seed", args.video_split_seed)),
action_chunk_size=int(args.action_chunk_size),
sample_stride=int(source.get("sample_stride", args.video_sample_stride)),
use_delta_actions=bool(args.use_delta_actions),
num_past_frames=int(args.num_past_frames),
motion_fps=int(source.get("motion_fps", args.video_motion_fps)),
video_sample_fps=float(source.get("video_sample_fps", args.video_video_sample_fps)),
video_size_pixels=int(source.get("video_size_pixels", args.video_video_size_pixels)),
video_decoder=str(source.get("video_decoder", args.video_video_decoder)),
dataset_root=str(source.get("dataset_root", "")),
metadata_jsonl=str(source.get("metadata_jsonl", source.get("manifest_path", ""))),
motion_dir_key=str(source.get("motion_dir_key", "converted_path")),
video_path_key=str(source.get("video_path_key", "video_path")),
instruction_key=str(source.get("instruction_key", "video_caption")),
name_key=str(source.get("name_key", "name")),
id_key=str(source.get("id_key", "id")),
dataset_name=str(source.get("dataset_name", "video")),
source_name=str(source.get("name", source.get("dataset_name", "video"))),
records_cache_dir=str(source.get("records_cache_dir", args.video_records_cache_dir)),
record_build_workers=int(
source.get("record_build_workers", args.video_record_build_workers)
),
record_build_chunk_size=int(
source.get("record_build_chunk_size", args.video_record_build_chunk_size)
),
build_base_sample_index=False,
)
cache_path = (
dataset._records_cache_path() if dataset.metadata_jsonl else dataset._legacy_records_cache_path()
)
_print_ready(
"READY",
task="video",
split=split,
source=dataset.source_name,
records_cache=cache_path,
motions=len(dataset.motion_records),
)
def prebuild_video_clip(args: argparse.Namespace, split: str) -> None:
sources = _load_sources(args.video_clip_sources_json, arg_name="video-clip-sources-json")
if not sources:
_print_ready("SKIP", task="video_clip", split=split, reason="empty_sources")
return
for source in sources:
dataset = SeedVideoClipMotionDataset(
root_dir=str(source.get("root_dir", "")),
video_dir=str(source.get("video_dir", "")),
split=split,
val_ratio=float(source.get("val_ratio", args.video_clip_val_ratio)),
split_seed=int(source.get("split_seed", args.video_clip_split_seed)),
action_chunk_size=int(args.action_chunk_size),
sample_stride=int(source.get("sample_stride", args.video_clip_sample_stride)),
use_delta_actions=bool(args.use_delta_actions),
num_past_frames=int(args.num_past_frames),
motion_fps=int(source.get("motion_fps", args.video_clip_motion_fps)),
video_sample_fps=float(source.get("video_sample_fps", args.video_clip_video_sample_fps)),
video_size_pixels=int(source.get("video_size_pixels", args.video_clip_video_size_pixels)),
video_decoder=str(source.get("video_decoder", args.video_clip_video_decoder)),
dataset_root=str(source.get("dataset_root", "")),
metadata_jsonl=str(source.get("metadata_jsonl", source.get("manifest_path", ""))),
motion_dir_key=str(source.get("motion_dir_key", "converted_path")),
video_path_key=str(source.get("video_path_key", "video_path")),
instruction_key=str(source.get("instruction_key", "video_caption")),
name_key=str(source.get("name_key", "name")),
id_key=str(source.get("id_key", "id")),
dataset_name=str(source.get("dataset_name", "video_clip")),
source_name=str(source.get("name", source.get("dataset_name", "video_clip"))),
trust_metadata=_as_bool(
source.get("trust_metadata", args.video_clip_trust_metadata)
),
records_cache_dir=str(source.get("records_cache_dir", args.video_clip_records_cache_dir)),
record_build_workers=int(
source.get("record_build_workers", args.video_clip_record_build_workers)
),
record_build_chunk_size=int(
source.get("record_build_chunk_size", args.video_clip_record_build_chunk_size)
),
action_exec_horizon=int(
source.get("action_exec_horizon", args.video_clip_action_exec_horizon)
),
clip_stride_frames=int(
source.get("clip_stride_frames", args.video_clip_clip_stride_frames)
),
)
records_cache = (
dataset._records_cache_path() if dataset.metadata_jsonl else dataset._legacy_records_cache_path()
)
_print_ready(
"READY",
task="video_clip",
split=split,
source=dataset.source_name,
records_cache=records_cache,
sample_index_cache=dataset._clip_sample_index_cache_path(),
motions=len(dataset.motion_records),
samples=len(dataset),
)
def main() -> None:
args = parse_args()
tasks = set(_csv_items(args.tasks))
splits = _csv_items(args.splits)
if not tasks:
raise ValueError("tasks is empty")
if not splits:
raise ValueError("splits is empty")
_print_ready("START", tasks=",".join(sorted(tasks)), splits=",".join(splits))
for split in splits:
if "text" in tasks:
prebuild_text(args, split)
if "audio" in tasks:
_prebuild_audio_family(
args,
split,
task_name="audio",
raw_sources_json=args.audio_sources_json,
val_ratio=float(args.audio_val_ratio),
split_seed=int(args.audio_split_seed),
sample_stride=int(args.audio_sample_stride),
motion_fps=int(args.audio_motion_fps),
audio_sampling_rate=int(args.audio_audio_sampling_rate),
audio_window_seconds=float(args.audio_audio_window_seconds),
audio_left_context_seconds=float(args.audio_audio_left_context_seconds),
full_audio=bool(args.audio_full_audio),
align_start_hold=bool(args.audio_align_start_hold),
default_audio_task="dance",
)
if "audio_command" in tasks:
_prebuild_audio_family(
args,
split,
task_name="audio_command",
raw_sources_json=args.audio_command_sources_json,
val_ratio=float(args.audio_command_val_ratio),
split_seed=int(args.audio_command_split_seed),
sample_stride=int(args.audio_command_sample_stride),
motion_fps=int(args.audio_command_motion_fps),
audio_sampling_rate=int(args.audio_command_audio_sampling_rate),
audio_window_seconds=float(args.audio_command_audio_window_seconds),
audio_left_context_seconds=float(args.audio_command_audio_left_context_seconds),
full_audio=bool(args.audio_command_full_audio),
align_start_hold=bool(args.audio_command_align_start_hold),
default_audio_task="command",
)
if "video" in tasks:
prebuild_video(args, split)
if "video_clip" in tasks:
prebuild_video_clip(args, split)
_print_ready("DONE", status="ok")
if __name__ == "__main__":
main()