Download scripts/data/prebuild_multimodal_records_cache.py from Tinghong-Ye22/code-HFD_lzk: direct link, hf CLI and curl.
- Browser
- Download file 21.3 kB
-
https://huggingface.co/Tinghong-Ye22/code-HFD_lzk/resolve/main/scripts/data/prebuild_multimodal_records_cache.py
- Command line
-
hf download hf://Tinghong-Ye22/code-HFD_lzk/scripts/data/prebuild_multimodal_records_cache.py
-
curl -L -o prebuild_multimodal_records_cache.py https://huggingface.co/Tinghong-Ye22/code-HFD_lzk/resolve/main/scripts/data/prebuild_multimodal_records_cache.py
21.3 kB
| from __future__ import annotations | |
| import argparse | |
| import json | |
| import os | |
| import sys | |
| from pathlib import Path | |
| from typing import Any | |
| REPO_ROOT = Path(__file__).resolve().parents[2] | |
| SRC_ROOT = REPO_ROOT / "src" | |
| if str(SRC_ROOT) not in sys.path: | |
| sys.path.insert(0, str(SRC_ROOT)) | |
| from psi.data.seed.seed_audio_motion_dataset import SeedAudioMotionDataset | |
| from psi.data.seed.seed_motion_dataset import SeedMotionDataset | |
| from psi.data.seed.seed_video_motion_dataset import SeedVideoClipMotionDataset, SeedVideoMotionDataset | |
| def _as_bool(value: Any) -> bool: | |
| if isinstance(value, bool): | |
| return value | |
| if isinstance(value, (int, float)): | |
| return bool(value) | |
| if isinstance(value, str): | |
| return value.strip().lower() in {"1", "true", "yes", "y", "on"} | |
| return bool(value) | |
| def _csv_items(value: str) -> list[str]: | |
| return [item.strip() for item in str(value).split(",") if item.strip()] | |
| def _load_sources(raw: str, *, arg_name: str) -> list[dict[str, Any]]: | |
| raw = str(raw or "").strip() | |
| if not raw: | |
| return [] | |
| try: | |
| data = json.loads(raw) | |
| except json.JSONDecodeError as exc: | |
| raise ValueError(f"Bad JSON in {arg_name}") from exc | |
| if not isinstance(data, list): | |
| raise ValueError(f"{arg_name} must be a JSON list") | |
| return [dict(item) for item in data] | |
| def _print_ready(prefix: str, **items: Any) -> None: | |
| fields = [f"{key}={value}" for key, value in items.items() if value not in (None, "")] | |
| print(f"[{prefix}] " + " ".join(fields), flush=True) | |
| def parse_args() -> argparse.Namespace: | |
| parser = argparse.ArgumentParser( | |
| description="Prebuild persistent multimodal records caches outside training startup." | |
| ) | |
| parser.add_argument("--tasks", default="text,audio,audio_command,video_clip") | |
| parser.add_argument("--splits", default="train,val") | |
| parser.add_argument("--action-chunk-size", type=int, default=30) | |
| parser.add_argument("--num-past-frames", type=int, default=0) | |
| parser.add_argument("--omni-hidden-root", type=str, default="") | |
| parser.add_argument( | |
| "--use-delta-actions", | |
| action=argparse.BooleanOptionalAction, | |
| default=False, | |
| ) | |
| parser.add_argument("--text-root-dir", type=str, default="") | |
| parser.add_argument("--text-metadata-csv", type=str, default="metadata/seed_metadata_v004.csv") | |
| parser.add_argument("--text-records-cache-dir", type=str, default="") | |
| parser.add_argument("--text-val-ratio", type=float, default=0.1) | |
| parser.add_argument("--text-split-seed", type=int, default=42) | |
| parser.add_argument("--text-split-source-namespace", type=str, default="") | |
| parser.add_argument("--text-upsample-rate", type=int, default=1) | |
| parser.add_argument("--text-state-history-stride", type=int, default=30) | |
| parser.add_argument("--audio-sources-json", type=str, default="") | |
| parser.add_argument("--audio-records-cache-dir", type=str, default="") | |
| parser.add_argument("--audio-val-ratio", type=float, default=0.1) | |
| parser.add_argument("--audio-split-seed", type=int, default=42) | |
| parser.add_argument("--audio-sample-stride", type=int, default=1) | |
| parser.add_argument("--audio-sample-step-frames", type=int, default=0) | |
| parser.add_argument("--audio-motion-fps", type=int, default=50) | |
| parser.add_argument("--audio-audio-sampling-rate", type=int, default=16000) | |
| parser.add_argument("--audio-audio-window-seconds", type=float, default=1.0) | |
| parser.add_argument("--audio-audio-left-context-seconds", type=float, default=0.0) | |
| parser.add_argument( | |
| "--audio-full-audio", | |
| action=argparse.BooleanOptionalAction, | |
| default=True, | |
| ) | |
| parser.add_argument( | |
| "--audio-align-start-hold", | |
| action=argparse.BooleanOptionalAction, | |
| default=True, | |
| ) | |
| parser.add_argument("--audio-command-sources-json", type=str, default="") | |
| parser.add_argument("--audio-command-records-cache-dir", type=str, default="") | |
| parser.add_argument("--audio-command-val-ratio", type=float, default=0.1) | |
| parser.add_argument("--audio-command-split-seed", type=int, default=42) | |
| parser.add_argument("--audio-command-sample-stride", type=int, default=1) | |
| parser.add_argument("--audio-command-sample-step-frames", type=int, default=0) | |
| parser.add_argument("--audio-command-motion-fps", type=int, default=50) | |
| parser.add_argument("--audio-command-audio-sampling-rate", type=int, default=16000) | |
| parser.add_argument("--audio-command-audio-window-seconds", type=float, default=1.0) | |
| parser.add_argument("--audio-command-audio-left-context-seconds", type=float, default=0.0) | |
| parser.add_argument( | |
| "--audio-command-full-audio", | |
| action=argparse.BooleanOptionalAction, | |
| default=True, | |
| ) | |
| parser.add_argument( | |
| "--audio-command-align-start-hold", | |
| action=argparse.BooleanOptionalAction, | |
| default=True, | |
| ) | |
| parser.add_argument("--video-sources-json", type=str, default="") | |
| parser.add_argument("--video-records-cache-dir", type=str, default="") | |
| parser.add_argument("--video-val-ratio", type=float, default=0.1) | |
| parser.add_argument("--video-split-seed", type=int, default=42) | |
| parser.add_argument("--video-sample-stride", type=int, default=1) | |
| parser.add_argument("--video-motion-fps", type=int, default=50) | |
| parser.add_argument("--video-video-sample-fps", type=float, default=1.0) | |
| parser.add_argument("--video-video-size-pixels", type=int, default=50176) | |
| parser.add_argument("--video-video-decoder", type=str, default="ffmpeg") | |
| parser.add_argument("--video-record-build-workers", type=int, default=0) | |
| parser.add_argument("--video-record-build-chunk-size", type=int, default=4096) | |
| parser.add_argument("--video-clip-sources-json", type=str, default="") | |
| parser.add_argument("--video-clip-records-cache-dir", type=str, default="") | |
| parser.add_argument("--video-clip-val-ratio", type=float, default=0.1) | |
| parser.add_argument("--video-clip-split-seed", type=int, default=42) | |
| parser.add_argument("--video-clip-sample-stride", type=int, default=1) | |
| parser.add_argument("--video-clip-motion-fps", type=int, default=50) | |
| parser.add_argument("--video-clip-video-sample-fps", type=float, default=8.0) | |
| parser.add_argument("--video-clip-video-size-pixels", type=int, default=50176) | |
| parser.add_argument("--video-clip-video-decoder", type=str, default="ffmpeg") | |
| parser.add_argument("--video-clip-record-build-workers", type=int, default=0) | |
| parser.add_argument("--video-clip-record-build-chunk-size", type=int, default=4096) | |
| parser.add_argument("--video-clip-action-exec-horizon", type=int, default=0) | |
| parser.add_argument("--video-clip-clip-stride-frames", type=int, default=0) | |
| parser.add_argument( | |
| "--video-clip-trust-metadata", | |
| action=argparse.BooleanOptionalAction, | |
| default=False, | |
| ) | |
| return parser.parse_args() | |
| def prebuild_text(args: argparse.Namespace, split: str) -> None: | |
| if not str(args.text_root_dir).strip(): | |
| _print_ready("SKIP", task="text", split=split, reason="empty_root_dir") | |
| return | |
| dataset = SeedMotionDataset( | |
| root_dir=args.text_root_dir, | |
| metadata_csv=args.text_metadata_csv, | |
| split=split, | |
| val_ratio=float(args.text_val_ratio), | |
| split_seed=int(args.text_split_seed), | |
| action_chunk_size=int(args.action_chunk_size), | |
| upsample_rate=int(args.text_upsample_rate), | |
| use_delta_actions=bool(args.use_delta_actions), | |
| num_past_frames=int(args.num_past_frames), | |
| state_history_stride=int(args.text_state_history_stride), | |
| records_cache_dir=str(args.text_records_cache_dir), | |
| split_source_namespace=str(args.text_split_source_namespace), | |
| ) | |
| _print_ready( | |
| "READY", | |
| task="text", | |
| split=split, | |
| records_cache=dataset._records_cache_path(), | |
| motions=len(dataset.motion_records), | |
| samples=dataset.num_samples, | |
| ) | |
| def _prebuild_audio_family( | |
| args: argparse.Namespace, | |
| split: str, | |
| *, | |
| task_name: str, | |
| raw_sources_json: str, | |
| val_ratio: float, | |
| split_seed: int, | |
| sample_stride: int, | |
| sample_step_frames: int, | |
| motion_fps: int, | |
| audio_sampling_rate: int, | |
| audio_window_seconds: float, | |
| audio_left_context_seconds: float, | |
| full_audio: bool, | |
| align_start_hold: bool, | |
| default_audio_task: str, | |
| ) -> None: | |
| sources = _load_sources(raw_sources_json, arg_name=f"{task_name}-sources-json") | |
| if not sources: | |
| _print_ready("SKIP", task=task_name, split=split, reason="empty_sources") | |
| return | |
| for source in sources: | |
| dataset = SeedAudioMotionDataset( | |
| root_dir=str(source.get("root_dir", "")), | |
| video_dir=str(source.get("video_dir", "")), | |
| split=split, | |
| val_ratio=float(source.get("val_ratio", val_ratio)), | |
| split_seed=int(source.get("split_seed", split_seed)), | |
| action_chunk_size=int(args.action_chunk_size), | |
| sample_stride=int(source.get("sample_stride", sample_stride)), | |
| sample_step_frames=int(source.get("sample_step_frames", sample_step_frames)), | |
| use_delta_actions=bool(args.use_delta_actions), | |
| num_past_frames=int(args.num_past_frames), | |
| motion_fps=int(source.get("motion_fps", motion_fps)), | |
| audio_sampling_rate=int(source.get("audio_sampling_rate", audio_sampling_rate)), | |
| audio_window_seconds=float(source.get("audio_window_seconds", audio_window_seconds)), | |
| audio_left_context_seconds=float( | |
| source.get("audio_left_context_seconds", audio_left_context_seconds) | |
| ), | |
| full_audio=_as_bool(source.get("full_audio", full_audio)), | |
| align_start_hold=_as_bool(source.get("align_start_hold", align_start_hold)), | |
| use_audio_instruction=True, | |
| dataset_root=str(source.get("dataset_root", "")), | |
| metadata_jsonl=str(source.get("metadata_jsonl", source.get("manifest_path", ""))), | |
| motion_dir_key=str(source.get("motion_dir_key", "converted_path")), | |
| audio_path_key=str(source.get("audio_path_key", source.get("video_path_key", "video_path"))), | |
| instruction_key=str(source.get("instruction_key", "audio_caption")), | |
| name_key=str(source.get("name_key", "name")), | |
| id_key=str(source.get("id_key", "id")), | |
| motion_metadata_csv=str(source.get("motion_metadata_csv", "")), | |
| motion_metadata_key=str(source.get("motion_metadata_key", "filename")), | |
| motion_metadata_path_key=str(source.get("motion_metadata_path_key", "move_g1_path")), | |
| audio_file_extensions=str(source.get("audio_file_extensions", ".wav,.flac,.mp3,.m4a,.aac,.ogg,.npy")), | |
| dataset_name=str(source.get("dataset_name", task_name)), | |
| source_name=str(source.get("name", source.get("dataset_name", task_name))), | |
| split_source_namespace=str(source.get("split_source_namespace", "")), | |
| require_audio=_as_bool(source.get("require_audio", True)), | |
| audio_task=str(source.get("audio_task", default_audio_task)), | |
| records_cache_dir=str(source.get("records_cache_dir", getattr(args, f"{task_name}_records_cache_dir", ""))), | |
| omni_hidden_root=str(source.get("omni_hidden_root", args.omni_hidden_root)), | |
| ) | |
| cache_path = ( | |
| dataset._records_cache_path() if dataset.metadata_jsonl else dataset._legacy_records_cache_path() | |
| ) | |
| _print_ready( | |
| "READY", | |
| task=task_name, | |
| split=split, | |
| source=dataset.source_name, | |
| records_cache=cache_path, | |
| motions=len(dataset.motion_records), | |
| samples=dataset.num_samples, | |
| ) | |
| if os.environ.get("PSI_PREBUILD_OMNI_OSS_KEY_INDEX", "1").strip().lower() in { | |
| "1", | |
| "true", | |
| "yes", | |
| "y", | |
| "on", | |
| }: | |
| from psi.data.seed.omni_oss_sampler import load_or_build_audio_key_index | |
| key_index = load_or_build_audio_key_index( | |
| dataset, | |
| dataset._build_omni_oss_audio_key_index, | |
| ) | |
| _print_ready( | |
| "READY", | |
| task=task_name, | |
| split=split, | |
| source=dataset.source_name, | |
| oss_audio_keys=len(key_index), | |
| oss_audio_associations=sum(len(value) for value in key_index.values()), | |
| ) | |
| def prebuild_video(args: argparse.Namespace, split: str) -> None: | |
| sources = _load_sources(args.video_sources_json, arg_name="video-sources-json") | |
| if not sources: | |
| _print_ready("SKIP", task="video", split=split, reason="empty_sources") | |
| return | |
| for source in sources: | |
| dataset = SeedVideoMotionDataset( | |
| root_dir=str(source.get("root_dir", "")), | |
| video_dir=str(source.get("video_dir", "")), | |
| split=split, | |
| val_ratio=float(source.get("val_ratio", args.video_val_ratio)), | |
| split_seed=int(source.get("split_seed", args.video_split_seed)), | |
| action_chunk_size=int(args.action_chunk_size), | |
| sample_stride=int(source.get("sample_stride", args.video_sample_stride)), | |
| use_delta_actions=bool(args.use_delta_actions), | |
| num_past_frames=int(args.num_past_frames), | |
| motion_fps=int(source.get("motion_fps", args.video_motion_fps)), | |
| video_sample_fps=float(source.get("video_sample_fps", args.video_video_sample_fps)), | |
| video_size_pixels=int(source.get("video_size_pixels", args.video_video_size_pixels)), | |
| video_decoder=str(source.get("video_decoder", args.video_video_decoder)), | |
| dataset_root=str(source.get("dataset_root", "")), | |
| metadata_jsonl=str(source.get("metadata_jsonl", source.get("manifest_path", ""))), | |
| motion_dir_key=str(source.get("motion_dir_key", "converted_path")), | |
| video_path_key=str(source.get("video_path_key", "video_path")), | |
| instruction_key=str(source.get("instruction_key", "video_caption")), | |
| name_key=str(source.get("name_key", "name")), | |
| id_key=str(source.get("id_key", "id")), | |
| dataset_name=str(source.get("dataset_name", "video")), | |
| source_name=str(source.get("name", source.get("dataset_name", "video"))), | |
| split_source_namespace=str(source.get("split_source_namespace", "")), | |
| records_cache_dir=str(source.get("records_cache_dir", args.video_records_cache_dir)), | |
| record_build_workers=int( | |
| source.get("record_build_workers", args.video_record_build_workers) | |
| ), | |
| record_build_chunk_size=int( | |
| source.get("record_build_chunk_size", args.video_record_build_chunk_size) | |
| ), | |
| build_base_sample_index=False, | |
| ) | |
| cache_path = ( | |
| dataset._records_cache_path() if dataset.metadata_jsonl else dataset._legacy_records_cache_path() | |
| ) | |
| _print_ready( | |
| "READY", | |
| task="video", | |
| split=split, | |
| source=dataset.source_name, | |
| records_cache=cache_path, | |
| motions=len(dataset.motion_records), | |
| ) | |
| def prebuild_video_clip(args: argparse.Namespace, split: str) -> None: | |
| sources = _load_sources(args.video_clip_sources_json, arg_name="video-clip-sources-json") | |
| if not sources: | |
| _print_ready("SKIP", task="video_clip", split=split, reason="empty_sources") | |
| return | |
| for source in sources: | |
| dataset = SeedVideoClipMotionDataset( | |
| root_dir=str(source.get("root_dir", "")), | |
| video_dir=str(source.get("video_dir", "")), | |
| split=split, | |
| val_ratio=float(source.get("val_ratio", args.video_clip_val_ratio)), | |
| split_seed=int(source.get("split_seed", args.video_clip_split_seed)), | |
| action_chunk_size=int(args.action_chunk_size), | |
| sample_stride=int(source.get("sample_stride", args.video_clip_sample_stride)), | |
| use_delta_actions=bool(args.use_delta_actions), | |
| num_past_frames=int(args.num_past_frames), | |
| motion_fps=int(source.get("motion_fps", args.video_clip_motion_fps)), | |
| video_sample_fps=float(source.get("video_sample_fps", args.video_clip_video_sample_fps)), | |
| video_size_pixels=int(source.get("video_size_pixels", args.video_clip_video_size_pixels)), | |
| video_decoder=str(source.get("video_decoder", args.video_clip_video_decoder)), | |
| dataset_root=str(source.get("dataset_root", "")), | |
| metadata_jsonl=str(source.get("metadata_jsonl", source.get("manifest_path", ""))), | |
| motion_dir_key=str(source.get("motion_dir_key", "converted_path")), | |
| video_path_key=str(source.get("video_path_key", "video_path")), | |
| instruction_key=str(source.get("instruction_key", "video_caption")), | |
| name_key=str(source.get("name_key", "name")), | |
| id_key=str(source.get("id_key", "id")), | |
| dataset_name=str(source.get("dataset_name", "video_clip")), | |
| source_name=str(source.get("name", source.get("dataset_name", "video_clip"))), | |
| split_source_namespace=str(source.get("split_source_namespace", "")), | |
| trust_metadata=_as_bool( | |
| source.get("trust_metadata", args.video_clip_trust_metadata) | |
| ), | |
| records_cache_dir=str(source.get("records_cache_dir", args.video_clip_records_cache_dir)), | |
| record_build_workers=int( | |
| source.get("record_build_workers", args.video_clip_record_build_workers) | |
| ), | |
| record_build_chunk_size=int( | |
| source.get("record_build_chunk_size", args.video_clip_record_build_chunk_size) | |
| ), | |
| action_exec_horizon=int( | |
| source.get("action_exec_horizon", args.video_clip_action_exec_horizon) | |
| ), | |
| clip_stride_frames=int( | |
| source.get("clip_stride_frames", args.video_clip_clip_stride_frames) | |
| ), | |
| ) | |
| records_cache = ( | |
| dataset._records_cache_path() if dataset.metadata_jsonl else dataset._legacy_records_cache_path() | |
| ) | |
| _print_ready( | |
| "READY", | |
| task="video_clip", | |
| split=split, | |
| source=dataset.source_name, | |
| records_cache=records_cache, | |
| sample_index_cache=dataset._clip_sample_index_cache_path(), | |
| motions=len(dataset.motion_records), | |
| samples=len(dataset), | |
| ) | |
| def main() -> None: | |
| args = parse_args() | |
| tasks = set(_csv_items(args.tasks)) | |
| splits = _csv_items(args.splits) | |
| if not tasks: | |
| raise ValueError("tasks is empty") | |
| if not splits: | |
| raise ValueError("splits is empty") | |
| _print_ready("START", tasks=",".join(sorted(tasks)), splits=",".join(splits)) | |
| for split in splits: | |
| if "text" in tasks: | |
| prebuild_text(args, split) | |
| if "audio" in tasks: | |
| _prebuild_audio_family( | |
| args, | |
| split, | |
| task_name="audio", | |
| raw_sources_json=args.audio_sources_json, | |
| val_ratio=float(args.audio_val_ratio), | |
| split_seed=int(args.audio_split_seed), | |
| sample_stride=int(args.audio_sample_stride), | |
| sample_step_frames=int(args.audio_sample_step_frames), | |
| motion_fps=int(args.audio_motion_fps), | |
| audio_sampling_rate=int(args.audio_audio_sampling_rate), | |
| audio_window_seconds=float(args.audio_audio_window_seconds), | |
| audio_left_context_seconds=float(args.audio_audio_left_context_seconds), | |
| full_audio=bool(args.audio_full_audio), | |
| align_start_hold=bool(args.audio_align_start_hold), | |
| default_audio_task="dance", | |
| ) | |
| if "audio_command" in tasks: | |
| _prebuild_audio_family( | |
| args, | |
| split, | |
| task_name="audio_command", | |
| raw_sources_json=args.audio_command_sources_json, | |
| val_ratio=float(args.audio_command_val_ratio), | |
| split_seed=int(args.audio_command_split_seed), | |
| sample_stride=int(args.audio_command_sample_stride), | |
| sample_step_frames=int(args.audio_command_sample_step_frames), | |
| motion_fps=int(args.audio_command_motion_fps), | |
| audio_sampling_rate=int(args.audio_command_audio_sampling_rate), | |
| audio_window_seconds=float(args.audio_command_audio_window_seconds), | |
| audio_left_context_seconds=float(args.audio_command_audio_left_context_seconds), | |
| full_audio=bool(args.audio_command_full_audio), | |
| align_start_hold=bool(args.audio_command_align_start_hold), | |
| default_audio_task="command", | |
| ) | |
| if "video" in tasks: | |
| prebuild_video(args, split) | |
| if "video_clip" in tasks: | |
| prebuild_video_clip(args, split) | |
| _print_ready("DONE", status="ok") | |
| if __name__ == "__main__": | |
| main() | |