Download scripts/data/build_video_clip_dataset.py from Tinghong-Ye22/code-Psi0-yth-v21: direct link, hf CLI and curl.
- Browser
- Download file 35.8 kB
-
https://huggingface.co/Tinghong-Ye22/code-Psi0-yth-v21/resolve/main/scripts/data/build_video_clip_dataset.py
- Command line
-
hf download hf://Tinghong-Ye22/code-Psi0-yth-v21/scripts/data/build_video_clip_dataset.py
-
curl -L -o build_video_clip_dataset.py https://huggingface.co/Tinghong-Ye22/code-Psi0-yth-v21/resolve/main/scripts/data/build_video_clip_dataset.py
35.8 kB
| #!/usr/bin/env python3 | |
| from __future__ import annotations | |
| import argparse | |
| import hashlib | |
| import json | |
| import os | |
| import random | |
| import shutil | |
| import subprocess | |
| import uuid | |
| from concurrent.futures import ThreadPoolExecutor, as_completed | |
| from functools import lru_cache | |
| from pathlib import Path | |
| from typing import Iterable | |
| import pandas as pd | |
| from tqdm import tqdm | |
| MOTION_CSVS = ( | |
| "joint_pos.csv", | |
| "joint_vel.csv", | |
| "body_pos.csv", | |
| "body_quat.csv", | |
| "body_lin_vel.csv", | |
| "body_ang_vel.csv", | |
| ) | |
| def ffmpeg_bin() -> str: | |
| return os.environ.get("PSI_FFMPEG_BIN", "ffmpeg") | |
| def ffprobe_bin() -> str: | |
| return os.environ.get("PSI_FFPROBE_BIN", "ffprobe") | |
| def parse_args() -> argparse.Namespace: | |
| parser = argparse.ArgumentParser(description="Build short video-clip datasets for Psi0 video_clip training.") | |
| parser.add_argument("--dataset", choices=["dance", "motion_example"], required=True) | |
| parser.add_argument("--output-root", type=str, required=True) | |
| parser.add_argument("--clip-frames", type=int, default=50) | |
| parser.add_argument("--action-chunk-frames", type=int, default=30) | |
| parser.add_argument("--chunk-stride-frames", type=int, default=20) | |
| parser.add_argument("--motion-fps", type=float, default=50.0) | |
| parser.add_argument("--clips-per-video", type=int, default=2) | |
| parser.add_argument("--clip-selection-mode", choices=["start", "middle_random", "all"], default="middle_random") | |
| parser.add_argument("--random-seed", type=int, default=42) | |
| parser.add_argument("--middle-fraction", type=float, default=0.8) | |
| parser.add_argument("--num-workers", type=int, default=1) | |
| parser.add_argument("--chunk-workers", type=int, default=int(os.environ.get("PSI_CHUNK_WORKERS", "1"))) | |
| parser.add_argument("--ffmpeg-threads", type=int, default=int(os.environ.get("PSI_FFMPEG_THREADS", "1"))) | |
| parser.add_argument("--x264-preset", type=str, default=os.environ.get("PSI_X264_PRESET", "")) | |
| parser.add_argument("--chunk-progress-interval", type=int, default=0) | |
| parser.add_argument( | |
| "--existing-video-check", | |
| choices=["ffprobe", "size"], | |
| default=os.environ.get("PSI_EXISTING_VIDEO_CHECK", "ffprobe"), | |
| help="How to validate already-existing output clips before skipping them.", | |
| ) | |
| parser.add_argument("--limit", type=int, default=0) | |
| parser.add_argument("--overwrite", action="store_true", default=False) | |
| parser.add_argument( | |
| "--recover-existing-only", | |
| action="store_true", | |
| default=os.environ.get("RECOVER_EXISTING_ONLY", "0") == "1", | |
| help="Only register fully-existing source clips into .source_done; do not create missing clips.", | |
| ) | |
| parser.add_argument( | |
| "--metadata-only", | |
| action="store_true", | |
| default=os.environ.get("METADATA_ONLY", "0") == "1", | |
| help="Write dense clip metadata only. Do not crop mp4 clips or write per-chunk motion CSV directories.", | |
| ) | |
| parser.add_argument("--dance-motion-root", type=str, default="") | |
| parser.add_argument("--dance-video-root", type=str, default="") | |
| parser.add_argument("--motion-example-root", type=str, default="") | |
| parser.add_argument("--metadata-jsonl", type=str, default="metadata.jsonl") | |
| parser.add_argument("--shard-num", type=int, default=1) | |
| parser.add_argument("--shard-id", type=int, default=0) | |
| return parser.parse_args() | |
| def read_num_frames(motion_dir: Path) -> int: | |
| joint_path = motion_dir / "joint_pos.csv" | |
| if not joint_path.exists(): | |
| return 0 | |
| return int(len(pd.read_csv(joint_path))) | |
| def read_motion_tables(motion_dir: Path) -> dict[str, pd.DataFrame]: | |
| tables = {} | |
| for csv_name in MOTION_CSVS: | |
| csv_path = motion_dir / csv_name | |
| if csv_path.exists(): | |
| tables[csv_name] = pd.read_csv(csv_path) | |
| return tables | |
| def _parse_rate(value: str) -> float: | |
| if not value or value == "0/0": | |
| return 0.0 | |
| if "/" in value: | |
| num, den = value.split("/", 1) | |
| den_f = float(den) | |
| return float(num) / den_f if den_f else 0.0 | |
| return float(value) | |
| def probe_video(video_path: Path) -> tuple[int, int, float, float]: | |
| cmd = [ | |
| ffprobe_bin(), | |
| "-v", | |
| "error", | |
| "-select_streams", | |
| "v:0", | |
| "-show_entries", | |
| "stream=width,height,avg_frame_rate,r_frame_rate,duration:format=duration", | |
| "-of", | |
| "json", | |
| str(video_path), | |
| ] | |
| proc = subprocess.run(cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE, check=False) | |
| if proc.returncode != 0: | |
| msg = proc.stderr.decode("utf-8", errors="ignore") | |
| raise RuntimeError(f"ffprobe failed for {video_path}: {msg}") | |
| info = json.loads(proc.stdout.decode("utf-8")) | |
| streams = info.get("streams") or [] | |
| if not streams: | |
| raise RuntimeError(f"No video stream found by ffprobe: {video_path}") | |
| stream = streams[0] | |
| width = int(stream.get("width") or 0) | |
| height = int(stream.get("height") or 0) | |
| fps = _parse_rate(str(stream.get("avg_frame_rate") or stream.get("r_frame_rate") or "0/0")) | |
| duration = float(stream.get("duration") or info.get("format", {}).get("duration") or 0.0) | |
| if width <= 0 or height <= 0 or fps <= 0 or duration <= 0: | |
| raise RuntimeError( | |
| f"Bad video metadata at {video_path}: width={width}, height={height}, " | |
| f"fps={fps}, duration={duration}" | |
| ) | |
| return width, height, fps, duration | |
| def available_ffmpeg_encoders() -> set[str]: | |
| proc = subprocess.run( | |
| [ffmpeg_bin(), "-hide_banner", "-encoders"], | |
| stdout=subprocess.PIPE, | |
| stderr=subprocess.PIPE, | |
| check=False, | |
| ) | |
| output = proc.stdout.decode("utf-8", errors="ignore") | |
| encoders = set() | |
| for line in output.splitlines(): | |
| parts = line.split() | |
| if len(parts) >= 2 and parts[0].startswith("V"): | |
| encoders.add(parts[1]) | |
| return encoders | |
| def video_encoder_args(x264_preset: str = "") -> list[str]: | |
| encoders = available_ffmpeg_encoders() | |
| if "libx264" in encoders: | |
| args = ["-c:v", "libx264"] | |
| if x264_preset: | |
| args.extend(["-preset", x264_preset]) | |
| args.extend(["-pix_fmt", "yuv420p"]) | |
| return args | |
| if "h264_nvenc" in encoders: | |
| return ["-c:v", "h264_nvenc", "-pix_fmt", "yuv420p"] | |
| if "mpeg4" in encoders: | |
| return ["-c:v", "mpeg4", "-q:v", "2", "-pix_fmt", "yuv420p"] | |
| raise RuntimeError("ffmpeg has no usable video encoder: need libx264, h264_nvenc, or mpeg4") | |
| def copy_motion_clip( | |
| src_motion_dir: Path, | |
| dst_motion_dir: Path, | |
| start_frame: int, | |
| clip_frames: int, | |
| motion_tables: dict[str, pd.DataFrame] | None = None, | |
| ) -> None: | |
| dst_motion_dir.mkdir(parents=True, exist_ok=True) | |
| tables = motion_tables if motion_tables is not None else read_motion_tables(src_motion_dir) | |
| for csv_name in MOTION_CSVS: | |
| df = tables.get(csv_name) | |
| if df is None: | |
| continue | |
| clip_df = df.iloc[start_frame : start_frame + clip_frames].copy() | |
| if len(clip_df) != clip_frames: | |
| raise ValueError( | |
| f"{src_motion_dir / csv_name} has only {len(clip_df)} rows for requested clip" | |
| ) | |
| clip_df.to_csv(dst_motion_dir / csv_name, index=False) | |
| info_src = src_motion_dir / "info.txt" | |
| if info_src.exists(): | |
| shutil.copy2(info_src, dst_motion_dir / "info_source.txt") | |
| metadata_src = src_motion_dir / "metadata.txt" | |
| if metadata_src.exists(): | |
| shutil.copy2(metadata_src, dst_motion_dir / "metadata_source.txt") | |
| (dst_motion_dir / "metadata.txt").write_text( | |
| "\n".join( | |
| [ | |
| "Motion Information: video_clip_50f", | |
| "==================================", | |
| "", | |
| f"source_motion_dir: {src_motion_dir}", | |
| f"source_start_frame: {start_frame}", | |
| f"frames: {clip_frames}", | |
| "", | |
| ] | |
| ), | |
| encoding="utf-8", | |
| ) | |
| def crop_video( | |
| src_video: Path, | |
| dst_video: Path, | |
| start_sec: float, | |
| duration_sec: float, | |
| overwrite: bool, | |
| existing_video_check: str, | |
| ffmpeg_threads: int, | |
| x264_preset: str, | |
| ) -> None: | |
| if dst_video.exists() and not overwrite: | |
| if video_clip_ready(dst_video, duration_sec, existing_video_check): | |
| return | |
| dst_video.unlink() | |
| dst_video.parent.mkdir(parents=True, exist_ok=True) | |
| tmp_video = dst_video.with_name(f".{dst_video.name}.{os.getpid()}.{uuid.uuid4().hex}.tmp.mp4") | |
| cmd = [ | |
| ffmpeg_bin(), | |
| "-hide_banner", | |
| "-loglevel", | |
| "error", | |
| "-y", | |
| "-ss", | |
| f"{start_sec:.6f}", | |
| "-t", | |
| f"{duration_sec:.6f}", | |
| "-i", | |
| str(src_video), | |
| "-threads", | |
| str(max(1, int(ffmpeg_threads))), | |
| "-map", | |
| "0:v:0", | |
| "-an", | |
| *video_encoder_args(x264_preset), | |
| str(tmp_video), | |
| ] | |
| try: | |
| proc = subprocess.run(cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE, check=False) | |
| if proc.returncode != 0: | |
| msg = proc.stderr.decode("utf-8", errors="ignore") | |
| raise RuntimeError(f"ffmpeg failed for {src_video}: {msg}") | |
| if not video_clip_ready(tmp_video, duration_sec, "size"): | |
| raise RuntimeError(f"ffmpeg wrote invalid/too-short video for {src_video}") | |
| tmp_video.replace(dst_video) | |
| except Exception: | |
| try: | |
| tmp_video.unlink() | |
| except FileNotFoundError: | |
| pass | |
| raise | |
| def motion_clip_ready(motion_dir: Path) -> bool: | |
| return (motion_dir / "joint_pos.csv").exists() and (motion_dir / "joint_pos.csv").stat().st_size > 0 | |
| def video_clip_ready(video_path: Path, expected_duration_sec: float, check_mode: str = "ffprobe") -> bool: | |
| if not video_path.exists() or video_path.stat().st_size <= 0: | |
| return False | |
| if check_mode == "size": | |
| return True | |
| try: | |
| _, _, _, duration_sec = probe_video(video_path) | |
| except Exception: | |
| return False | |
| return duration_sec >= max(0.1, float(expected_duration_sec) * 0.5) | |
| def write_motion_clip_atomic( | |
| src_motion_dir: Path, | |
| dst_motion_dir: Path, | |
| start_frame: int, | |
| clip_frames: int, | |
| motion_tables: dict[str, pd.DataFrame] | None = None, | |
| ) -> None: | |
| if motion_clip_ready(dst_motion_dir): | |
| return | |
| dst_motion_dir.parent.mkdir(parents=True, exist_ok=True) | |
| tmp_motion_dir = dst_motion_dir.with_name( | |
| f".{dst_motion_dir.name}.{os.getpid()}.{uuid.uuid4().hex}.tmp" | |
| ) | |
| try: | |
| if tmp_motion_dir.exists(): | |
| shutil.rmtree(tmp_motion_dir) | |
| copy_motion_clip(src_motion_dir, tmp_motion_dir, start_frame, clip_frames, motion_tables) | |
| if not motion_clip_ready(tmp_motion_dir): | |
| raise RuntimeError(f"motion clip was not written correctly: {tmp_motion_dir}") | |
| if dst_motion_dir.exists(): | |
| if motion_clip_ready(dst_motion_dir): | |
| shutil.rmtree(tmp_motion_dir) | |
| return | |
| shutil.rmtree(dst_motion_dir) | |
| try: | |
| tmp_motion_dir.replace(dst_motion_dir) | |
| except FileExistsError: | |
| if motion_clip_ready(dst_motion_dir): | |
| shutil.rmtree(tmp_motion_dir) | |
| return | |
| shutil.rmtree(dst_motion_dir) | |
| tmp_motion_dir.replace(dst_motion_dir) | |
| except Exception: | |
| if tmp_motion_dir.exists(): | |
| shutil.rmtree(tmp_motion_dir) | |
| raise | |
| def source_rng(source_id: str, seed: int) -> random.Random: | |
| digest = hashlib.sha1(f"{seed}:{source_id}".encode("utf-8")).hexdigest() | |
| return random.Random(int(digest[:16], 16)) | |
| def clip_start_candidates( | |
| n_frames: int, | |
| *, | |
| args: argparse.Namespace, | |
| motion_chunk_frames: int, | |
| max_video_start_frame: int | None = None, | |
| ) -> list[int]: | |
| max_start_frame = n_frames - motion_chunk_frames | |
| if max_video_start_frame is not None: | |
| max_start_frame = min(max_start_frame, int(max_video_start_frame)) | |
| if max_start_frame < 0: | |
| return [] | |
| stride = max(1, int(args.chunk_stride_frames)) | |
| all_candidates = list(range(0, max_start_frame + 1, stride)) | |
| if max_start_frame not in all_candidates: | |
| all_candidates.append(max_start_frame) | |
| if args.clip_selection_mode == "start": | |
| return all_candidates | |
| if args.clip_selection_mode == "all": | |
| return all_candidates | |
| middle_fraction = min(max(float(args.middle_fraction), 0.0), 1.0) | |
| if middle_fraction <= 0.0: | |
| return all_candidates | |
| margin = int(round(max_start_frame * (1.0 - middle_fraction) / 2.0)) | |
| lo = min(max_start_frame, max(0, margin)) | |
| hi = max(lo, max_start_frame - margin) | |
| middle_candidates = [frame for frame in all_candidates if lo <= frame <= hi] | |
| return middle_candidates or all_candidates | |
| def select_clip_starts( | |
| source: dict, | |
| n_frames: int, | |
| *, | |
| args: argparse.Namespace, | |
| motion_chunk_frames: int, | |
| max_video_start_frame: int | None = None, | |
| ) -> list[int]: | |
| candidates = clip_start_candidates( | |
| n_frames, | |
| args=args, | |
| motion_chunk_frames=motion_chunk_frames, | |
| max_video_start_frame=max_video_start_frame, | |
| ) | |
| if not candidates: | |
| return [] | |
| if args.clip_selection_mode == "all": | |
| return candidates | |
| if args.clip_selection_mode == "start": | |
| return candidates[: args.clips_per_video] | |
| rng = source_rng(str(source["source_id"]), int(args.random_seed)) | |
| if len(candidates) <= args.clips_per_video: | |
| selected = candidates | |
| else: | |
| selected = rng.sample(candidates, args.clips_per_video) | |
| return sorted(selected) | |
| def write_jsonl(path: Path, rows: Iterable[dict]) -> int: | |
| path.parent.mkdir(parents=True, exist_ok=True) | |
| count = 0 | |
| tmp_path = path.with_name(f".{path.name}.{os.getpid()}.{uuid.uuid4().hex}.tmp") | |
| with tmp_path.open("w", encoding="utf-8") as f: | |
| for row in rows: | |
| f.write(json.dumps(row, ensure_ascii=False) + "\n") | |
| count += 1 | |
| tmp_path.replace(path) | |
| return count | |
| def stable_hash_mod(value: str, modulo: int) -> int: | |
| digest = hashlib.sha1(value.encode("utf-8")).digest() | |
| return int.from_bytes(digest[:8], "big") % int(modulo) | |
| def source_shard_key(source: dict) -> str: | |
| value = ( | |
| source.get("rel_name") | |
| or source.get("source_id") | |
| or source.get("name") | |
| or source.get("video_path") | |
| ) | |
| return str(value) | |
| def safe_source_name(value: str, max_len: int = 96) -> str: | |
| safe = "".join(ch if ch.isalnum() or ch in "._-" else "_" for ch in value) | |
| safe = safe.strip("._-") | |
| return (safe or "source")[:max_len] | |
| def source_done_path(output_root: Path, source: dict) -> Path: | |
| key = source_shard_key(source) | |
| digest = hashlib.sha1(key.encode("utf-8")).hexdigest()[:16] | |
| return output_root / ".source_done" / f"{safe_source_name(key)}.{digest}.json" | |
| def source_done_signature(args: argparse.Namespace) -> dict: | |
| return { | |
| "version": 1, | |
| "dataset": str(args.dataset), | |
| "metadata_only": bool(args.metadata_only), | |
| "clip_frames": int(args.clip_frames), | |
| "action_chunk_frames": int(args.action_chunk_frames), | |
| "motion_chunk_frames": int(args.action_chunk_frames) + 1, | |
| "chunk_stride_frames": int(args.chunk_stride_frames), | |
| "motion_fps": float(args.motion_fps), | |
| "clips_per_video": int(args.clips_per_video), | |
| "clip_selection_mode": str(args.clip_selection_mode), | |
| "random_seed": int(args.random_seed), | |
| "middle_fraction": float(args.middle_fraction), | |
| } | |
| def normalize_source_done_signature(signature: dict | None) -> dict: | |
| if not isinstance(signature, dict): | |
| return {} | |
| normalized = dict(signature) | |
| for key in ("existing_video_check", "encoder_args", "ffmpeg_threads", "x264_preset"): | |
| normalized.pop(key, None) | |
| return normalized | |
| def build_clip_metadata_row( | |
| source: dict, | |
| *, | |
| args: argparse.Namespace, | |
| output_root: Path, | |
| motion_out_root: Path, | |
| video_out_root: Path, | |
| src_motion_dir: Path, | |
| src_video_path: Path, | |
| source_video_duration_sec: float, | |
| motion_chunk_frames: int, | |
| chunk_duration_sec: float, | |
| chunk_idx: int, | |
| chunk_start_frame: int, | |
| ) -> dict: | |
| chunk_start_sec = float(chunk_start_frame) / float(args.motion_fps) | |
| chunk_name = f"{source.get('rel_name', source['name'])}.chunk{chunk_idx:02d}.f{chunk_start_frame:06d}" | |
| dst_motion_dir = motion_out_root / chunk_name | |
| dst_video_path = video_out_root / f"{chunk_name}.mp4" | |
| row = { | |
| "id": f"{source['source_id']}.chunk{chunk_idx:02d}.f{chunk_start_frame:06d}", | |
| "name": chunk_name, | |
| "converted_path": str(dst_motion_dir.relative_to(output_root)), | |
| "video_path": str(dst_video_path.relative_to(output_root)), | |
| "video_caption": source["caption"], | |
| "duration": chunk_duration_sec, | |
| "motion_fps": float(args.motion_fps), | |
| "clip_frames": int(motion_chunk_frames), | |
| "context_frames": int(args.clip_frames), | |
| "action_chunk_frames": int(args.action_chunk_frames), | |
| "motion_chunk_frames": int(motion_chunk_frames), | |
| "chunk_stride_frames": int(args.chunk_stride_frames), | |
| "clip_selection_mode": str(args.clip_selection_mode), | |
| "random_seed": int(args.random_seed), | |
| "middle_fraction": float(args.middle_fraction), | |
| "context_start_frame": int(chunk_start_frame), | |
| "chunk_start_frame": int(chunk_start_frame), | |
| "chunk_start_sec": float(chunk_start_sec), | |
| "source_video_duration_sec": float(source_video_duration_sec), | |
| "source_motion_dir": str(src_motion_dir), | |
| "source_video_path": str(src_video_path), | |
| "motion_type": source["motion_type"], | |
| } | |
| if args.metadata_only: | |
| row["metadata_only"] = True | |
| return row | |
| def read_source_done( | |
| output_root: Path, | |
| source: dict, | |
| args: argparse.Namespace, | |
| ) -> tuple[list[dict], int, int] | None: | |
| path = source_done_path(output_root, source) | |
| if not path.exists(): | |
| return None | |
| try: | |
| payload = json.loads(path.read_text(encoding="utf-8")) | |
| except Exception: | |
| return None | |
| if normalize_source_done_signature(payload.get("signature")) != normalize_source_done_signature(source_done_signature(args)): | |
| return None | |
| rows = payload.get("rows") | |
| if not isinstance(rows, list): | |
| return None | |
| skipped_short = int(payload.get("skipped_short") or 0) | |
| return rows, skipped_short, 0 | |
| def write_source_done( | |
| output_root: Path, | |
| source: dict, | |
| args: argparse.Namespace, | |
| rows: list[dict], | |
| skipped_short: int, | |
| ) -> None: | |
| path = source_done_path(output_root, source) | |
| path.parent.mkdir(parents=True, exist_ok=True) | |
| payload = { | |
| "signature": source_done_signature(args), | |
| "source_key": source_shard_key(source), | |
| "source_id": str(source.get("source_id") or ""), | |
| "name": str(source.get("name") or ""), | |
| "rel_name": str(source.get("rel_name") or ""), | |
| "rows": rows, | |
| "row_count": len(rows), | |
| "skipped_short": int(skipped_short), | |
| } | |
| tmp_path = path.with_name(f".{path.name}.{os.getpid()}.{uuid.uuid4().hex}.tmp") | |
| tmp_path.write_text(json.dumps(payload, ensure_ascii=False), encoding="utf-8") | |
| tmp_path.replace(path) | |
| def build_existing_source_rows( | |
| source: dict, | |
| *, | |
| args: argparse.Namespace, | |
| motion_out_root: Path, | |
| video_out_root: Path, | |
| output_root: Path, | |
| motion_chunk_frames: int, | |
| chunk_duration_sec: float, | |
| ) -> tuple[list[dict], int, int]: | |
| done_result = read_source_done(output_root, source, args) | |
| if done_result is not None: | |
| return done_result | |
| metadata_rows = [] | |
| src_motion_dir: Path = source["motion_dir"] | |
| src_video_path: Path = source["video_path"] | |
| n_frames = read_num_frames(src_motion_dir) | |
| if n_frames < motion_chunk_frames: | |
| write_source_done(output_root, source, args, metadata_rows, 1) | |
| return metadata_rows, 1, 0 | |
| try: | |
| _, _, _, source_video_duration_sec = probe_video(src_video_path) | |
| except Exception as exc: | |
| print(f"[RECOVER_MISS] {src_video_path}: {exc}") | |
| return metadata_rows, 0, 0 | |
| max_video_start_frame = int( | |
| (source_video_duration_sec - chunk_duration_sec) * float(args.motion_fps) | |
| ) | |
| if max_video_start_frame < 0: | |
| write_source_done(output_root, source, args, metadata_rows, 1) | |
| return metadata_rows, 1, 0 | |
| clip_starts = select_clip_starts( | |
| source, | |
| n_frames, | |
| args=args, | |
| motion_chunk_frames=motion_chunk_frames, | |
| max_video_start_frame=max_video_start_frame, | |
| ) | |
| if not clip_starts: | |
| write_source_done(output_root, source, args, metadata_rows, 1) | |
| return metadata_rows, 1, 0 | |
| for chunk_idx, chunk_start_frame in enumerate(clip_starts): | |
| row = build_clip_metadata_row( | |
| source, | |
| args=args, | |
| output_root=output_root, | |
| motion_out_root=motion_out_root, | |
| video_out_root=video_out_root, | |
| src_motion_dir=src_motion_dir, | |
| src_video_path=src_video_path, | |
| source_video_duration_sec=source_video_duration_sec, | |
| motion_chunk_frames=motion_chunk_frames, | |
| chunk_duration_sec=chunk_duration_sec, | |
| chunk_idx=chunk_idx, | |
| chunk_start_frame=chunk_start_frame, | |
| ) | |
| dst_motion_dir = output_root / str(row["converted_path"]) | |
| dst_video_path = output_root / str(row["video_path"]) | |
| if not video_clip_ready(dst_video_path, chunk_duration_sec, "size"): | |
| return [], 0, 0 | |
| if not motion_clip_ready(dst_motion_dir): | |
| return [], 0, 0 | |
| metadata_rows.append(row) | |
| write_source_done(output_root, source, args, metadata_rows, 0) | |
| return metadata_rows, 0, 0 | |
| def shard_metadata_path(output_root: Path, shard_id: int, shard_num: int) -> Path: | |
| return output_root / f"metadata.shard-{shard_id:05d}-of-{shard_num:05d}.jsonl" | |
| def merge_shard_metadata(output_root: Path, shard_num: int, final_path: Path) -> bool: | |
| shard_paths = [shard_metadata_path(output_root, shard_id, shard_num) for shard_id in range(shard_num)] | |
| missing = [path for path in shard_paths if not path.exists()] | |
| if missing: | |
| print( | |
| f"[SHARD] metadata merge pending: {len(missing)}/{shard_num} shard file(s) missing" | |
| ) | |
| return False | |
| rows = [] | |
| for path in shard_paths: | |
| with path.open("r", encoding="utf-8") as f: | |
| for line in f: | |
| line = line.strip() | |
| if line: | |
| rows.append(json.loads(line)) | |
| rows.sort(key=lambda row: str(row.get("name", ""))) | |
| count = write_jsonl(final_path, rows) | |
| print(f"[SHARD] merged metadata rows={count} output={final_path}") | |
| return True | |
| def iter_dance_rows(args: argparse.Namespace): | |
| motion_root = Path(args.dance_motion_root) | |
| video_root = Path(args.dance_video_root) | |
| if not motion_root.exists(): | |
| raise FileNotFoundError(f"dance motion root does not exist: {motion_root}") | |
| if not video_root.exists(): | |
| raise FileNotFoundError(f"dance video root does not exist: {video_root}") | |
| motion_dirs = sorted(path for path in motion_root.iterdir() if path.is_dir()) | |
| if args.limit > 0: | |
| motion_dirs = motion_dirs[: args.limit] | |
| for motion_dir in motion_dirs: | |
| video_path = video_root / f"{motion_dir.name}.mp4" | |
| if video_path.exists(): | |
| yield { | |
| "name": motion_dir.name, | |
| "rel_name": motion_dir.name, | |
| "motion_dir": motion_dir, | |
| "video_path": video_path, | |
| "caption": "Generate a motion sequence that matches the provided video.", | |
| "source_id": motion_dir.name, | |
| "motion_type": "dance", | |
| } | |
| def iter_motion_example_rows(args: argparse.Namespace): | |
| root = Path(args.motion_example_root) | |
| metadata_path = root / args.metadata_jsonl | |
| if not metadata_path.exists(): | |
| raise FileNotFoundError(f"motion_example metadata does not exist: {metadata_path}") | |
| rows = [] | |
| with metadata_path.open("r", encoding="utf-8") as f: | |
| for line in f: | |
| line = line.strip() | |
| if line: | |
| rows.append(json.loads(line)) | |
| if args.limit > 0: | |
| rows = rows[: args.limit] | |
| for row in rows: | |
| motion_value = row.get("converted_path") or row.get("motion_dir") or row.get("motion_path") | |
| video_value = row.get("video_path") or row.get("original_video") or row.get("real_path") | |
| if not motion_value or not video_value: | |
| continue | |
| motion_dir = root / str(motion_value) | |
| video_path = root / str(video_value) | |
| if motion_dir.exists() and video_path.exists(): | |
| yield { | |
| "name": str(row.get("name") or motion_dir.name), | |
| "rel_name": str(Path(str(motion_value))).replace("/", "__"), | |
| "motion_dir": motion_dir, | |
| "video_path": video_path, | |
| "caption": str(row.get("video_caption") or row.get("caption") or ""), | |
| "source_id": str(row.get("id") or row.get("name") or motion_dir.name), | |
| "motion_type": str(row.get("motion_type") or ""), | |
| } | |
| def build_source_clips( | |
| source: dict, | |
| *, | |
| args: argparse.Namespace, | |
| motion_out_root: Path, | |
| video_out_root: Path, | |
| output_root: Path, | |
| motion_chunk_frames: int, | |
| chunk_duration_sec: float, | |
| ) -> tuple[list[dict], int, int]: | |
| done_result = read_source_done(output_root, source, args) | |
| if done_result is not None: | |
| return done_result | |
| metadata_rows = [] | |
| skipped_short = 0 | |
| failed = 0 | |
| def finish(rows: list[dict], skipped: int, failed_count: int) -> tuple[list[dict], int, int]: | |
| if failed_count == 0: | |
| try: | |
| write_source_done(output_root, source, args, rows, skipped) | |
| except Exception as exc: | |
| print(f"[WARN] source done marker failed for {source_shard_key(source)}: {exc}") | |
| return rows, skipped, failed_count | |
| src_motion_dir: Path = source["motion_dir"] | |
| src_video_path: Path = source["video_path"] | |
| motion_tables = read_motion_tables(src_motion_dir) | |
| joint_pos = motion_tables.get("joint_pos.csv") | |
| n_frames = int(len(joint_pos)) if joint_pos is not None else 0 | |
| if n_frames < motion_chunk_frames: | |
| return finish(metadata_rows, 1, 0) | |
| try: | |
| _, _, _, source_video_duration_sec = probe_video(src_video_path) | |
| except Exception as exc: | |
| print(f"[FAILED] {src_video_path}: {exc}") | |
| return metadata_rows, 0, 1 | |
| max_video_start_frame = int( | |
| (source_video_duration_sec - chunk_duration_sec) * float(args.motion_fps) | |
| ) | |
| if max_video_start_frame < 0: | |
| return finish(metadata_rows, 1, 0) | |
| clip_starts = select_clip_starts( | |
| source, | |
| n_frames, | |
| args=args, | |
| motion_chunk_frames=motion_chunk_frames, | |
| max_video_start_frame=max_video_start_frame, | |
| ) | |
| if not clip_starts: | |
| return finish(metadata_rows, 1, 0) | |
| chunk_progress_interval = max(0, int(args.chunk_progress_interval)) | |
| total_chunks = len(clip_starts) | |
| source_name = str(source.get("rel_name") or source.get("name") or source.get("source_id")) | |
| def build_one_chunk(chunk_idx: int, chunk_start_frame: int) -> tuple[dict | None, int]: | |
| row = build_clip_metadata_row( | |
| source, | |
| args=args, | |
| output_root=output_root, | |
| motion_out_root=motion_out_root, | |
| video_out_root=video_out_root, | |
| src_motion_dir=src_motion_dir, | |
| src_video_path=src_video_path, | |
| source_video_duration_sec=source_video_duration_sec, | |
| motion_chunk_frames=motion_chunk_frames, | |
| chunk_duration_sec=chunk_duration_sec, | |
| chunk_idx=chunk_idx, | |
| chunk_start_frame=chunk_start_frame, | |
| ) | |
| if args.metadata_only: | |
| return row, 0 | |
| chunk_start_sec = float(row["chunk_start_sec"]) | |
| dst_motion_dir = output_root / str(row["converted_path"]) | |
| dst_video_path = output_root / str(row["video_path"]) | |
| try: | |
| crop_video( | |
| src_video_path, | |
| dst_video_path, | |
| chunk_start_sec, | |
| chunk_duration_sec, | |
| overwrite=args.overwrite, | |
| existing_video_check=args.existing_video_check, | |
| ffmpeg_threads=args.ffmpeg_threads, | |
| x264_preset=str(args.x264_preset), | |
| ) | |
| if args.overwrite or not motion_clip_ready(dst_motion_dir): | |
| write_motion_clip_atomic( | |
| src_motion_dir, | |
| dst_motion_dir, | |
| chunk_start_frame, | |
| motion_chunk_frames, | |
| motion_tables, | |
| ) | |
| except Exception as exc: | |
| print(f"[FAILED] {src_motion_dir}: {exc}") | |
| return None, 1 | |
| return row, 0 | |
| def note_progress(done_chunks: int) -> None: | |
| if ( | |
| chunk_progress_interval > 0 | |
| and total_chunks >= chunk_progress_interval | |
| and (done_chunks % chunk_progress_interval == 0 or done_chunks == total_chunks) | |
| ): | |
| tqdm.write( | |
| f"[CHUNKS] dataset={args.dataset} shard={args.shard_id}/{args.shard_num} " | |
| f"source={source_name} chunks={done_chunks}/{total_chunks} " | |
| f"saved={len(metadata_rows)} failed={failed}" | |
| ) | |
| chunk_tasks = list(enumerate(clip_starts)) | |
| chunk_workers = max(1, int(args.chunk_workers)) | |
| if chunk_workers == 1 or len(chunk_tasks) <= 1: | |
| for done_chunks, (chunk_idx, chunk_start_frame) in enumerate(chunk_tasks, start=1): | |
| row, failed_count = build_one_chunk(chunk_idx, chunk_start_frame) | |
| failed += failed_count | |
| if row is not None: | |
| metadata_rows.append(row) | |
| note_progress(done_chunks) | |
| else: | |
| with ThreadPoolExecutor(max_workers=chunk_workers) as executor: | |
| futures = [ | |
| executor.submit(build_one_chunk, chunk_idx, chunk_start_frame) | |
| for chunk_idx, chunk_start_frame in chunk_tasks | |
| ] | |
| for done_chunks, future in enumerate(as_completed(futures), start=1): | |
| row, failed_count = future.result() | |
| failed += failed_count | |
| if row is not None: | |
| metadata_rows.append(row) | |
| note_progress(done_chunks) | |
| metadata_rows.sort(key=lambda row: str(row.get("name", ""))) | |
| return finish(metadata_rows, skipped_short, failed) | |
| def main() -> None: | |
| args = parse_args() | |
| if args.clip_frames <= 0: | |
| raise ValueError("--clip-frames must be positive") | |
| if args.action_chunk_frames <= 0: | |
| raise ValueError("--action-chunk-frames must be positive") | |
| if args.chunk_stride_frames <= 0: | |
| raise ValueError("--chunk-stride-frames must be positive") | |
| if args.action_chunk_frames > args.clip_frames: | |
| raise ValueError("--action-chunk-frames must be <= --clip-frames") | |
| if args.motion_fps <= 0: | |
| raise ValueError("--motion-fps must be positive") | |
| if args.clips_per_video <= 0: | |
| raise ValueError("--clips-per-video must be positive") | |
| if not 0.0 <= float(args.middle_fraction) <= 1.0: | |
| raise ValueError("--middle-fraction must be in [0, 1]") | |
| if args.num_workers <= 0: | |
| raise ValueError("--num-workers must be positive") | |
| if args.chunk_workers <= 0: | |
| raise ValueError("--chunk-workers must be positive") | |
| if args.ffmpeg_threads <= 0: | |
| raise ValueError("--ffmpeg-threads must be positive") | |
| if args.chunk_progress_interval < 0: | |
| raise ValueError("--chunk-progress-interval must be non-negative") | |
| if args.recover_existing_only and args.overwrite: | |
| raise ValueError("--recover-existing-only cannot be used with --overwrite") | |
| if args.shard_num <= 0: | |
| raise ValueError("--shard-num must be positive") | |
| if args.shard_id < 0 or args.shard_id >= args.shard_num: | |
| raise ValueError("--shard-id must satisfy 0 <= shard_id < shard_num") | |
| output_root = Path(args.output_root) | |
| motion_out_root = output_root / "motion" | |
| video_out_root = output_root / "videos" | |
| final_metadata_path = output_root / "metadata.jsonl" | |
| metadata_path = ( | |
| shard_metadata_path(output_root, args.shard_id, args.shard_num) | |
| if args.shard_num > 1 | |
| else final_metadata_path | |
| ) | |
| motion_chunk_frames = args.action_chunk_frames + 1 | |
| chunk_duration_sec = float(args.action_chunk_frames) / float(args.motion_fps) | |
| source_iter = iter_dance_rows(args) if args.dataset == "dance" else iter_motion_example_rows(args) | |
| source_rows = list(source_iter) | |
| total_source_rows = len(source_rows) | |
| if args.shard_num > 1: | |
| source_rows = [ | |
| source for source in source_rows | |
| if stable_hash_mod(source_shard_key(source), args.shard_num) == args.shard_id | |
| ] | |
| print( | |
| f"[SHARD] dataset={args.dataset} shard={args.shard_id}/{args.shard_num} " | |
| f"sources={len(source_rows)}/{total_source_rows}" | |
| ) | |
| metadata_rows = [] | |
| skipped_short = 0 | |
| failed = 0 | |
| build_kwargs = { | |
| "args": args, | |
| "motion_out_root": motion_out_root, | |
| "video_out_root": video_out_root, | |
| "output_root": output_root, | |
| "motion_chunk_frames": motion_chunk_frames, | |
| "chunk_duration_sec": chunk_duration_sec, | |
| } | |
| build_fn = build_existing_source_rows if args.recover_existing_only else build_source_clips | |
| progress_prefix = "recover" if args.recover_existing_only else "build" | |
| if args.num_workers == 1: | |
| results = ( | |
| build_fn(source, **build_kwargs) | |
| for source in source_rows | |
| ) | |
| iterator = tqdm(results, total=len(source_rows), desc=f"{progress_prefix} {args.dataset} clips") | |
| for rows, skipped, failed_count in iterator: | |
| metadata_rows.extend(rows) | |
| skipped_short += skipped | |
| failed += failed_count | |
| else: | |
| with ThreadPoolExecutor(max_workers=args.num_workers) as executor: | |
| futures = [ | |
| executor.submit(build_fn, source, **build_kwargs) | |
| for source in source_rows | |
| ] | |
| for future in tqdm( | |
| as_completed(futures), | |
| total=len(futures), | |
| desc=f"{progress_prefix} {args.dataset} clips x{args.num_workers}", | |
| ): | |
| rows, skipped, failed_count = future.result() | |
| metadata_rows.extend(rows) | |
| skipped_short += skipped | |
| failed += failed_count | |
| if args.recover_existing_only: | |
| print( | |
| f"[RECOVER_DONE] dataset={args.dataset} recovered_rows={len(metadata_rows)} " | |
| f"skipped_short={skipped_short} output_root={output_root}" | |
| ) | |
| return | |
| metadata_rows.sort(key=lambda row: str(row.get("name", ""))) | |
| count = write_jsonl(metadata_path, metadata_rows) | |
| print( | |
| f"[DONE] dataset={args.dataset} rows={count} skipped_short={skipped_short} " | |
| f"failed={failed} output_root={output_root}" | |
| ) | |
| if args.shard_num > 1: | |
| merge_shard_metadata(output_root, args.shard_num, final_metadata_path) | |
| if __name__ == "__main__": | |
| main() | |