code-HFD / scripts /data /build_video_clip_dataset.py
Tinghong-Ye22's picture
Add files using upload-large-folder tool
9fc9873 verified
Raw History Blame Contribute Delete
43 kB
#!/usr/bin/env python3 from __future__ import annotations import argparse import hashlib import heapq import json import os import random import shutil import subprocess import uuid from concurrent.futures import ThreadPoolExecutor, as_completed from functools import lru_cache from pathlib import Path from typing import Iterable import pandas as pd from tqdm import tqdm MOTION_CSVS = ( "joint_pos.csv", "joint_vel.csv", "body_pos.csv", "body_quat.csv", "body_lin_vel.csv", "body_ang_vel.csv", ) def ffmpeg_bin() -> str: return os.environ.get("PSI_FFMPEG_BIN", "ffmpeg") def ffprobe_bin() -> str: return os.environ.get("PSI_FFPROBE_BIN", "ffprobe") def parse_args() -> argparse.Namespace: parser = argparse.ArgumentParser(description="Build short video-clip datasets for Psi0 video_clip training.") parser.add_argument( "--dataset", choices=["dance", "motion_example", "manifest"], required=True, help="manifest is the generic path used by 2264/distilled-style datasets.", ) parser.add_argument("--output-root", type=str, required=True) parser.add_argument("--clip-frames", type=int, default=50) parser.add_argument("--action-chunk-frames", type=int, default=30) parser.add_argument("--chunk-stride-frames", type=int, default=20) parser.add_argument("--motion-fps", type=float, default=50.0) parser.add_argument("--clips-per-video", type=int, default=2) parser.add_argument("--clip-selection-mode", choices=["start", "middle_random", "all"], default="middle_random") parser.add_argument("--random-seed", type=int, default=42) parser.add_argument("--middle-fraction", type=float, default=0.8) parser.add_argument("--num-workers", type=int, default=1) parser.add_argument("--chunk-workers", type=int, default=int(os.environ.get("PSI_CHUNK_WORKERS", "1"))) parser.add_argument("--ffmpeg-threads", type=int, default=int(os.environ.get("PSI_FFMPEG_THREADS", "1"))) parser.add_argument("--x264-preset", type=str, default=os.environ.get("PSI_X264_PRESET", "")) parser.add_argument("--chunk-progress-interval", type=int, default=0) parser.add_argument( "--existing-video-check", choices=["ffprobe", "size"], default=os.environ.get("PSI_EXISTING_VIDEO_CHECK", "ffprobe"), help="How to validate already-existing output clips before skipping them.", ) parser.add_argument("--limit", type=int, default=0) parser.add_argument("--overwrite", action="store_true", default=False) parser.add_argument( "--recover-existing-only", action="store_true", default=os.environ.get("RECOVER_EXISTING_ONLY", "0") == "1", help="Only register fully-existing source clips into .source_done; do not create missing clips.", ) parser.add_argument( "--metadata-only", action="store_true", default=os.environ.get("METADATA_ONLY", "0") == "1", help="Write dense clip metadata only. Do not crop mp4 clips or write per-chunk motion CSV directories.", ) parser.add_argument("--dance-motion-root", type=str, default="") parser.add_argument("--dance-video-root", type=str, default="") parser.add_argument("--motion-example-root", type=str, default="") parser.add_argument("--metadata-jsonl", type=str, default="metadata.jsonl") parser.add_argument("--manifest-root", type=str, default="") parser.add_argument( "--motion-metadata-jsonl", type=str, default="", help="Optional second manifest containing converted_path, joined by --metadata-join-key.", ) parser.add_argument("--metadata-join-key", type=str, default="name") parser.add_argument("--motion-dir-key", type=str, default="converted_path") parser.add_argument("--video-path-key", type=str, default="video_path") parser.add_argument("--name-key", type=str, default="name") parser.add_argument("--id-key", type=str, default="id") parser.add_argument("--caption-key", type=str, default="video_caption") parser.add_argument("--motion-type-key", type=str, default="motion_type") parser.add_argument("--motion-type-default", type=str, default="") parser.add_argument( "--rel-name-style", choices=["name", "motion_path"], default="name", help="Output clip stem: source name, or flattened converted-motion path.", ) parser.add_argument( "--use-manifest-duration", action=argparse.BooleanOptionalAction, default=True, help="Use a positive duration field before falling back to ffprobe.", ) parser.add_argument("--shard-num", type=int, default=1) parser.add_argument("--shard-id", type=int, default=0) return parser.parse_args() def read_num_frames(motion_dir: Path) -> int: joint_path = motion_dir / "joint_pos.csv" if not joint_path.exists(): return 0 return int(len(pd.read_csv(joint_path))) def read_motion_tables(motion_dir: Path) -> dict[str, pd.DataFrame]: tables = {} for csv_name in MOTION_CSVS: csv_path = motion_dir / csv_name if csv_path.exists(): tables[csv_name] = pd.read_csv(csv_path) return tables def _parse_rate(value: str) -> float: if not value or value == "0/0": return 0.0 if "/" in value: num, den = value.split("/", 1) den_f = float(den) return float(num) / den_f if den_f else 0.0 return float(value) def probe_video(video_path: Path) -> tuple[int, int, float, float]: cmd = [ ffprobe_bin(), "-v", "error", "-select_streams", "v:0", "-show_entries", "stream=width,height,avg_frame_rate,r_frame_rate,duration:format=duration", "-of", "json", str(video_path), ] proc = subprocess.run(cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE, check=False) if proc.returncode != 0: msg = proc.stderr.decode("utf-8", errors="ignore") raise RuntimeError(f"ffprobe failed for {video_path}: {msg}") info = json.loads(proc.stdout.decode("utf-8")) streams = info.get("streams") or [] if not streams: raise RuntimeError(f"No video stream found by ffprobe: {video_path}") stream = streams[0] width = int(stream.get("width") or 0) height = int(stream.get("height") or 0) fps = _parse_rate(str(stream.get("avg_frame_rate") or stream.get("r_frame_rate") or "0/0")) duration = float(stream.get("duration") or info.get("format", {}).get("duration") or 0.0) if width <= 0 or height <= 0 or fps <= 0 or duration <= 0: raise RuntimeError( f"Bad video metadata at {video_path}: width={width}, height={height}, " f"fps={fps}, duration={duration}" ) return width, height, fps, duration @lru_cache(maxsize=1) def available_ffmpeg_encoders() -> set[str]: proc = subprocess.run( [ffmpeg_bin(), "-hide_banner", "-encoders"], stdout=subprocess.PIPE, stderr=subprocess.PIPE, check=False, ) output = proc.stdout.decode("utf-8", errors="ignore") encoders = set() for line in output.splitlines(): parts = line.split() if len(parts) >= 2 and parts[0].startswith("V"): encoders.add(parts[1]) return encoders def video_encoder_args(x264_preset: str = "") -> list[str]: encoders = available_ffmpeg_encoders() if "libx264" in encoders: args = ["-c:v", "libx264"] if x264_preset: args.extend(["-preset", x264_preset]) args.extend(["-pix_fmt", "yuv420p"]) return args if "h264_nvenc" in encoders: return ["-c:v", "h264_nvenc", "-pix_fmt", "yuv420p"] if "mpeg4" in encoders: return ["-c:v", "mpeg4", "-q:v", "2", "-pix_fmt", "yuv420p"] raise RuntimeError("ffmpeg has no usable video encoder: need libx264, h264_nvenc, or mpeg4") def copy_motion_clip( src_motion_dir: Path, dst_motion_dir: Path, start_frame: int, clip_frames: int, motion_tables: dict[str, pd.DataFrame] | None = None, ) -> None: dst_motion_dir.mkdir(parents=True, exist_ok=True) tables = motion_tables if motion_tables is not None else read_motion_tables(src_motion_dir) for csv_name in MOTION_CSVS: df = tables.get(csv_name) if df is None: continue clip_df = df.iloc[start_frame : start_frame + clip_frames].copy() if len(clip_df) != clip_frames: raise ValueError( f"{src_motion_dir / csv_name} has only {len(clip_df)} rows for requested clip" ) clip_df.to_csv(dst_motion_dir / csv_name, index=False) info_src = src_motion_dir / "info.txt" if info_src.exists(): shutil.copy2(info_src, dst_motion_dir / "info_source.txt") metadata_src = src_motion_dir / "metadata.txt" if metadata_src.exists(): shutil.copy2(metadata_src, dst_motion_dir / "metadata_source.txt") (dst_motion_dir / "metadata.txt").write_text( "\n".join( [ "Motion Information: video_clip_50f", "==================================", "", f"source_motion_dir: {src_motion_dir}", f"source_start_frame: {start_frame}", f"frames: {clip_frames}", "", ] ), encoding="utf-8", ) def crop_video( src_video: Path, dst_video: Path, start_sec: float, duration_sec: float, overwrite: bool, existing_video_check: str, ffmpeg_threads: int, x264_preset: str, ) -> None: if dst_video.exists() and not overwrite: if video_clip_ready(dst_video, duration_sec, existing_video_check): return dst_video.unlink() dst_video.parent.mkdir(parents=True, exist_ok=True) tmp_video = dst_video.with_name(f".{dst_video.name}.{os.getpid()}.{uuid.uuid4().hex}.tmp.mp4") cmd = [ ffmpeg_bin(), "-hide_banner", "-loglevel", "error", "-y", "-ss", f"{start_sec:.6f}", "-t", f"{duration_sec:.6f}", "-i", str(src_video), "-threads", str(max(1, int(ffmpeg_threads))), "-map", "0:v:0", "-an", *video_encoder_args(x264_preset), str(tmp_video), ] try: proc = subprocess.run(cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE, check=False) if proc.returncode != 0: msg = proc.stderr.decode("utf-8", errors="ignore") raise RuntimeError(f"ffmpeg failed for {src_video}: {msg}") if not video_clip_ready(tmp_video, duration_sec, "size"): raise RuntimeError(f"ffmpeg wrote invalid/too-short video for {src_video}") tmp_video.replace(dst_video) except Exception: try: tmp_video.unlink() except FileNotFoundError: pass raise def motion_clip_ready(motion_dir: Path) -> bool: return (motion_dir / "joint_pos.csv").exists() and (motion_dir / "joint_pos.csv").stat().st_size > 0 def video_clip_ready(video_path: Path, expected_duration_sec: float, check_mode: str = "ffprobe") -> bool: if not video_path.exists() or video_path.stat().st_size <= 0: return False if check_mode == "size": return True try: _, _, _, duration_sec = probe_video(video_path) except Exception: return False return duration_sec >= max(0.1, float(expected_duration_sec) * 0.5) def write_motion_clip_atomic( src_motion_dir: Path, dst_motion_dir: Path, start_frame: int, clip_frames: int, motion_tables: dict[str, pd.DataFrame] | None = None, ) -> None: if motion_clip_ready(dst_motion_dir): return dst_motion_dir.parent.mkdir(parents=True, exist_ok=True) tmp_motion_dir = dst_motion_dir.with_name( f".{dst_motion_dir.name}.{os.getpid()}.{uuid.uuid4().hex}.tmp" ) try: if tmp_motion_dir.exists(): shutil.rmtree(tmp_motion_dir) copy_motion_clip(src_motion_dir, tmp_motion_dir, start_frame, clip_frames, motion_tables) if not motion_clip_ready(tmp_motion_dir): raise RuntimeError(f"motion clip was not written correctly: {tmp_motion_dir}") if dst_motion_dir.exists(): if motion_clip_ready(dst_motion_dir): shutil.rmtree(tmp_motion_dir) return shutil.rmtree(dst_motion_dir) try: tmp_motion_dir.replace(dst_motion_dir) except FileExistsError: if motion_clip_ready(dst_motion_dir): shutil.rmtree(tmp_motion_dir) return shutil.rmtree(dst_motion_dir) tmp_motion_dir.replace(dst_motion_dir) except Exception: if tmp_motion_dir.exists(): shutil.rmtree(tmp_motion_dir) raise def source_rng(source_id: str, seed: int) -> random.Random: digest = hashlib.sha1(f"{seed}:{source_id}".encode("utf-8")).hexdigest() return random.Random(int(digest[:16], 16)) def clip_start_candidates( n_frames: int, *, args: argparse.Namespace, motion_chunk_frames: int, max_video_start_frame: int | None = None, ) -> list[int]: max_start_frame = n_frames - motion_chunk_frames if max_video_start_frame is not None: max_start_frame = min(max_start_frame, int(max_video_start_frame)) if max_start_frame < 0: return [] stride = max(1, int(args.chunk_stride_frames)) all_candidates = list(range(0, max_start_frame + 1, stride)) if max_start_frame not in all_candidates: all_candidates.append(max_start_frame) if args.clip_selection_mode == "start": return all_candidates if args.clip_selection_mode == "all": return all_candidates middle_fraction = min(max(float(args.middle_fraction), 0.0), 1.0) if middle_fraction <= 0.0: return all_candidates margin = int(round(max_start_frame * (1.0 - middle_fraction) / 2.0)) lo = min(max_start_frame, max(0, margin)) hi = max(lo, max_start_frame - margin) middle_candidates = [frame for frame in all_candidates if lo <= frame <= hi] return middle_candidates or all_candidates def select_clip_starts( source: dict, n_frames: int, *, args: argparse.Namespace, motion_chunk_frames: int, max_video_start_frame: int | None = None, ) -> list[int]: candidates = clip_start_candidates( n_frames, args=args, motion_chunk_frames=motion_chunk_frames, max_video_start_frame=max_video_start_frame, ) if not candidates: return [] if args.clip_selection_mode == "all": return candidates if args.clip_selection_mode == "start": return candidates[: args.clips_per_video] rng = source_rng(str(source["source_id"]), int(args.random_seed)) if len(candidates) <= args.clips_per_video: selected = candidates else: selected = rng.sample(candidates, args.clips_per_video) return sorted(selected) def write_jsonl(path: Path, rows: Iterable[dict]) -> int: path.parent.mkdir(parents=True, exist_ok=True) count = 0 tmp_path = path.with_name(f".{path.name}.{os.getpid()}.{uuid.uuid4().hex}.tmp") with tmp_path.open("w", encoding="utf-8") as f: for row in rows: f.write(json.dumps(row, ensure_ascii=False) + "\n") count += 1 tmp_path.replace(path) return count def stable_hash_mod(value: str, modulo: int) -> int: digest = hashlib.sha1(value.encode("utf-8")).digest() return int.from_bytes(digest[:8], "big") % int(modulo) def source_shard_key(source: dict) -> str: value = ( source.get("rel_name") or source.get("source_id") or source.get("name") or source.get("video_path") ) return str(value) def safe_source_name(value: str, max_len: int = 96) -> str: safe = "".join(ch if ch.isalnum() or ch in "._-" else "_" for ch in value) safe = safe.strip("._-") return (safe or "source")[:max_len] def source_done_path(output_root: Path, source: dict) -> Path: key = source_shard_key(source) digest = hashlib.sha1(key.encode("utf-8")).hexdigest()[:16] return output_root / ".source_done" / f"{safe_source_name(key)}.{digest}.json" def source_done_signature(args: argparse.Namespace) -> dict: return { "version": 1, "dataset": str(args.dataset), "metadata_only": bool(args.metadata_only), "clip_frames": int(args.clip_frames), "action_chunk_frames": int(args.action_chunk_frames), "motion_chunk_frames": int(args.action_chunk_frames) + 1, "chunk_stride_frames": int(args.chunk_stride_frames), "motion_fps": float(args.motion_fps), "clips_per_video": int(args.clips_per_video), "clip_selection_mode": str(args.clip_selection_mode), "random_seed": int(args.random_seed), "middle_fraction": float(args.middle_fraction), "manifest_root": str(args.manifest_root), "metadata_jsonl": str(args.metadata_jsonl), "motion_metadata_jsonl": str(args.motion_metadata_jsonl), "metadata_join_key": str(args.metadata_join_key), "motion_dir_key": str(args.motion_dir_key), "video_path_key": str(args.video_path_key), "name_key": str(args.name_key), "id_key": str(args.id_key), "caption_key": str(args.caption_key), "motion_type_key": str(args.motion_type_key), "motion_type_default": str(args.motion_type_default), "rel_name_style": str(args.rel_name_style), "use_manifest_duration": bool(args.use_manifest_duration), } def normalize_source_done_signature(signature: dict | None) -> dict: if not isinstance(signature, dict): return {} normalized = dict(signature) for key in ("existing_video_check", "encoder_args", "ffmpeg_threads", "x264_preset"): normalized.pop(key, None) return normalized def build_clip_metadata_row( source: dict, *, args: argparse.Namespace, output_root: Path, motion_out_root: Path, video_out_root: Path, src_motion_dir: Path, src_video_path: Path, source_video_duration_sec: float, motion_chunk_frames: int, chunk_duration_sec: float, chunk_idx: int, chunk_start_frame: int, ) -> dict: chunk_start_sec = float(chunk_start_frame) / float(args.motion_fps) chunk_name = f"{source.get('rel_name', source['name'])}.chunk{chunk_idx:02d}.f{chunk_start_frame:06d}" dst_motion_dir = motion_out_root / chunk_name dst_video_path = video_out_root / f"{chunk_name}.mp4" row = { "id": f"{source['source_id']}.chunk{chunk_idx:02d}.f{chunk_start_frame:06d}", "name": chunk_name, "converted_path": str(dst_motion_dir.relative_to(output_root)), "video_path": str(dst_video_path.relative_to(output_root)), "video_caption": source["caption"], "duration": chunk_duration_sec, "motion_fps": float(args.motion_fps), "clip_frames": int(motion_chunk_frames), "context_frames": int(args.clip_frames), "action_chunk_frames": int(args.action_chunk_frames), "motion_chunk_frames": int(motion_chunk_frames), "chunk_stride_frames": int(args.chunk_stride_frames), "clip_selection_mode": str(args.clip_selection_mode), "random_seed": int(args.random_seed), "middle_fraction": float(args.middle_fraction), "context_start_frame": int(chunk_start_frame), "chunk_start_frame": int(chunk_start_frame), "chunk_start_sec": float(chunk_start_sec), "source_video_duration_sec": float(source_video_duration_sec), "source_motion_dir": str(src_motion_dir), "source_video_path": str(src_video_path), "motion_type": source["motion_type"], } if args.metadata_only: row["metadata_only"] = True return row def read_source_done( output_root: Path, source: dict, args: argparse.Namespace, ) -> tuple[list[dict], int, int] | None: path = source_done_path(output_root, source) if not path.exists(): return None try: payload = json.loads(path.read_text(encoding="utf-8")) except Exception: return None if normalize_source_done_signature(payload.get("signature")) != normalize_source_done_signature(source_done_signature(args)): return None rows = payload.get("rows") if not isinstance(rows, list): return None skipped_short = int(payload.get("skipped_short") or 0) return rows, skipped_short, 0 def write_source_done( output_root: Path, source: dict, args: argparse.Namespace, rows: list[dict], skipped_short: int, ) -> None: path = source_done_path(output_root, source) path.parent.mkdir(parents=True, exist_ok=True) payload = { "signature": source_done_signature(args), "source_key": source_shard_key(source), "source_id": str(source.get("source_id") or ""), "name": str(source.get("name") or ""), "rel_name": str(source.get("rel_name") or ""), "rows": rows, "row_count": len(rows), "skipped_short": int(skipped_short), } tmp_path = path.with_name(f".{path.name}.{os.getpid()}.{uuid.uuid4().hex}.tmp") tmp_path.write_text(json.dumps(payload, ensure_ascii=False), encoding="utf-8") tmp_path.replace(path) def build_existing_source_rows( source: dict, *, args: argparse.Namespace, motion_out_root: Path, video_out_root: Path, output_root: Path, motion_chunk_frames: int, chunk_duration_sec: float, ) -> tuple[list[dict], int, int]: done_result = read_source_done(output_root, source, args) if done_result is not None: return done_result metadata_rows = [] src_motion_dir: Path = source["motion_dir"] src_video_path: Path = source["video_path"] n_frames = read_num_frames(src_motion_dir) if n_frames < motion_chunk_frames: write_source_done(output_root, source, args, metadata_rows, 1) return metadata_rows, 1, 0 try: _, _, _, source_video_duration_sec = probe_video(src_video_path) except Exception as exc: print(f"[RECOVER_MISS] {src_video_path}: {exc}") return metadata_rows, 0, 0 max_video_start_frame = int( (source_video_duration_sec - chunk_duration_sec) * float(args.motion_fps) ) if max_video_start_frame < 0: write_source_done(output_root, source, args, metadata_rows, 1) return metadata_rows, 1, 0 clip_starts = select_clip_starts( source, n_frames, args=args, motion_chunk_frames=motion_chunk_frames, max_video_start_frame=max_video_start_frame, ) if not clip_starts: write_source_done(output_root, source, args, metadata_rows, 1) return metadata_rows, 1, 0 for chunk_idx, chunk_start_frame in enumerate(clip_starts): row = build_clip_metadata_row( source, args=args, output_root=output_root, motion_out_root=motion_out_root, video_out_root=video_out_root, src_motion_dir=src_motion_dir, src_video_path=src_video_path, source_video_duration_sec=source_video_duration_sec, motion_chunk_frames=motion_chunk_frames, chunk_duration_sec=chunk_duration_sec, chunk_idx=chunk_idx, chunk_start_frame=chunk_start_frame, ) dst_motion_dir = output_root / str(row["converted_path"]) dst_video_path = output_root / str(row["video_path"]) if not video_clip_ready(dst_video_path, chunk_duration_sec, "size"): return [], 0, 0 if not motion_clip_ready(dst_motion_dir): return [], 0, 0 metadata_rows.append(row) write_source_done(output_root, source, args, metadata_rows, 0) return metadata_rows, 0, 0 def shard_metadata_path(output_root: Path, shard_id: int, shard_num: int) -> Path: return output_root / f"metadata.shard-{shard_id:05d}-of-{shard_num:05d}.jsonl" def merge_shard_metadata(output_root: Path, shard_num: int, final_path: Path) -> bool: shard_paths = [shard_metadata_path(output_root, shard_id, shard_num) for shard_id in range(shard_num)] missing = [path for path in shard_paths if not path.exists()] if missing: print( f"[SHARD] metadata merge pending: {len(missing)}/{shard_num} shard file(s) missing" ) return False # Every shard is already sorted by name. A streaming k-way merge keeps # memory bounded even for the 14M-row 2264 manifest. handles = [path.open("r", encoding="utf-8") for path in shard_paths] heap: list[tuple[str, int, str]] = [] def push_next(handle_id: int) -> None: for line in handles[handle_id]: if not line.strip(): continue row = json.loads(line) heapq.heappush(heap, (str(row.get("name", "")), handle_id, line)) return tmp_path = final_path.with_name(f".{final_path.name}.{os.getpid()}.{uuid.uuid4().hex}.tmp") count = 0 try: for handle_id in range(len(handles)): push_next(handle_id) with tmp_path.open("w", encoding="utf-8") as output: while heap: _, handle_id, line = heapq.heappop(heap) output.write(line if line.endswith("\n") else line + "\n") count += 1 push_next(handle_id) os.replace(tmp_path, final_path) finally: for handle in handles: handle.close() try: tmp_path.unlink() except FileNotFoundError: pass print(f"[SHARD] merged metadata rows={count} output={final_path}") return True def iter_dance_rows(args: argparse.Namespace): motion_root = Path(args.dance_motion_root) video_root = Path(args.dance_video_root) if not motion_root.exists(): raise FileNotFoundError(f"dance motion root does not exist: {motion_root}") if not video_root.exists(): raise FileNotFoundError(f"dance video root does not exist: {video_root}") motion_dirs = sorted(path for path in motion_root.iterdir() if path.is_dir()) if args.limit > 0: motion_dirs = motion_dirs[: args.limit] for motion_dir in motion_dirs: video_path = video_root / f"{motion_dir.name}.mp4" if video_path.exists(): yield { "name": motion_dir.name, "rel_name": motion_dir.name, "motion_dir": motion_dir, "video_path": video_path, "caption": "Generate a motion sequence that matches the provided video.", "source_id": motion_dir.name, "motion_type": "dance", } def iter_motion_example_rows(args: argparse.Namespace): root = Path(args.motion_example_root) metadata_path = root / args.metadata_jsonl if not metadata_path.exists(): raise FileNotFoundError(f"motion_example metadata does not exist: {metadata_path}") rows = [] with metadata_path.open("r", encoding="utf-8") as f: for line in f: line = line.strip() if line: rows.append(json.loads(line)) if args.limit > 0: rows = rows[: args.limit] for row in rows: motion_value = row.get("converted_path") or row.get("motion_dir") or row.get("motion_path") video_value = row.get("video_path") or row.get("original_video") or row.get("real_path") if not motion_value or not video_value: continue motion_dir = root / str(motion_value) video_path = root / str(video_value) if motion_dir.exists() and video_path.exists(): yield { "name": str(row.get("name") or motion_dir.name), "rel_name": str(Path(str(motion_value))).replace("/", "__"), "motion_dir": motion_dir, "video_path": video_path, "caption": str(row.get("video_caption") or row.get("caption") or ""), "source_id": str(row.get("id") or row.get("name") or motion_dir.name), "motion_type": str(row.get("motion_type") or ""), } def _read_jsonl_rows(path: Path) -> list[dict]: if not path.exists(): raise FileNotFoundError(f"metadata does not exist: {path}") rows: list[dict] = [] with path.open("r", encoding="utf-8") as f: for line_no, line in enumerate(f, start=1): line = line.strip() if not line: continue try: value = json.loads(line) except json.JSONDecodeError as exc: raise ValueError(f"Invalid JSON at {path}:{line_no}") from exc if not isinstance(value, dict): raise ValueError(f"Expected a JSON object at {path}:{line_no}") rows.append(value) return rows def iter_manifest_rows(args: argparse.Namespace): root = Path(args.manifest_root) if not root.exists(): raise FileNotFoundError(f"manifest dataset root does not exist: {root}") metadata_path = Path(args.metadata_jsonl) if not metadata_path.is_absolute(): metadata_path = root / metadata_path rows = _read_jsonl_rows(metadata_path) if args.limit > 0: rows = rows[: args.limit] motion_by_key: dict[str, dict] = {} if str(args.motion_metadata_jsonl).strip(): motion_metadata_path = Path(args.motion_metadata_jsonl) if not motion_metadata_path.is_absolute(): motion_metadata_path = root / motion_metadata_path for row in _read_jsonl_rows(motion_metadata_path): key = str(row.get(args.metadata_join_key) or "").strip() if not key: raise ValueError( f"Missing join key {args.metadata_join_key!r} in {motion_metadata_path}" ) if key in motion_by_key: raise ValueError( f"Duplicate join key {key!r} in {motion_metadata_path}" ) motion_by_key[key] = row default_caption = "Generate a motion sequence that matches the provided video." for row_idx, row in enumerate(rows): join_key = str(row.get(args.metadata_join_key) or "").strip() motion_row = motion_by_key.get(join_key, row) if motion_by_key and join_key not in motion_by_key: continue motion_value = motion_row.get(args.motion_dir_key) or row.get(args.motion_dir_key) video_value = row.get(args.video_path_key) or motion_row.get(args.video_path_key) if not motion_value or not video_value: continue motion_dir = Path(str(motion_value)) video_path = Path(str(video_value)) if not motion_dir.is_absolute(): motion_dir = root / motion_dir if not video_path.is_absolute(): video_path = root / video_path if not motion_dir.exists() or not video_path.exists(): continue name = str(row.get(args.name_key) or motion_row.get(args.name_key) or motion_dir.name) if args.rel_name_style == "motion_path": rel_name = str(Path(str(motion_value))).replace("/", "__") else: rel_name = name caption = str( row.get(args.caption_key) or row.get("video_caption") or row.get("caption") or row.get("instruction") or default_caption ).strip() duration = float(row.get("duration") or row.get("video_duration_sec") or 0.0) yield { "name": name, "rel_name": rel_name, "motion_dir": motion_dir.resolve(), "video_path": video_path.resolve(), "caption": caption or default_caption, "source_id": str(row.get(args.id_key) or motion_row.get(args.id_key) or row_idx), "motion_type": str( row.get(args.motion_type_key) or motion_row.get(args.motion_type_key) or args.motion_type_default or "manifest" ), "video_duration_sec": duration if args.use_manifest_duration and duration > 0 else 0.0, } def build_source_clips( source: dict, *, args: argparse.Namespace, motion_out_root: Path, video_out_root: Path, output_root: Path, motion_chunk_frames: int, chunk_duration_sec: float, ) -> tuple[list[dict], int, int]: done_result = read_source_done(output_root, source, args) if done_result is not None: return done_result metadata_rows = [] skipped_short = 0 failed = 0 def finish(rows: list[dict], skipped: int, failed_count: int) -> tuple[list[dict], int, int]: if failed_count == 0: try: write_source_done(output_root, source, args, rows, skipped) except Exception as exc: print(f"[WARN] source done marker failed for {source_shard_key(source)}: {exc}") return rows, skipped, failed_count src_motion_dir: Path = source["motion_dir"] src_video_path: Path = source["video_path"] motion_tables = read_motion_tables(src_motion_dir) joint_pos = motion_tables.get("joint_pos.csv") n_frames = int(len(joint_pos)) if joint_pos is not None else 0 if n_frames < motion_chunk_frames: return finish(metadata_rows, 1, 0) source_video_duration_sec = float(source.get("video_duration_sec") or 0.0) if source_video_duration_sec <= 0: try: _, _, _, source_video_duration_sec = probe_video(src_video_path) except Exception as exc: print(f"[FAILED] {src_video_path}: {exc}") return metadata_rows, 0, 1 max_video_start_frame = int( (source_video_duration_sec - chunk_duration_sec) * float(args.motion_fps) ) if max_video_start_frame < 0: return finish(metadata_rows, 1, 0) clip_starts = select_clip_starts( source, n_frames, args=args, motion_chunk_frames=motion_chunk_frames, max_video_start_frame=max_video_start_frame, ) if not clip_starts: return finish(metadata_rows, 1, 0) chunk_progress_interval = max(0, int(args.chunk_progress_interval)) total_chunks = len(clip_starts) source_name = str(source.get("rel_name") or source.get("name") or source.get("source_id")) def build_one_chunk(chunk_idx: int, chunk_start_frame: int) -> tuple[dict | None, int]: row = build_clip_metadata_row( source, args=args, output_root=output_root, motion_out_root=motion_out_root, video_out_root=video_out_root, src_motion_dir=src_motion_dir, src_video_path=src_video_path, source_video_duration_sec=source_video_duration_sec, motion_chunk_frames=motion_chunk_frames, chunk_duration_sec=chunk_duration_sec, chunk_idx=chunk_idx, chunk_start_frame=chunk_start_frame, ) if args.metadata_only: return row, 0 chunk_start_sec = float(row["chunk_start_sec"]) dst_motion_dir = output_root / str(row["converted_path"]) dst_video_path = output_root / str(row["video_path"]) try: crop_video( src_video_path, dst_video_path, chunk_start_sec, chunk_duration_sec, overwrite=args.overwrite, existing_video_check=args.existing_video_check, ffmpeg_threads=args.ffmpeg_threads, x264_preset=str(args.x264_preset), ) if args.overwrite or not motion_clip_ready(dst_motion_dir): write_motion_clip_atomic( src_motion_dir, dst_motion_dir, chunk_start_frame, motion_chunk_frames, motion_tables, ) except Exception as exc: print(f"[FAILED] {src_motion_dir}: {exc}") return None, 1 return row, 0 def note_progress(done_chunks: int) -> None: if ( chunk_progress_interval > 0 and total_chunks >= chunk_progress_interval and (done_chunks % chunk_progress_interval == 0 or done_chunks == total_chunks) ): tqdm.write( f"[CHUNKS] dataset={args.dataset} shard={args.shard_id}/{args.shard_num} " f"source={source_name} chunks={done_chunks}/{total_chunks} " f"saved={len(metadata_rows)} failed={failed}" ) chunk_tasks = list(enumerate(clip_starts)) chunk_workers = max(1, int(args.chunk_workers)) if chunk_workers == 1 or len(chunk_tasks) <= 1: for done_chunks, (chunk_idx, chunk_start_frame) in enumerate(chunk_tasks, start=1): row, failed_count = build_one_chunk(chunk_idx, chunk_start_frame) failed += failed_count if row is not None: metadata_rows.append(row) note_progress(done_chunks) else: with ThreadPoolExecutor(max_workers=chunk_workers) as executor: futures = [ executor.submit(build_one_chunk, chunk_idx, chunk_start_frame) for chunk_idx, chunk_start_frame in chunk_tasks ] for done_chunks, future in enumerate(as_completed(futures), start=1): row, failed_count = future.result() failed += failed_count if row is not None: metadata_rows.append(row) note_progress(done_chunks) metadata_rows.sort(key=lambda row: str(row.get("name", ""))) return finish(metadata_rows, skipped_short, failed) def main() -> None: args = parse_args() if args.clip_frames <= 0: raise ValueError("--clip-frames must be positive") if args.action_chunk_frames <= 0: raise ValueError("--action-chunk-frames must be positive") if args.chunk_stride_frames <= 0: raise ValueError("--chunk-stride-frames must be positive") if args.action_chunk_frames > args.clip_frames: raise ValueError("--action-chunk-frames must be <= --clip-frames") if args.motion_fps <= 0: raise ValueError("--motion-fps must be positive") if args.clips_per_video <= 0: raise ValueError("--clips-per-video must be positive") if not 0.0 <= float(args.middle_fraction) <= 1.0: raise ValueError("--middle-fraction must be in [0, 1]") if args.num_workers <= 0: raise ValueError("--num-workers must be positive") if args.chunk_workers <= 0: raise ValueError("--chunk-workers must be positive") if args.ffmpeg_threads <= 0: raise ValueError("--ffmpeg-threads must be positive") if args.chunk_progress_interval < 0: raise ValueError("--chunk-progress-interval must be non-negative") if args.recover_existing_only and args.overwrite: raise ValueError("--recover-existing-only cannot be used with --overwrite") if args.shard_num <= 0: raise ValueError("--shard-num must be positive") if args.shard_id < 0 or args.shard_id >= args.shard_num: raise ValueError("--shard-id must satisfy 0 <= shard_id < shard_num") output_root = Path(args.output_root) motion_out_root = output_root / "motion" video_out_root = output_root / "videos" final_metadata_path = output_root / "metadata.jsonl" metadata_path = ( shard_metadata_path(output_root, args.shard_id, args.shard_num) if args.shard_num > 1 else final_metadata_path ) motion_chunk_frames = args.action_chunk_frames + 1 chunk_duration_sec = float(args.action_chunk_frames) / float(args.motion_fps) if args.dataset == "dance": source_iter = iter_dance_rows(args) elif args.dataset == "motion_example": source_iter = iter_motion_example_rows(args) else: source_iter = iter_manifest_rows(args) source_rows = list(source_iter) total_source_rows = len(source_rows) if args.shard_num > 1: source_rows = [ source for source in source_rows if stable_hash_mod(source_shard_key(source), args.shard_num) == args.shard_id ] print( f"[SHARD] dataset={args.dataset} shard={args.shard_id}/{args.shard_num} " f"sources={len(source_rows)}/{total_source_rows}" ) metadata_rows = [] skipped_short = 0 failed = 0 build_kwargs = { "args": args, "motion_out_root": motion_out_root, "video_out_root": video_out_root, "output_root": output_root, "motion_chunk_frames": motion_chunk_frames, "chunk_duration_sec": chunk_duration_sec, } build_fn = build_existing_source_rows if args.recover_existing_only else build_source_clips progress_prefix = "recover" if args.recover_existing_only else "build" if args.num_workers == 1: results = ( build_fn(source, **build_kwargs) for source in source_rows ) iterator = tqdm(results, total=len(source_rows), desc=f"{progress_prefix} {args.dataset} clips") for rows, skipped, failed_count in iterator: metadata_rows.extend(rows) skipped_short += skipped failed += failed_count else: with ThreadPoolExecutor(max_workers=args.num_workers) as executor: futures = [ executor.submit(build_fn, source, **build_kwargs) for source in source_rows ] for future in tqdm( as_completed(futures), total=len(futures), desc=f"{progress_prefix} {args.dataset} clips x{args.num_workers}", ): rows, skipped, failed_count = future.result() metadata_rows.extend(rows) skipped_short += skipped failed += failed_count if args.recover_existing_only: print( f"[RECOVER_DONE] dataset={args.dataset} recovered_rows={len(metadata_rows)} " f"skipped_short={skipped_short} output_root={output_root}" ) return metadata_rows.sort(key=lambda row: str(row.get("name", ""))) count = write_jsonl(metadata_path, metadata_rows) print( f"[DONE] dataset={args.dataset} rows={count} skipped_short={skipped_short} " f"failed={failed} output_root={output_root}" ) if args.shard_num > 1: merge_shard_metadata(output_root, args.shard_num, final_metadata_path) if __name__ == "__main__": main()
#!/usr/bin/env python3
from __future__ import annotations
import argparse
import hashlib
import heapq
import json
import os
import random
import shutil
import subprocess
import uuid
from concurrent.futures import ThreadPoolExecutor, as_completed
from functools import lru_cache
from pathlib import Path
from typing import Iterable
import pandas as pd
from tqdm import tqdm
MOTION_CSVS = (
"joint_pos.csv",
"joint_vel.csv",
"body_pos.csv",
"body_quat.csv",
"body_lin_vel.csv",
"body_ang_vel.csv",
)
def ffmpeg_bin() -> str:
return os.environ.get("PSI_FFMPEG_BIN", "ffmpeg")
def ffprobe_bin() -> str:
return os.environ.get("PSI_FFPROBE_BIN", "ffprobe")
def parse_args() -> argparse.Namespace:
parser = argparse.ArgumentParser(description="Build short video-clip datasets for Psi0 video_clip training.")
parser.add_argument(
"--dataset",
choices=["dance", "motion_example", "manifest"],
required=True,
help="manifest is the generic path used by 2264/distilled-style datasets.",
)
parser.add_argument("--output-root", type=str, required=True)
parser.add_argument("--clip-frames", type=int, default=50)
parser.add_argument("--action-chunk-frames", type=int, default=30)
parser.add_argument("--chunk-stride-frames", type=int, default=20)
parser.add_argument("--motion-fps", type=float, default=50.0)
parser.add_argument("--clips-per-video", type=int, default=2)
parser.add_argument("--clip-selection-mode", choices=["start", "middle_random", "all"], default="middle_random")
parser.add_argument("--random-seed", type=int, default=42)
parser.add_argument("--middle-fraction", type=float, default=0.8)
parser.add_argument("--num-workers", type=int, default=1)
parser.add_argument("--chunk-workers", type=int, default=int(os.environ.get("PSI_CHUNK_WORKERS", "1")))
parser.add_argument("--ffmpeg-threads", type=int, default=int(os.environ.get("PSI_FFMPEG_THREADS", "1")))
parser.add_argument("--x264-preset", type=str, default=os.environ.get("PSI_X264_PRESET", ""))
parser.add_argument("--chunk-progress-interval", type=int, default=0)
parser.add_argument(
"--existing-video-check",
choices=["ffprobe", "size"],
default=os.environ.get("PSI_EXISTING_VIDEO_CHECK", "ffprobe"),
help="How to validate already-existing output clips before skipping them.",
)
parser.add_argument("--limit", type=int, default=0)
parser.add_argument("--overwrite", action="store_true", default=False)
parser.add_argument(
"--recover-existing-only",
action="store_true",
default=os.environ.get("RECOVER_EXISTING_ONLY", "0") == "1",
help="Only register fully-existing source clips into .source_done; do not create missing clips.",
)
parser.add_argument(
"--metadata-only",
action="store_true",
default=os.environ.get("METADATA_ONLY", "0") == "1",
help="Write dense clip metadata only. Do not crop mp4 clips or write per-chunk motion CSV directories.",
)
parser.add_argument("--dance-motion-root", type=str, default="")
parser.add_argument("--dance-video-root", type=str, default="")
parser.add_argument("--motion-example-root", type=str, default="")
parser.add_argument("--metadata-jsonl", type=str, default="metadata.jsonl")
parser.add_argument("--manifest-root", type=str, default="")
parser.add_argument(
"--motion-metadata-jsonl",
type=str,
default="",
help="Optional second manifest containing converted_path, joined by --metadata-join-key.",
)
parser.add_argument("--metadata-join-key", type=str, default="name")
parser.add_argument("--motion-dir-key", type=str, default="converted_path")
parser.add_argument("--video-path-key", type=str, default="video_path")
parser.add_argument("--name-key", type=str, default="name")
parser.add_argument("--id-key", type=str, default="id")
parser.add_argument("--caption-key", type=str, default="video_caption")