code-Psi0-yth-v21 / scripts /data /build_video_clip_dataset.py
Tinghong-Ye22's picture
Add files using upload-large-folder tool
c4f8e4a verified
Raw History Blame Contribute Delete
35.8 kB
#!/usr/bin/env python3
from __future__ import annotations
import argparse
import hashlib
import json
import os
import random
import shutil
import subprocess
import uuid
from concurrent.futures import ThreadPoolExecutor, as_completed
from functools import lru_cache
from pathlib import Path
from typing import Iterable
import pandas as pd
from tqdm import tqdm
MOTION_CSVS = (
"joint_pos.csv",
"joint_vel.csv",
"body_pos.csv",
"body_quat.csv",
"body_lin_vel.csv",
"body_ang_vel.csv",
)
def ffmpeg_bin() -> str:
return os.environ.get("PSI_FFMPEG_BIN", "ffmpeg")
def ffprobe_bin() -> str:
return os.environ.get("PSI_FFPROBE_BIN", "ffprobe")
def parse_args() -> argparse.Namespace:
parser = argparse.ArgumentParser(description="Build short video-clip datasets for Psi0 video_clip training.")
parser.add_argument("--dataset", choices=["dance", "motion_example"], required=True)
parser.add_argument("--output-root", type=str, required=True)
parser.add_argument("--clip-frames", type=int, default=50)
parser.add_argument("--action-chunk-frames", type=int, default=30)
parser.add_argument("--chunk-stride-frames", type=int, default=20)
parser.add_argument("--motion-fps", type=float, default=50.0)
parser.add_argument("--clips-per-video", type=int, default=2)
parser.add_argument("--clip-selection-mode", choices=["start", "middle_random", "all"], default="middle_random")
parser.add_argument("--random-seed", type=int, default=42)
parser.add_argument("--middle-fraction", type=float, default=0.8)
parser.add_argument("--num-workers", type=int, default=1)
parser.add_argument("--chunk-workers", type=int, default=int(os.environ.get("PSI_CHUNK_WORKERS", "1")))
parser.add_argument("--ffmpeg-threads", type=int, default=int(os.environ.get("PSI_FFMPEG_THREADS", "1")))
parser.add_argument("--x264-preset", type=str, default=os.environ.get("PSI_X264_PRESET", ""))
parser.add_argument("--chunk-progress-interval", type=int, default=0)
parser.add_argument(
"--existing-video-check",
choices=["ffprobe", "size"],
default=os.environ.get("PSI_EXISTING_VIDEO_CHECK", "ffprobe"),
help="How to validate already-existing output clips before skipping them.",
)
parser.add_argument("--limit", type=int, default=0)
parser.add_argument("--overwrite", action="store_true", default=False)
parser.add_argument(
"--recover-existing-only",
action="store_true",
default=os.environ.get("RECOVER_EXISTING_ONLY", "0") == "1",
help="Only register fully-existing source clips into .source_done; do not create missing clips.",
)
parser.add_argument(
"--metadata-only",
action="store_true",
default=os.environ.get("METADATA_ONLY", "0") == "1",
help="Write dense clip metadata only. Do not crop mp4 clips or write per-chunk motion CSV directories.",
)
parser.add_argument("--dance-motion-root", type=str, default="")
parser.add_argument("--dance-video-root", type=str, default="")
parser.add_argument("--motion-example-root", type=str, default="")
parser.add_argument("--metadata-jsonl", type=str, default="metadata.jsonl")
parser.add_argument("--shard-num", type=int, default=1)
parser.add_argument("--shard-id", type=int, default=0)
return parser.parse_args()
def read_num_frames(motion_dir: Path) -> int:
joint_path = motion_dir / "joint_pos.csv"
if not joint_path.exists():
return 0
return int(len(pd.read_csv(joint_path)))
def read_motion_tables(motion_dir: Path) -> dict[str, pd.DataFrame]:
tables = {}
for csv_name in MOTION_CSVS:
csv_path = motion_dir / csv_name
if csv_path.exists():
tables[csv_name] = pd.read_csv(csv_path)
return tables
def _parse_rate(value: str) -> float:
if not value or value == "0/0":
return 0.0
if "/" in value:
num, den = value.split("/", 1)
den_f = float(den)
return float(num) / den_f if den_f else 0.0
return float(value)
def probe_video(video_path: Path) -> tuple[int, int, float, float]:
cmd = [
ffprobe_bin(),
"-v",
"error",
"-select_streams",
"v:0",
"-show_entries",
"stream=width,height,avg_frame_rate,r_frame_rate,duration:format=duration",
"-of",
"json",
str(video_path),
]
proc = subprocess.run(cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE, check=False)
if proc.returncode != 0:
msg = proc.stderr.decode("utf-8", errors="ignore")
raise RuntimeError(f"ffprobe failed for {video_path}: {msg}")
info = json.loads(proc.stdout.decode("utf-8"))
streams = info.get("streams") or []
if not streams:
raise RuntimeError(f"No video stream found by ffprobe: {video_path}")
stream = streams[0]
width = int(stream.get("width") or 0)
height = int(stream.get("height") or 0)
fps = _parse_rate(str(stream.get("avg_frame_rate") or stream.get("r_frame_rate") or "0/0"))
duration = float(stream.get("duration") or info.get("format", {}).get("duration") or 0.0)
if width <= 0 or height <= 0 or fps <= 0 or duration <= 0:
raise RuntimeError(
f"Bad video metadata at {video_path}: width={width}, height={height}, "
f"fps={fps}, duration={duration}"
)
return width, height, fps, duration
@lru_cache(maxsize=1)
def available_ffmpeg_encoders() -> set[str]:
proc = subprocess.run(
[ffmpeg_bin(), "-hide_banner", "-encoders"],
stdout=subprocess.PIPE,
stderr=subprocess.PIPE,
check=False,
)
output = proc.stdout.decode("utf-8", errors="ignore")
encoders = set()
for line in output.splitlines():
parts = line.split()
if len(parts) >= 2 and parts[0].startswith("V"):
encoders.add(parts[1])
return encoders
def video_encoder_args(x264_preset: str = "") -> list[str]:
encoders = available_ffmpeg_encoders()
if "libx264" in encoders:
args = ["-c:v", "libx264"]
if x264_preset:
args.extend(["-preset", x264_preset])
args.extend(["-pix_fmt", "yuv420p"])
return args
if "h264_nvenc" in encoders:
return ["-c:v", "h264_nvenc", "-pix_fmt", "yuv420p"]
if "mpeg4" in encoders:
return ["-c:v", "mpeg4", "-q:v", "2", "-pix_fmt", "yuv420p"]
raise RuntimeError("ffmpeg has no usable video encoder: need libx264, h264_nvenc, or mpeg4")
def copy_motion_clip(
src_motion_dir: Path,
dst_motion_dir: Path,
start_frame: int,
clip_frames: int,
motion_tables: dict[str, pd.DataFrame] | None = None,
) -> None:
dst_motion_dir.mkdir(parents=True, exist_ok=True)
tables = motion_tables if motion_tables is not None else read_motion_tables(src_motion_dir)
for csv_name in MOTION_CSVS:
df = tables.get(csv_name)
if df is None:
continue
clip_df = df.iloc[start_frame : start_frame + clip_frames].copy()
if len(clip_df) != clip_frames:
raise ValueError(
f"{src_motion_dir / csv_name} has only {len(clip_df)} rows for requested clip"
)
clip_df.to_csv(dst_motion_dir / csv_name, index=False)
info_src = src_motion_dir / "info.txt"
if info_src.exists():
shutil.copy2(info_src, dst_motion_dir / "info_source.txt")
metadata_src = src_motion_dir / "metadata.txt"
if metadata_src.exists():
shutil.copy2(metadata_src, dst_motion_dir / "metadata_source.txt")
(dst_motion_dir / "metadata.txt").write_text(
"\n".join(
[
"Motion Information: video_clip_50f",
"==================================",
"",
f"source_motion_dir: {src_motion_dir}",
f"source_start_frame: {start_frame}",
f"frames: {clip_frames}",
"",
]
),
encoding="utf-8",
)
def crop_video(
src_video: Path,
dst_video: Path,
start_sec: float,
duration_sec: float,
overwrite: bool,
existing_video_check: str,
ffmpeg_threads: int,
x264_preset: str,
) -> None:
if dst_video.exists() and not overwrite:
if video_clip_ready(dst_video, duration_sec, existing_video_check):
return
dst_video.unlink()
dst_video.parent.mkdir(parents=True, exist_ok=True)
tmp_video = dst_video.with_name(f".{dst_video.name}.{os.getpid()}.{uuid.uuid4().hex}.tmp.mp4")
cmd = [
ffmpeg_bin(),
"-hide_banner",
"-loglevel",
"error",
"-y",
"-ss",
f"{start_sec:.6f}",
"-t",
f"{duration_sec:.6f}",
"-i",
str(src_video),
"-threads",
str(max(1, int(ffmpeg_threads))),
"-map",
"0:v:0",
"-an",
*video_encoder_args(x264_preset),
str(tmp_video),
]
try:
proc = subprocess.run(cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE, check=False)
if proc.returncode != 0:
msg = proc.stderr.decode("utf-8", errors="ignore")
raise RuntimeError(f"ffmpeg failed for {src_video}: {msg}")
if not video_clip_ready(tmp_video, duration_sec, "size"):
raise RuntimeError(f"ffmpeg wrote invalid/too-short video for {src_video}")
tmp_video.replace(dst_video)
except Exception:
try:
tmp_video.unlink()
except FileNotFoundError:
pass
raise
def motion_clip_ready(motion_dir: Path) -> bool:
return (motion_dir / "joint_pos.csv").exists() and (motion_dir / "joint_pos.csv").stat().st_size > 0
def video_clip_ready(video_path: Path, expected_duration_sec: float, check_mode: str = "ffprobe") -> bool:
if not video_path.exists() or video_path.stat().st_size <= 0:
return False
if check_mode == "size":
return True
try:
_, _, _, duration_sec = probe_video(video_path)
except Exception:
return False
return duration_sec >= max(0.1, float(expected_duration_sec) * 0.5)
def write_motion_clip_atomic(
src_motion_dir: Path,
dst_motion_dir: Path,
start_frame: int,
clip_frames: int,
motion_tables: dict[str, pd.DataFrame] | None = None,
) -> None:
if motion_clip_ready(dst_motion_dir):
return
dst_motion_dir.parent.mkdir(parents=True, exist_ok=True)
tmp_motion_dir = dst_motion_dir.with_name(
f".{dst_motion_dir.name}.{os.getpid()}.{uuid.uuid4().hex}.tmp"
)
try:
if tmp_motion_dir.exists():
shutil.rmtree(tmp_motion_dir)
copy_motion_clip(src_motion_dir, tmp_motion_dir, start_frame, clip_frames, motion_tables)
if not motion_clip_ready(tmp_motion_dir):
raise RuntimeError(f"motion clip was not written correctly: {tmp_motion_dir}")
if dst_motion_dir.exists():
if motion_clip_ready(dst_motion_dir):
shutil.rmtree(tmp_motion_dir)
return
shutil.rmtree(dst_motion_dir)
try:
tmp_motion_dir.replace(dst_motion_dir)
except FileExistsError:
if motion_clip_ready(dst_motion_dir):
shutil.rmtree(tmp_motion_dir)
return
shutil.rmtree(dst_motion_dir)
tmp_motion_dir.replace(dst_motion_dir)
except Exception:
if tmp_motion_dir.exists():
shutil.rmtree(tmp_motion_dir)
raise
def source_rng(source_id: str, seed: int) -> random.Random:
digest = hashlib.sha1(f"{seed}:{source_id}".encode("utf-8")).hexdigest()
return random.Random(int(digest[:16], 16))
def clip_start_candidates(
n_frames: int,
*,
args: argparse.Namespace,
motion_chunk_frames: int,
max_video_start_frame: int | None = None,
) -> list[int]:
max_start_frame = n_frames - motion_chunk_frames
if max_video_start_frame is not None:
max_start_frame = min(max_start_frame, int(max_video_start_frame))
if max_start_frame < 0:
return []
stride = max(1, int(args.chunk_stride_frames))
all_candidates = list(range(0, max_start_frame + 1, stride))
if max_start_frame not in all_candidates:
all_candidates.append(max_start_frame)
if args.clip_selection_mode == "start":
return all_candidates
if args.clip_selection_mode == "all":
return all_candidates
middle_fraction = min(max(float(args.middle_fraction), 0.0), 1.0)
if middle_fraction <= 0.0:
return all_candidates
margin = int(round(max_start_frame * (1.0 - middle_fraction) / 2.0))
lo = min(max_start_frame, max(0, margin))
hi = max(lo, max_start_frame - margin)
middle_candidates = [frame for frame in all_candidates if lo <= frame <= hi]
return middle_candidates or all_candidates
def select_clip_starts(
source: dict,
n_frames: int,
*,
args: argparse.Namespace,
motion_chunk_frames: int,
max_video_start_frame: int | None = None,
) -> list[int]:
candidates = clip_start_candidates(
n_frames,
args=args,
motion_chunk_frames=motion_chunk_frames,
max_video_start_frame=max_video_start_frame,
)
if not candidates:
return []
if args.clip_selection_mode == "all":
return candidates
if args.clip_selection_mode == "start":
return candidates[: args.clips_per_video]
rng = source_rng(str(source["source_id"]), int(args.random_seed))
if len(candidates) <= args.clips_per_video:
selected = candidates
else:
selected = rng.sample(candidates, args.clips_per_video)
return sorted(selected)
def write_jsonl(path: Path, rows: Iterable[dict]) -> int:
path.parent.mkdir(parents=True, exist_ok=True)
count = 0
tmp_path = path.with_name(f".{path.name}.{os.getpid()}.{uuid.uuid4().hex}.tmp")
with tmp_path.open("w", encoding="utf-8") as f:
for row in rows:
f.write(json.dumps(row, ensure_ascii=False) + "\n")
count += 1
tmp_path.replace(path)
return count
def stable_hash_mod(value: str, modulo: int) -> int:
digest = hashlib.sha1(value.encode("utf-8")).digest()
return int.from_bytes(digest[:8], "big") % int(modulo)
def source_shard_key(source: dict) -> str:
value = (
source.get("rel_name")
or source.get("source_id")
or source.get("name")
or source.get("video_path")
)
return str(value)
def safe_source_name(value: str, max_len: int = 96) -> str:
safe = "".join(ch if ch.isalnum() or ch in "._-" else "_" for ch in value)
safe = safe.strip("._-")
return (safe or "source")[:max_len]
def source_done_path(output_root: Path, source: dict) -> Path:
key = source_shard_key(source)
digest = hashlib.sha1(key.encode("utf-8")).hexdigest()[:16]
return output_root / ".source_done" / f"{safe_source_name(key)}.{digest}.json"
def source_done_signature(args: argparse.Namespace) -> dict:
return {
"version": 1,
"dataset": str(args.dataset),
"metadata_only": bool(args.metadata_only),
"clip_frames": int(args.clip_frames),
"action_chunk_frames": int(args.action_chunk_frames),
"motion_chunk_frames": int(args.action_chunk_frames) + 1,
"chunk_stride_frames": int(args.chunk_stride_frames),
"motion_fps": float(args.motion_fps),
"clips_per_video": int(args.clips_per_video),
"clip_selection_mode": str(args.clip_selection_mode),
"random_seed": int(args.random_seed),
"middle_fraction": float(args.middle_fraction),
}
def normalize_source_done_signature(signature: dict | None) -> dict:
if not isinstance(signature, dict):
return {}
normalized = dict(signature)
for key in ("existing_video_check", "encoder_args", "ffmpeg_threads", "x264_preset"):
normalized.pop(key, None)
return normalized
def build_clip_metadata_row(
source: dict,
*,
args: argparse.Namespace,
output_root: Path,
motion_out_root: Path,
video_out_root: Path,
src_motion_dir: Path,
src_video_path: Path,
source_video_duration_sec: float,
motion_chunk_frames: int,
chunk_duration_sec: float,
chunk_idx: int,
chunk_start_frame: int,
) -> dict:
chunk_start_sec = float(chunk_start_frame) / float(args.motion_fps)
chunk_name = f"{source.get('rel_name', source['name'])}.chunk{chunk_idx:02d}.f{chunk_start_frame:06d}"
dst_motion_dir = motion_out_root / chunk_name
dst_video_path = video_out_root / f"{chunk_name}.mp4"
row = {
"id": f"{source['source_id']}.chunk{chunk_idx:02d}.f{chunk_start_frame:06d}",
"name": chunk_name,
"converted_path": str(dst_motion_dir.relative_to(output_root)),
"video_path": str(dst_video_path.relative_to(output_root)),
"video_caption": source["caption"],
"duration": chunk_duration_sec,
"motion_fps": float(args.motion_fps),
"clip_frames": int(motion_chunk_frames),
"context_frames": int(args.clip_frames),
"action_chunk_frames": int(args.action_chunk_frames),
"motion_chunk_frames": int(motion_chunk_frames),
"chunk_stride_frames": int(args.chunk_stride_frames),
"clip_selection_mode": str(args.clip_selection_mode),
"random_seed": int(args.random_seed),
"middle_fraction": float(args.middle_fraction),
"context_start_frame": int(chunk_start_frame),
"chunk_start_frame": int(chunk_start_frame),
"chunk_start_sec": float(chunk_start_sec),
"source_video_duration_sec": float(source_video_duration_sec),
"source_motion_dir": str(src_motion_dir),
"source_video_path": str(src_video_path),
"motion_type": source["motion_type"],
}
if args.metadata_only:
row["metadata_only"] = True
return row
def read_source_done(
output_root: Path,
source: dict,
args: argparse.Namespace,
) -> tuple[list[dict], int, int] | None:
path = source_done_path(output_root, source)
if not path.exists():
return None
try:
payload = json.loads(path.read_text(encoding="utf-8"))
except Exception:
return None
if normalize_source_done_signature(payload.get("signature")) != normalize_source_done_signature(source_done_signature(args)):
return None
rows = payload.get("rows")
if not isinstance(rows, list):
return None
skipped_short = int(payload.get("skipped_short") or 0)
return rows, skipped_short, 0
def write_source_done(
output_root: Path,
source: dict,
args: argparse.Namespace,
rows: list[dict],
skipped_short: int,
) -> None:
path = source_done_path(output_root, source)
path.parent.mkdir(parents=True, exist_ok=True)
payload = {
"signature": source_done_signature(args),
"source_key": source_shard_key(source),
"source_id": str(source.get("source_id") or ""),
"name": str(source.get("name") or ""),
"rel_name": str(source.get("rel_name") or ""),
"rows": rows,
"row_count": len(rows),
"skipped_short": int(skipped_short),
}
tmp_path = path.with_name(f".{path.name}.{os.getpid()}.{uuid.uuid4().hex}.tmp")
tmp_path.write_text(json.dumps(payload, ensure_ascii=False), encoding="utf-8")
tmp_path.replace(path)
def build_existing_source_rows(
source: dict,
*,
args: argparse.Namespace,
motion_out_root: Path,
video_out_root: Path,
output_root: Path,
motion_chunk_frames: int,
chunk_duration_sec: float,
) -> tuple[list[dict], int, int]:
done_result = read_source_done(output_root, source, args)
if done_result is not None:
return done_result
metadata_rows = []
src_motion_dir: Path = source["motion_dir"]
src_video_path: Path = source["video_path"]
n_frames = read_num_frames(src_motion_dir)
if n_frames < motion_chunk_frames:
write_source_done(output_root, source, args, metadata_rows, 1)
return metadata_rows, 1, 0
try:
_, _, _, source_video_duration_sec = probe_video(src_video_path)
except Exception as exc:
print(f"[RECOVER_MISS] {src_video_path}: {exc}")
return metadata_rows, 0, 0
max_video_start_frame = int(
(source_video_duration_sec - chunk_duration_sec) * float(args.motion_fps)
)
if max_video_start_frame < 0:
write_source_done(output_root, source, args, metadata_rows, 1)
return metadata_rows, 1, 0
clip_starts = select_clip_starts(
source,
n_frames,
args=args,
motion_chunk_frames=motion_chunk_frames,
max_video_start_frame=max_video_start_frame,
)
if not clip_starts:
write_source_done(output_root, source, args, metadata_rows, 1)
return metadata_rows, 1, 0
for chunk_idx, chunk_start_frame in enumerate(clip_starts):
row = build_clip_metadata_row(
source,
args=args,
output_root=output_root,
motion_out_root=motion_out_root,
video_out_root=video_out_root,
src_motion_dir=src_motion_dir,
src_video_path=src_video_path,
source_video_duration_sec=source_video_duration_sec,
motion_chunk_frames=motion_chunk_frames,
chunk_duration_sec=chunk_duration_sec,
chunk_idx=chunk_idx,
chunk_start_frame=chunk_start_frame,
)
dst_motion_dir = output_root / str(row["converted_path"])
dst_video_path = output_root / str(row["video_path"])
if not video_clip_ready(dst_video_path, chunk_duration_sec, "size"):
return [], 0, 0
if not motion_clip_ready(dst_motion_dir):
return [], 0, 0
metadata_rows.append(row)
write_source_done(output_root, source, args, metadata_rows, 0)
return metadata_rows, 0, 0
def shard_metadata_path(output_root: Path, shard_id: int, shard_num: int) -> Path:
return output_root / f"metadata.shard-{shard_id:05d}-of-{shard_num:05d}.jsonl"
def merge_shard_metadata(output_root: Path, shard_num: int, final_path: Path) -> bool:
shard_paths = [shard_metadata_path(output_root, shard_id, shard_num) for shard_id in range(shard_num)]
missing = [path for path in shard_paths if not path.exists()]
if missing:
print(
f"[SHARD] metadata merge pending: {len(missing)}/{shard_num} shard file(s) missing"
)
return False
rows = []
for path in shard_paths:
with path.open("r", encoding="utf-8") as f:
for line in f:
line = line.strip()
if line:
rows.append(json.loads(line))
rows.sort(key=lambda row: str(row.get("name", "")))
count = write_jsonl(final_path, rows)
print(f"[SHARD] merged metadata rows={count} output={final_path}")
return True
def iter_dance_rows(args: argparse.Namespace):
motion_root = Path(args.dance_motion_root)
video_root = Path(args.dance_video_root)
if not motion_root.exists():
raise FileNotFoundError(f"dance motion root does not exist: {motion_root}")
if not video_root.exists():
raise FileNotFoundError(f"dance video root does not exist: {video_root}")
motion_dirs = sorted(path for path in motion_root.iterdir() if path.is_dir())
if args.limit > 0:
motion_dirs = motion_dirs[: args.limit]
for motion_dir in motion_dirs:
video_path = video_root / f"{motion_dir.name}.mp4"
if video_path.exists():
yield {
"name": motion_dir.name,
"rel_name": motion_dir.name,
"motion_dir": motion_dir,
"video_path": video_path,
"caption": "Generate a motion sequence that matches the provided video.",
"source_id": motion_dir.name,
"motion_type": "dance",
}
def iter_motion_example_rows(args: argparse.Namespace):
root = Path(args.motion_example_root)
metadata_path = root / args.metadata_jsonl
if not metadata_path.exists():
raise FileNotFoundError(f"motion_example metadata does not exist: {metadata_path}")
rows = []
with metadata_path.open("r", encoding="utf-8") as f:
for line in f:
line = line.strip()
if line:
rows.append(json.loads(line))
if args.limit > 0:
rows = rows[: args.limit]
for row in rows:
motion_value = row.get("converted_path") or row.get("motion_dir") or row.get("motion_path")
video_value = row.get("video_path") or row.get("original_video") or row.get("real_path")
if not motion_value or not video_value:
continue
motion_dir = root / str(motion_value)
video_path = root / str(video_value)
if motion_dir.exists() and video_path.exists():
yield {
"name": str(row.get("name") or motion_dir.name),
"rel_name": str(Path(str(motion_value))).replace("/", "__"),
"motion_dir": motion_dir,
"video_path": video_path,
"caption": str(row.get("video_caption") or row.get("caption") or ""),
"source_id": str(row.get("id") or row.get("name") or motion_dir.name),
"motion_type": str(row.get("motion_type") or ""),
}
def build_source_clips(
source: dict,
*,
args: argparse.Namespace,
motion_out_root: Path,
video_out_root: Path,
output_root: Path,
motion_chunk_frames: int,
chunk_duration_sec: float,
) -> tuple[list[dict], int, int]:
done_result = read_source_done(output_root, source, args)
if done_result is not None:
return done_result
metadata_rows = []
skipped_short = 0
failed = 0
def finish(rows: list[dict], skipped: int, failed_count: int) -> tuple[list[dict], int, int]:
if failed_count == 0:
try:
write_source_done(output_root, source, args, rows, skipped)
except Exception as exc:
print(f"[WARN] source done marker failed for {source_shard_key(source)}: {exc}")
return rows, skipped, failed_count
src_motion_dir: Path = source["motion_dir"]
src_video_path: Path = source["video_path"]
motion_tables = read_motion_tables(src_motion_dir)
joint_pos = motion_tables.get("joint_pos.csv")
n_frames = int(len(joint_pos)) if joint_pos is not None else 0
if n_frames < motion_chunk_frames:
return finish(metadata_rows, 1, 0)
try:
_, _, _, source_video_duration_sec = probe_video(src_video_path)
except Exception as exc:
print(f"[FAILED] {src_video_path}: {exc}")
return metadata_rows, 0, 1
max_video_start_frame = int(
(source_video_duration_sec - chunk_duration_sec) * float(args.motion_fps)
)
if max_video_start_frame < 0:
return finish(metadata_rows, 1, 0)
clip_starts = select_clip_starts(
source,
n_frames,
args=args,
motion_chunk_frames=motion_chunk_frames,
max_video_start_frame=max_video_start_frame,
)
if not clip_starts:
return finish(metadata_rows, 1, 0)
chunk_progress_interval = max(0, int(args.chunk_progress_interval))
total_chunks = len(clip_starts)
source_name = str(source.get("rel_name") or source.get("name") or source.get("source_id"))
def build_one_chunk(chunk_idx: int, chunk_start_frame: int) -> tuple[dict | None, int]:
row = build_clip_metadata_row(
source,
args=args,
output_root=output_root,
motion_out_root=motion_out_root,
video_out_root=video_out_root,
src_motion_dir=src_motion_dir,
src_video_path=src_video_path,
source_video_duration_sec=source_video_duration_sec,
motion_chunk_frames=motion_chunk_frames,
chunk_duration_sec=chunk_duration_sec,
chunk_idx=chunk_idx,
chunk_start_frame=chunk_start_frame,
)
if args.metadata_only:
return row, 0
chunk_start_sec = float(row["chunk_start_sec"])
dst_motion_dir = output_root / str(row["converted_path"])
dst_video_path = output_root / str(row["video_path"])
try:
crop_video(
src_video_path,
dst_video_path,
chunk_start_sec,
chunk_duration_sec,
overwrite=args.overwrite,
existing_video_check=args.existing_video_check,
ffmpeg_threads=args.ffmpeg_threads,
x264_preset=str(args.x264_preset),
)
if args.overwrite or not motion_clip_ready(dst_motion_dir):
write_motion_clip_atomic(
src_motion_dir,
dst_motion_dir,
chunk_start_frame,
motion_chunk_frames,
motion_tables,
)
except Exception as exc:
print(f"[FAILED] {src_motion_dir}: {exc}")
return None, 1
return row, 0
def note_progress(done_chunks: int) -> None:
if (
chunk_progress_interval > 0
and total_chunks >= chunk_progress_interval
and (done_chunks % chunk_progress_interval == 0 or done_chunks == total_chunks)
):
tqdm.write(
f"[CHUNKS] dataset={args.dataset} shard={args.shard_id}/{args.shard_num} "
f"source={source_name} chunks={done_chunks}/{total_chunks} "
f"saved={len(metadata_rows)} failed={failed}"
)
chunk_tasks = list(enumerate(clip_starts))
chunk_workers = max(1, int(args.chunk_workers))
if chunk_workers == 1 or len(chunk_tasks) <= 1:
for done_chunks, (chunk_idx, chunk_start_frame) in enumerate(chunk_tasks, start=1):
row, failed_count = build_one_chunk(chunk_idx, chunk_start_frame)
failed += failed_count
if row is not None:
metadata_rows.append(row)
note_progress(done_chunks)
else:
with ThreadPoolExecutor(max_workers=chunk_workers) as executor:
futures = [
executor.submit(build_one_chunk, chunk_idx, chunk_start_frame)
for chunk_idx, chunk_start_frame in chunk_tasks
]
for done_chunks, future in enumerate(as_completed(futures), start=1):
row, failed_count = future.result()
failed += failed_count
if row is not None:
metadata_rows.append(row)
note_progress(done_chunks)
metadata_rows.sort(key=lambda row: str(row.get("name", "")))
return finish(metadata_rows, skipped_short, failed)
def main() -> None:
args = parse_args()
if args.clip_frames <= 0:
raise ValueError("--clip-frames must be positive")
if args.action_chunk_frames <= 0:
raise ValueError("--action-chunk-frames must be positive")
if args.chunk_stride_frames <= 0:
raise ValueError("--chunk-stride-frames must be positive")
if args.action_chunk_frames > args.clip_frames:
raise ValueError("--action-chunk-frames must be <= --clip-frames")
if args.motion_fps <= 0:
raise ValueError("--motion-fps must be positive")
if args.clips_per_video <= 0:
raise ValueError("--clips-per-video must be positive")
if not 0.0 <= float(args.middle_fraction) <= 1.0:
raise ValueError("--middle-fraction must be in [0, 1]")
if args.num_workers <= 0:
raise ValueError("--num-workers must be positive")
if args.chunk_workers <= 0:
raise ValueError("--chunk-workers must be positive")
if args.ffmpeg_threads <= 0:
raise ValueError("--ffmpeg-threads must be positive")
if args.chunk_progress_interval < 0:
raise ValueError("--chunk-progress-interval must be non-negative")
if args.recover_existing_only and args.overwrite:
raise ValueError("--recover-existing-only cannot be used with --overwrite")
if args.shard_num <= 0:
raise ValueError("--shard-num must be positive")
if args.shard_id < 0 or args.shard_id >= args.shard_num:
raise ValueError("--shard-id must satisfy 0 <= shard_id < shard_num")
output_root = Path(args.output_root)
motion_out_root = output_root / "motion"
video_out_root = output_root / "videos"
final_metadata_path = output_root / "metadata.jsonl"
metadata_path = (
shard_metadata_path(output_root, args.shard_id, args.shard_num)
if args.shard_num > 1
else final_metadata_path
)
motion_chunk_frames = args.action_chunk_frames + 1
chunk_duration_sec = float(args.action_chunk_frames) / float(args.motion_fps)
source_iter = iter_dance_rows(args) if args.dataset == "dance" else iter_motion_example_rows(args)
source_rows = list(source_iter)
total_source_rows = len(source_rows)
if args.shard_num > 1:
source_rows = [
source for source in source_rows
if stable_hash_mod(source_shard_key(source), args.shard_num) == args.shard_id
]
print(
f"[SHARD] dataset={args.dataset} shard={args.shard_id}/{args.shard_num} "
f"sources={len(source_rows)}/{total_source_rows}"
)
metadata_rows = []
skipped_short = 0
failed = 0
build_kwargs = {
"args": args,
"motion_out_root": motion_out_root,
"video_out_root": video_out_root,
"output_root": output_root,
"motion_chunk_frames": motion_chunk_frames,
"chunk_duration_sec": chunk_duration_sec,
}
build_fn = build_existing_source_rows if args.recover_existing_only else build_source_clips
progress_prefix = "recover" if args.recover_existing_only else "build"
if args.num_workers == 1:
results = (
build_fn(source, **build_kwargs)
for source in source_rows
)
iterator = tqdm(results, total=len(source_rows), desc=f"{progress_prefix} {args.dataset} clips")
for rows, skipped, failed_count in iterator:
metadata_rows.extend(rows)
skipped_short += skipped
failed += failed_count
else:
with ThreadPoolExecutor(max_workers=args.num_workers) as executor:
futures = [
executor.submit(build_fn, source, **build_kwargs)
for source in source_rows
]
for future in tqdm(
as_completed(futures),
total=len(futures),
desc=f"{progress_prefix} {args.dataset} clips x{args.num_workers}",
):
rows, skipped, failed_count = future.result()
metadata_rows.extend(rows)
skipped_short += skipped
failed += failed_count
if args.recover_existing_only:
print(
f"[RECOVER_DONE] dataset={args.dataset} recovered_rows={len(metadata_rows)} "
f"skipped_short={skipped_short} output_root={output_root}"
)
return
metadata_rows.sort(key=lambda row: str(row.get("name", "")))
count = write_jsonl(metadata_path, metadata_rows)
print(
f"[DONE] dataset={args.dataset} rows={count} skipped_short={skipped_short} "
f"failed={failed} output_root={output_root}"
)
if args.shard_num > 1:
merge_shard_metadata(output_root, args.shard_num, final_metadata_path)
if __name__ == "__main__":
main()