#!/usr/bin/env python3 from __future__ import annotations import argparse import json import math import shutil from pathlib import Path from typing import List import numpy as np import torch from psi.config.config import LaunchConfig from psi.data.seed.motion_repr import quat_inv_wxyz, quat_mul_wxyz, recover_seed_abs_from_textop_action from psi.data.seed.progress import build_progress_conditions from psi.data.seed.seed_video_motion_dataset import ( _probe_video, load_video_array, load_video_clip_array, load_video_embed_cache, video_embed_cache_path, ) from psi.models.motion_distill import MotionDistillPsi0Model from psi.models.psi0 import Psi0Model from psi.utils import parse_args_to_tyro_config, seed_everything from seed_audio_to_motion import resolve_motion_dir, resolve_video_path from seed_text_to_motion import ( TARGET_FPS, build_model_states_from_abs, finite_difference, load_pose_from_motion_dir, load_source_frame_count, normalize_quaternion_rows, quaternion_angular_velocity, recover_seed_abs_from_textop_action_with_joint_delta, resolve_ckpt_step, sanitize_motion_name, should_decode_textop_joint_from_delta, write_csv, write_metadata, ) def cfg_get(obj, key: str, default=None): if isinstance(obj, dict): return obj.get(key, default) return getattr(obj, key, default) def video_data_cfg(launch_config: LaunchConfig, config_key: str = "video"): return cfg_get(launch_config.data, config_key, cfg_get(launch_config.data, "video", launch_config.data)) def rollout_video_motion( *, model: Psi0Model, maxmin, odim: int, device: str, video: np.ndarray | None, video_embed: dict | None, video_path: Path, instruction: str, video_size_pixels: int, video_sample_fps: float, video_decoder: str, video_clip_mode: bool, video_clip_motion_fps: float, normalize_state: bool, textop_actions: bool, decode_textop_joint_from_delta: bool, num_past_frames: int, use_future_progress_embedding: bool, enable_rtc: bool, action_exec_horizon: int, rtc_max_delay: int, num_inference_steps: int, target_frames: int, init_abs_state: np.ndarray, ) -> tuple[np.ndarray, np.ndarray, int]: horizon = int(model.action_horizon) exec_horizon = int(action_exec_horizon) if enable_rtc else horizon if exec_horizon <= 0 or exec_horizon > horizon: raise ValueError(f"Invalid action_exec_horizon={exec_horizon}, action_horizon={horizon}") rtc_delay = horizon - exec_horizon if enable_rtc and rtc_delay > 0 and rtc_delay >= int(rtc_max_delay): raise ValueError( f"RTC requires action_horizon - action_exec_horizon < max_delay, " f"got {horizon} - {exec_horizon} = {rtc_delay}, max_delay={rtc_max_delay}" ) if not textop_actions: raise ValueError("This video inference path is for the 39D TextOp action checkpoint.") if target_frames <= 0: target_frames = exec_horizon num_chunks = int(math.ceil(float(target_frames) / float(exec_horizon))) current_abs = np.asarray(init_abs_state, dtype=np.float32).copy() state_horizon = int(num_past_frames) + 1 state_history: List[np.ndarray] = [current_abs.copy() for _ in range(state_horizon)] all_abs: List[np.ndarray] = [] all_delta: List[np.ndarray] = [] prev_actions_norm: np.ndarray | None = None remaining_frames = int(target_frames) for _ in range(num_chunks): if video_clip_mode: chunk_index = len(all_abs) video_clip_start_sec = float(chunk_index * exec_horizon) / float(video_clip_motion_fps) video_clip_duration_sec = float(horizon) / float(video_clip_motion_fps) chunk_video, _, _ = load_video_clip_array( video_path, start_sec=video_clip_start_sec, duration_sec=video_clip_duration_sec, video_sample_fps=video_sample_fps, video_size_pixels=video_size_pixels, video_decoder=video_decoder, ) chunk_video_embed = None else: chunk_video = video chunk_video_embed = video_embed frame_cursor = target_frames - remaining_frames states = build_model_states_from_abs( state_history[-state_horizon:], textop_actions=textop_actions, odim=odim, ) if normalize_state: states = maxmin.normalize_state_func(states) states_t = torch.from_numpy(states).unsqueeze(0).to(device) future_progress_t = None if use_future_progress_embedding: if video_clip_mode: # Training video_clip samples are cropped to one local horizon: # state frame 0 plus action frames 1..horizon. Keep progress local # at inference so action-token positions match the training distribution. progress_frame_id = 0 progress_total_frames = horizon + 1 else: progress_frame_id = frame_cursor progress_total_frames = target_frames _, future_progress = build_progress_conditions( frame_id=progress_frame_id, total_frames=progress_total_frames, action_horizon=horizon, ) future_progress_t = torch.from_numpy(future_progress[None]).to(device) with torch.inference_mode(): if enable_rtc and prev_actions_norm is not None and rtc_delay > 0: prev_rtc = np.concatenate( [ prev_actions_norm[exec_horizon:, :], np.zeros((exec_horizon, prev_actions_norm.shape[1]), dtype=np.float32), ], axis=0, ) prev_actions_t = torch.from_numpy(prev_rtc[None]).to(device) pred_norm = model.predict_action_with_training_rtc_flow( observations=None, states=states_t, instructions=[instruction], num_inference_steps=num_inference_steps, traj2ds=None, prev_actions=prev_actions_t, inference_delay=rtc_delay, max_delay=int(rtc_max_delay), videos=[chunk_video] if chunk_video_embed is None else None, video_embeds=[chunk_video_embed] if chunk_video_embed is not None else None, video_paths=[str(video_path)], video_size_pixels=int(video_size_pixels), video_sample_fps=float(video_sample_fps), future_progress=future_progress_t, )[0].float().cpu().numpy() else: pred_norm = model.predict_action( observations=None, states=states_t, instructions=[instruction], num_inference_steps=num_inference_steps, traj2ds=None, videos=[chunk_video] if chunk_video_embed is None else None, video_embeds=[chunk_video_embed] if chunk_video_embed is not None else None, video_paths=[str(video_path)], video_size_pixels=int(video_size_pixels), video_sample_fps=float(video_sample_fps), future_progress=future_progress_t, )[0].float().cpu().numpy() pred_norm_clipped = np.clip(np.asarray(pred_norm, dtype=np.float32), -1.0, 1.0) chunk_pred = np.asarray(maxmin.denormalize(pred_norm_clipped), dtype=np.float32) if decode_textop_joint_from_delta: chunk_abs = recover_seed_abs_from_textop_action_with_joint_delta(chunk_pred, current_abs) else: chunk_abs = recover_seed_abs_from_textop_action(chunk_pred, current_abs) prev_abs = np.concatenate([current_abs[None, :], chunk_abs[:-1]], axis=0) joint_delta = chunk_abs[:, :29] - prev_abs[:, :29] root_pos_delta = chunk_abs[:, 29:32] - prev_abs[:, 29:32] root_quat_delta = quat_mul_wxyz(quat_inv_wxyz(prev_abs[:, 32:36]), chunk_abs[:, 32:36]) chunk_delta = np.concatenate([joint_delta, root_pos_delta, root_quat_delta], axis=1) chunk_abs = normalize_quaternion_rows(chunk_abs.astype(np.float32)) if decode_textop_joint_from_delta: chunk_pred_for_rtc = chunk_pred.copy() chunk_pred_for_rtc[:, :29] = chunk_abs[:, :29] from seed_audio_to_motion import normalize_action_for_rtc prev_actions_norm = normalize_action_for_rtc(maxmin, chunk_pred_for_rtc) else: prev_actions_norm = pred_norm_clipped.copy() take = min(exec_horizon, remaining_frames) if take <= 0: break executed_abs = chunk_abs[:take] executed_delta = chunk_delta[:take].astype(np.float32) current_abs = executed_abs[-1].copy() for frame_abs in executed_abs: state_history.append(frame_abs.astype(np.float32).copy()) state_history = state_history[-state_horizon:] all_abs.append(executed_abs) all_delta.append(executed_delta) remaining_frames -= take pred_abs = np.concatenate(all_abs, axis=0)[:target_frames] pred_delta = np.concatenate(all_delta, axis=0)[:target_frames] pred_abs = normalize_quaternion_rows(pred_abs.astype(np.float32)) return pred_abs, pred_delta, num_chunks def write_video_info( path: Path, *, video_decoder: str, source_motion_dir: Path, video_path: Path, instruction: str, initial_pose_source: str, frames: int, textop_actions: bool, num_past_frames: int, enable_rtc: bool, action_exec_horizon: int, num_inference_steps: int, video_sample_fps: float, video_size_pixels: int, video_embed_path: str, source_video_fps: float, video_duration_sec: float, full_video: bool, video_config_key: str, video_clip_motion_fps: float, ) -> None: lines = [ "Motion Information: seed_video_to_motion_inference", "===================================================", "", f"video_decoder: {video_decoder}", f"video_config_key: {video_config_key}", f"source_motion_dir: {source_motion_dir}", f"video_path: {video_path}", f"instruction: {instruction}", f"initial_pose_source: {initial_pose_source}", f"target_fps: {TARGET_FPS}", f"num_inference_steps: {num_inference_steps}", f"trained_with_textop_actions: {textop_actions}", f"num_past_frames: {num_past_frames}", f"rtc_enabled: {enable_rtc}", f"action_exec_horizon: {action_exec_horizon}", f"video_sample_fps: {video_sample_fps}", f"video_size_pixels: {video_size_pixels}", f"video_embed_path: {video_embed_path}", f"source_video_fps: {source_video_fps}", f"video_duration_sec: {video_duration_sec}", f"full_video: {full_video}", f"video_clip_motion_fps: {video_clip_motion_fps}", "use_audio_in_video: False", "output_root_quaternion: wxyz", "output_joint_angles: 29D", "output_root_position: xyz", "", f"frames: {frames}", "", ] path.write_text("\n".join(lines), encoding="utf-8") def parse_args() -> argparse.Namespace: parser = argparse.ArgumentParser(description="Seed video->motion offline inference exporter") parser.add_argument("--run-dir", type=str, required=True) parser.add_argument("--ckpt-step", type=str, default="latest", help="integer step, 'latest', or 'best'") parser.add_argument("--output-dir", type=str, required=True) parser.add_argument("--motion-dir", type=str, required=True) parser.add_argument("--video-path", type=str, default="") parser.add_argument("--num-output-frames", type=int, default=0, help="0 means source motion length") parser.add_argument("--num-inference-steps", type=int, default=10) parser.add_argument("--device", type=str, default="cuda:0") parser.add_argument("--seed", type=int, default=42) parser.add_argument("--prompt", type=str, default="", help="Optional text instruction paired with the video") parser.add_argument("--export-delta", action="store_true", default=False) parser.add_argument("--disable-rtc", action="store_true", default=False) parser.add_argument("--video-config-key", type=str, default="video", help="data config key: video or video_clip") parser.add_argument("--video-clip", action="store_true", default=False, help="Use one aligned video clip per rollout chunk") parser.add_argument( "--video-clip-motion-fps", type=float, default=0.0, help="Override motion fps used to align rollout chunks to video clips; <=0 infers it from source motion/video.", ) return parser.parse_args() def main() -> None: args = parse_args() run_dir = Path(args.run_dir) output_dir = Path(args.output_dir) output_dir.mkdir(parents=True, exist_ok=True) cfg_template: LaunchConfig = parse_args_to_tyro_config(run_dir / "argv.txt") # type: ignore launch_config = cfg_template.model_validate_json((run_dir / "run_config.json").read_text()) seed_everything(launch_config.seed or args.seed) ckpt_step = resolve_ckpt_step(run_dir, args.ckpt_step) model_cls = ( MotionDistillPsi0Model if hasattr(launch_config.model, "motion_distill_stage") else Psi0Model ) model = model_cls.from_pretrained(run_dir, ckpt_step, launch_config, device=args.device) model.text_only = False model.audio_only = False model.video_only = True model.to(args.device) model.eval() video_cfg = video_data_cfg(launch_config, args.video_config_key) data_root = Path(cfg_get(video_cfg, "root_dir")) video_root = Path(cfg_get(video_cfg, "video_dir", "")) motion_dir = resolve_motion_dir(data_root, args.motion_dir) video_path = resolve_video_path(video_root, motion_dir, args.video_path) if not motion_dir.exists(): raise FileNotFoundError(f"motion_dir does not exist: {motion_dir}") if not video_path.exists(): raise FileNotFoundError(f"video_path does not exist: {video_path}") maxmin = launch_config.data.transform.field odim = int(launch_config.model.odim) textop_actions = bool(getattr(launch_config.data.transform.repack, "textop_actions", False)) decode_textop_joint_from_delta = textop_actions and should_decode_textop_joint_from_delta(launch_config) num_past_frames = int(getattr(launch_config.data.transform.repack, "num_past_frames", 0)) use_future_progress_embedding = bool(getattr(launch_config.model, "use_future_progress_embedding", False)) enable_rtc = bool(getattr(launch_config.model, "rtc", False)) and (not args.disable_rtc) action_exec_horizon = int(getattr(launch_config.model, "action_exec_horizon", int(model.action_horizon))) rtc_max_delay = int(getattr(launch_config.model, "max_delay", 0)) video_sample_fps = float(cfg_get(video_cfg, "video_sample_fps", 1.0)) video_size_pixels = int(cfg_get(video_cfg, "video_size_pixels", 50176)) video_decoder = str(cfg_get(video_cfg, "video_decoder", "opencv")) video_cfg_motion_fps = float(cfg_get(video_cfg, "motion_fps", TARGET_FPS)) video_embed_dir = "" if args.video_clip else str(cfg_get(video_cfg, "video_embed_dir", "")) instruction = args.prompt.strip() video_embed = None video_embed_path = "" if (not args.video_clip) and video_embed_dir.strip(): cache_path = video_embed_cache_path( video_path, video_embed_dir, video_sample_fps=video_sample_fps, video_size_pixels=video_size_pixels, ) if cache_path.exists(): video_embed = load_video_embed_cache(cache_path) video_embed_path = str(cache_path) if args.video_clip: _, _, source_video_fps, video_duration_sec = _probe_video(video_path) video = None elif video_embed is None: video, source_video_fps, video_duration_sec = load_video_array( video_path, video_sample_fps=video_sample_fps, video_size_pixels=video_size_pixels, video_decoder=video_decoder, ) else: video = None source_video_fps = float(video_embed["source_video_fps"]) video_duration_sec = float(video_embed["video_duration_sec"]) x0 = load_pose_from_motion_dir(motion_dir) source_frames = load_source_frame_count(motion_dir) target_frames = int(args.num_output_frames) if args.num_output_frames > 0 else int(source_frames or 0) if args.video_clip and float(args.video_clip_motion_fps) > 0: video_clip_motion_fps = float(args.video_clip_motion_fps) elif args.video_clip and video_cfg_motion_fps > 0: video_clip_motion_fps = float(video_cfg_motion_fps) elif args.video_clip and source_frames > 0 and video_duration_sec > 0: video_clip_motion_fps = float(source_frames) / float(video_duration_sec) else: video_clip_motion_fps = float(TARGET_FPS) pred_abs, pred_delta, rollout_chunks = rollout_video_motion( model=model, maxmin=maxmin, odim=odim, device=args.device, video=video, video_embed=video_embed, video_path=video_path, instruction=instruction, video_size_pixels=video_size_pixels, video_sample_fps=video_sample_fps, video_decoder=video_decoder, video_clip_mode=bool(args.video_clip), video_clip_motion_fps=float(video_clip_motion_fps), normalize_state=bool(getattr(maxmin, "normalize_state", False)), textop_actions=textop_actions, decode_textop_joint_from_delta=decode_textop_joint_from_delta, num_past_frames=num_past_frames, use_future_progress_embedding=use_future_progress_embedding, enable_rtc=enable_rtc, action_exec_horizon=action_exec_horizon, rtc_max_delay=rtc_max_delay, num_inference_steps=int(args.num_inference_steps), target_frames=target_frames, init_abs_state=x0, ) motion_name = sanitize_motion_name(motion_dir.name) sample_dir = output_dir / motion_name sample_dir.mkdir(parents=True, exist_ok=True) joint_pos = pred_abs[:, :29] body_pos = pred_abs[:, 29:32] body_quat = pred_abs[:, 32:36] dt = 1.0 / float(TARGET_FPS) joint_vel = finite_difference(joint_pos, dt) body_lin_vel = finite_difference(body_pos, dt) body_ang_vel = quaternion_angular_velocity(body_quat, dt) write_csv(sample_dir / "joint_pos.csv", joint_pos, [f"joint_{i}" for i in range(29)]) write_csv(sample_dir / "joint_vel.csv", joint_vel, [f"joint_vel_{i}" for i in range(29)]) write_csv(sample_dir / "body_pos.csv", body_pos, ["body_0_x", "body_0_y", "body_0_z"]) write_csv(sample_dir / "body_quat.csv", body_quat, ["body_0_w", "body_0_x", "body_0_y", "body_0_z"]) write_csv(sample_dir / "body_lin_vel.csv", body_lin_vel, ["body_0_vel_x", "body_0_vel_y", "body_0_vel_z"]) write_csv(sample_dir / "body_ang_vel.csv", body_ang_vel, ["body_0_angvel_x", "body_0_angvel_y", "body_0_angvel_z"]) np.save(sample_dir / "motion.npy", pred_abs) if args.export_delta: write_csv(sample_dir / "joint_pos_delta.csv", pred_delta[:, :29], [f"joint_{i}" for i in range(29)]) write_csv(sample_dir / "body_pos_delta.csv", pred_delta[:, 29:32], ["body_0_x", "body_0_y", "body_0_z"]) write_csv(sample_dir / "body_quat_delta.csv", pred_delta[:, 32:36], ["body_0_w", "body_0_x", "body_0_y", "body_0_z"]) np.save(sample_dir / "motion_delta.npy", pred_delta) write_metadata(sample_dir / "metadata.txt", motion_name, int(pred_abs.shape[0])) write_video_info( sample_dir / "info.txt", video_decoder=video_decoder, source_motion_dir=motion_dir, video_path=video_path, instruction=instruction, initial_pose_source=str(motion_dir), frames=int(pred_abs.shape[0]), textop_actions=textop_actions, num_past_frames=int(num_past_frames), enable_rtc=enable_rtc, action_exec_horizon=int(action_exec_horizon), num_inference_steps=int(args.num_inference_steps), video_sample_fps=float(video_sample_fps), video_size_pixels=int(video_size_pixels), video_embed_path=video_embed_path, source_video_fps=float(source_video_fps), video_duration_sec=float(video_duration_sec), full_video=not bool(args.video_clip), video_config_key=str(args.video_config_key), video_clip_motion_fps=float(video_clip_motion_fps), ) for src_name, dst_name in (("info.txt", "info_source.txt"), ("metadata.txt", "metadata_source.txt")): src_path = motion_dir / src_name if src_path.exists(): shutil.copy2(src_path, sample_dir / dst_name) sample_meta = { "run_dir": str(run_dir), "ckpt_step": ckpt_step, "source_motion_dir": str(motion_dir), "video_path": str(video_path), "instruction": instruction, "output_dir": str(sample_dir), "num_output_frames": int(pred_abs.shape[0]), "rollout_chunks": int(rollout_chunks), "textop_actions": textop_actions, "decode_textop_joint_from_delta": decode_textop_joint_from_delta, "num_past_frames": int(num_past_frames), "enable_rtc": enable_rtc, "action_exec_horizon": int(action_exec_horizon), "video_sample_fps": float(video_sample_fps), "video_size_pixels": int(video_size_pixels), "video_decoder": video_decoder, "video_embed_path": video_embed_path, "source_video_fps": float(source_video_fps), "video_duration_sec": float(video_duration_sec), "full_video": not bool(args.video_clip), "video_config_key": str(args.video_config_key), "video_clip_motion_fps": float(video_clip_motion_fps), "use_video_instruction": bool(cfg_get(video_cfg, "use_video_instruction", False)), "use_audio_in_video": False, } (sample_dir / "infer_meta.json").write_text(json.dumps(sample_meta, ensure_ascii=False, indent=2), encoding="utf-8") (output_dir / "manifest.json").write_text( json.dumps({"samples": [sample_meta]}, ensure_ascii=False, indent=2), encoding="utf-8", ) print(f"[OK] video sample -> {sample_dir}") print(f"[DONE] wrote manifest: {output_dir / 'manifest.json'}") if __name__ == "__main__": main()