code-Psi0-yth-v21 / scripts /infer /seed_video_to_motion.py
Tinghong-Ye22's picture
Add files using upload-large-folder tool
c4f8e4a verified
Raw History Blame Contribute Delete
22.8 kB
#!/usr/bin/env python3
from __future__ import annotations
import argparse
import json
import math
import shutil
from pathlib import Path
from typing import List
import numpy as np
import torch
from psi.config.config import LaunchConfig
from psi.data.seed.motion_repr import quat_inv_wxyz, quat_mul_wxyz, recover_seed_abs_from_textop_action
from psi.data.seed.progress import build_progress_conditions
from psi.data.seed.seed_video_motion_dataset import (
_probe_video,
load_video_array,
load_video_clip_array,
load_video_embed_cache,
video_embed_cache_path,
)
from psi.models.motion_distill import MotionDistillPsi0Model
from psi.models.psi0 import Psi0Model
from psi.utils import parse_args_to_tyro_config, seed_everything
from seed_audio_to_motion import resolve_motion_dir, resolve_video_path
from seed_text_to_motion import (
TARGET_FPS,
build_model_states_from_abs,
finite_difference,
load_pose_from_motion_dir,
load_source_frame_count,
normalize_quaternion_rows,
quaternion_angular_velocity,
recover_seed_abs_from_textop_action_with_joint_delta,
resolve_ckpt_step,
sanitize_motion_name,
should_decode_textop_joint_from_delta,
write_csv,
write_metadata,
)
def cfg_get(obj, key: str, default=None):
if isinstance(obj, dict):
return obj.get(key, default)
return getattr(obj, key, default)
def video_data_cfg(launch_config: LaunchConfig, config_key: str = "video"):
return cfg_get(launch_config.data, config_key, cfg_get(launch_config.data, "video", launch_config.data))
def rollout_video_motion(
*,
model: Psi0Model,
maxmin,
odim: int,
device: str,
video: np.ndarray | None,
video_embed: dict | None,
video_path: Path,
instruction: str,
video_size_pixels: int,
video_sample_fps: float,
video_decoder: str,
video_clip_mode: bool,
video_clip_motion_fps: float,
normalize_state: bool,
textop_actions: bool,
decode_textop_joint_from_delta: bool,
num_past_frames: int,
use_future_progress_embedding: bool,
enable_rtc: bool,
action_exec_horizon: int,
rtc_max_delay: int,
num_inference_steps: int,
target_frames: int,
init_abs_state: np.ndarray,
) -> tuple[np.ndarray, np.ndarray, int]:
horizon = int(model.action_horizon)
exec_horizon = int(action_exec_horizon) if enable_rtc else horizon
if exec_horizon <= 0 or exec_horizon > horizon:
raise ValueError(f"Invalid action_exec_horizon={exec_horizon}, action_horizon={horizon}")
rtc_delay = horizon - exec_horizon
if enable_rtc and rtc_delay > 0 and rtc_delay >= int(rtc_max_delay):
raise ValueError(
f"RTC requires action_horizon - action_exec_horizon < max_delay, "
f"got {horizon} - {exec_horizon} = {rtc_delay}, max_delay={rtc_max_delay}"
)
if not textop_actions:
raise ValueError("This video inference path is for the 39D TextOp action checkpoint.")
if target_frames <= 0:
target_frames = exec_horizon
num_chunks = int(math.ceil(float(target_frames) / float(exec_horizon)))
current_abs = np.asarray(init_abs_state, dtype=np.float32).copy()
state_horizon = int(num_past_frames) + 1
state_history: List[np.ndarray] = [current_abs.copy() for _ in range(state_horizon)]
all_abs: List[np.ndarray] = []
all_delta: List[np.ndarray] = []
prev_actions_norm: np.ndarray | None = None
remaining_frames = int(target_frames)
for _ in range(num_chunks):
if video_clip_mode:
chunk_index = len(all_abs)
video_clip_start_sec = float(chunk_index * exec_horizon) / float(video_clip_motion_fps)
video_clip_duration_sec = float(horizon) / float(video_clip_motion_fps)
chunk_video, _, _ = load_video_clip_array(
video_path,
start_sec=video_clip_start_sec,
duration_sec=video_clip_duration_sec,
video_sample_fps=video_sample_fps,
video_size_pixels=video_size_pixels,
video_decoder=video_decoder,
)
chunk_video_embed = None
else:
chunk_video = video
chunk_video_embed = video_embed
frame_cursor = target_frames - remaining_frames
states = build_model_states_from_abs(
state_history[-state_horizon:],
textop_actions=textop_actions,
odim=odim,
)
if normalize_state:
states = maxmin.normalize_state_func(states)
states_t = torch.from_numpy(states).unsqueeze(0).to(device)
future_progress_t = None
if use_future_progress_embedding:
if video_clip_mode:
# Training video_clip samples are cropped to one local horizon:
# state frame 0 plus action frames 1..horizon. Keep progress local
# at inference so action-token positions match the training distribution.
progress_frame_id = 0
progress_total_frames = horizon + 1
else:
progress_frame_id = frame_cursor
progress_total_frames = target_frames
_, future_progress = build_progress_conditions(
frame_id=progress_frame_id,
total_frames=progress_total_frames,
action_horizon=horizon,
)
future_progress_t = torch.from_numpy(future_progress[None]).to(device)
with torch.inference_mode():
if enable_rtc and prev_actions_norm is not None and rtc_delay > 0:
prev_rtc = np.concatenate(
[
prev_actions_norm[exec_horizon:, :],
np.zeros((exec_horizon, prev_actions_norm.shape[1]), dtype=np.float32),
],
axis=0,
)
prev_actions_t = torch.from_numpy(prev_rtc[None]).to(device)
pred_norm = model.predict_action_with_training_rtc_flow(
observations=None,
states=states_t,
instructions=[instruction],
num_inference_steps=num_inference_steps,
traj2ds=None,
prev_actions=prev_actions_t,
inference_delay=rtc_delay,
max_delay=int(rtc_max_delay),
videos=[chunk_video] if chunk_video_embed is None else None,
video_embeds=[chunk_video_embed] if chunk_video_embed is not None else None,
video_paths=[str(video_path)],
video_size_pixels=int(video_size_pixels),
video_sample_fps=float(video_sample_fps),
future_progress=future_progress_t,
)[0].float().cpu().numpy()
else:
pred_norm = model.predict_action(
observations=None,
states=states_t,
instructions=[instruction],
num_inference_steps=num_inference_steps,
traj2ds=None,
videos=[chunk_video] if chunk_video_embed is None else None,
video_embeds=[chunk_video_embed] if chunk_video_embed is not None else None,
video_paths=[str(video_path)],
video_size_pixels=int(video_size_pixels),
video_sample_fps=float(video_sample_fps),
future_progress=future_progress_t,
)[0].float().cpu().numpy()
pred_norm_clipped = np.clip(np.asarray(pred_norm, dtype=np.float32), -1.0, 1.0)
chunk_pred = np.asarray(maxmin.denormalize(pred_norm_clipped), dtype=np.float32)
if decode_textop_joint_from_delta:
chunk_abs = recover_seed_abs_from_textop_action_with_joint_delta(chunk_pred, current_abs)
else:
chunk_abs = recover_seed_abs_from_textop_action(chunk_pred, current_abs)
prev_abs = np.concatenate([current_abs[None, :], chunk_abs[:-1]], axis=0)
joint_delta = chunk_abs[:, :29] - prev_abs[:, :29]
root_pos_delta = chunk_abs[:, 29:32] - prev_abs[:, 29:32]
root_quat_delta = quat_mul_wxyz(quat_inv_wxyz(prev_abs[:, 32:36]), chunk_abs[:, 32:36])
chunk_delta = np.concatenate([joint_delta, root_pos_delta, root_quat_delta], axis=1)
chunk_abs = normalize_quaternion_rows(chunk_abs.astype(np.float32))
if decode_textop_joint_from_delta:
chunk_pred_for_rtc = chunk_pred.copy()
chunk_pred_for_rtc[:, :29] = chunk_abs[:, :29]
from seed_audio_to_motion import normalize_action_for_rtc
prev_actions_norm = normalize_action_for_rtc(maxmin, chunk_pred_for_rtc)
else:
prev_actions_norm = pred_norm_clipped.copy()
take = min(exec_horizon, remaining_frames)
if take <= 0:
break
executed_abs = chunk_abs[:take]
executed_delta = chunk_delta[:take].astype(np.float32)
current_abs = executed_abs[-1].copy()
for frame_abs in executed_abs:
state_history.append(frame_abs.astype(np.float32).copy())
state_history = state_history[-state_horizon:]
all_abs.append(executed_abs)
all_delta.append(executed_delta)
remaining_frames -= take
pred_abs = np.concatenate(all_abs, axis=0)[:target_frames]
pred_delta = np.concatenate(all_delta, axis=0)[:target_frames]
pred_abs = normalize_quaternion_rows(pred_abs.astype(np.float32))
return pred_abs, pred_delta, num_chunks
def write_video_info(
path: Path,
*,
video_decoder: str,
source_motion_dir: Path,
video_path: Path,
instruction: str,
initial_pose_source: str,
frames: int,
textop_actions: bool,
num_past_frames: int,
enable_rtc: bool,
action_exec_horizon: int,
num_inference_steps: int,
video_sample_fps: float,
video_size_pixels: int,
video_embed_path: str,
source_video_fps: float,
video_duration_sec: float,
full_video: bool,
video_config_key: str,
video_clip_motion_fps: float,
) -> None:
lines = [
"Motion Information: seed_video_to_motion_inference",
"===================================================",
"",
f"video_decoder: {video_decoder}",
f"video_config_key: {video_config_key}",
f"source_motion_dir: {source_motion_dir}",
f"video_path: {video_path}",
f"instruction: {instruction}",
f"initial_pose_source: {initial_pose_source}",
f"target_fps: {TARGET_FPS}",
f"num_inference_steps: {num_inference_steps}",
f"trained_with_textop_actions: {textop_actions}",
f"num_past_frames: {num_past_frames}",
f"rtc_enabled: {enable_rtc}",
f"action_exec_horizon: {action_exec_horizon}",
f"video_sample_fps: {video_sample_fps}",
f"video_size_pixels: {video_size_pixels}",
f"video_embed_path: {video_embed_path}",
f"source_video_fps: {source_video_fps}",
f"video_duration_sec: {video_duration_sec}",
f"full_video: {full_video}",
f"video_clip_motion_fps: {video_clip_motion_fps}",
"use_audio_in_video: False",
"output_root_quaternion: wxyz",
"output_joint_angles: 29D",
"output_root_position: xyz",
"",
f"frames: {frames}",
"",
]
path.write_text("\n".join(lines), encoding="utf-8")
def parse_args() -> argparse.Namespace:
parser = argparse.ArgumentParser(description="Seed video->motion offline inference exporter")
parser.add_argument("--run-dir", type=str, required=True)
parser.add_argument("--ckpt-step", type=str, default="latest", help="integer step, 'latest', or 'best'")
parser.add_argument("--output-dir", type=str, required=True)
parser.add_argument("--motion-dir", type=str, required=True)
parser.add_argument("--video-path", type=str, default="")
parser.add_argument("--num-output-frames", type=int, default=0, help="0 means source motion length")
parser.add_argument("--num-inference-steps", type=int, default=10)
parser.add_argument("--device", type=str, default="cuda:0")
parser.add_argument("--seed", type=int, default=42)
parser.add_argument("--prompt", type=str, default="", help="Optional text instruction paired with the video")
parser.add_argument("--export-delta", action="store_true", default=False)
parser.add_argument("--disable-rtc", action="store_true", default=False)
parser.add_argument("--video-config-key", type=str, default="video", help="data config key: video or video_clip")
parser.add_argument("--video-clip", action="store_true", default=False, help="Use one aligned video clip per rollout chunk")
parser.add_argument(
"--video-clip-motion-fps",
type=float,
default=0.0,
help="Override motion fps used to align rollout chunks to video clips; <=0 infers it from source motion/video.",
)
return parser.parse_args()
def main() -> None:
args = parse_args()
run_dir = Path(args.run_dir)
output_dir = Path(args.output_dir)
output_dir.mkdir(parents=True, exist_ok=True)
cfg_template: LaunchConfig = parse_args_to_tyro_config(run_dir / "argv.txt") # type: ignore
launch_config = cfg_template.model_validate_json((run_dir / "run_config.json").read_text())
seed_everything(launch_config.seed or args.seed)
ckpt_step = resolve_ckpt_step(run_dir, args.ckpt_step)
model_cls = (
MotionDistillPsi0Model
if hasattr(launch_config.model, "motion_distill_stage")
else Psi0Model
)
model = model_cls.from_pretrained(run_dir, ckpt_step, launch_config, device=args.device)
model.text_only = False
model.audio_only = False
model.video_only = True
model.to(args.device)
model.eval()
video_cfg = video_data_cfg(launch_config, args.video_config_key)
data_root = Path(cfg_get(video_cfg, "root_dir"))
video_root = Path(cfg_get(video_cfg, "video_dir", ""))
motion_dir = resolve_motion_dir(data_root, args.motion_dir)
video_path = resolve_video_path(video_root, motion_dir, args.video_path)
if not motion_dir.exists():
raise FileNotFoundError(f"motion_dir does not exist: {motion_dir}")
if not video_path.exists():
raise FileNotFoundError(f"video_path does not exist: {video_path}")
maxmin = launch_config.data.transform.field
odim = int(launch_config.model.odim)
textop_actions = bool(getattr(launch_config.data.transform.repack, "textop_actions", False))
decode_textop_joint_from_delta = textop_actions and should_decode_textop_joint_from_delta(launch_config)
num_past_frames = int(getattr(launch_config.data.transform.repack, "num_past_frames", 0))
use_future_progress_embedding = bool(getattr(launch_config.model, "use_future_progress_embedding", False))
enable_rtc = bool(getattr(launch_config.model, "rtc", False)) and (not args.disable_rtc)
action_exec_horizon = int(getattr(launch_config.model, "action_exec_horizon", int(model.action_horizon)))
rtc_max_delay = int(getattr(launch_config.model, "max_delay", 0))
video_sample_fps = float(cfg_get(video_cfg, "video_sample_fps", 1.0))
video_size_pixels = int(cfg_get(video_cfg, "video_size_pixels", 50176))
video_decoder = str(cfg_get(video_cfg, "video_decoder", "opencv"))
video_cfg_motion_fps = float(cfg_get(video_cfg, "motion_fps", TARGET_FPS))
video_embed_dir = "" if args.video_clip else str(cfg_get(video_cfg, "video_embed_dir", ""))
instruction = args.prompt.strip()
video_embed = None
video_embed_path = ""
if (not args.video_clip) and video_embed_dir.strip():
cache_path = video_embed_cache_path(
video_path,
video_embed_dir,
video_sample_fps=video_sample_fps,
video_size_pixels=video_size_pixels,
)
if cache_path.exists():
video_embed = load_video_embed_cache(cache_path)
video_embed_path = str(cache_path)
if args.video_clip:
_, _, source_video_fps, video_duration_sec = _probe_video(video_path)
video = None
elif video_embed is None:
video, source_video_fps, video_duration_sec = load_video_array(
video_path,
video_sample_fps=video_sample_fps,
video_size_pixels=video_size_pixels,
video_decoder=video_decoder,
)
else:
video = None
source_video_fps = float(video_embed["source_video_fps"])
video_duration_sec = float(video_embed["video_duration_sec"])
x0 = load_pose_from_motion_dir(motion_dir)
source_frames = load_source_frame_count(motion_dir)
target_frames = int(args.num_output_frames) if args.num_output_frames > 0 else int(source_frames or 0)
if args.video_clip and float(args.video_clip_motion_fps) > 0:
video_clip_motion_fps = float(args.video_clip_motion_fps)
elif args.video_clip and video_cfg_motion_fps > 0:
video_clip_motion_fps = float(video_cfg_motion_fps)
elif args.video_clip and source_frames > 0 and video_duration_sec > 0:
video_clip_motion_fps = float(source_frames) / float(video_duration_sec)
else:
video_clip_motion_fps = float(TARGET_FPS)
pred_abs, pred_delta, rollout_chunks = rollout_video_motion(
model=model,
maxmin=maxmin,
odim=odim,
device=args.device,
video=video,
video_embed=video_embed,
video_path=video_path,
instruction=instruction,
video_size_pixels=video_size_pixels,
video_sample_fps=video_sample_fps,
video_decoder=video_decoder,
video_clip_mode=bool(args.video_clip),
video_clip_motion_fps=float(video_clip_motion_fps),
normalize_state=bool(getattr(maxmin, "normalize_state", False)),
textop_actions=textop_actions,
decode_textop_joint_from_delta=decode_textop_joint_from_delta,
num_past_frames=num_past_frames,
use_future_progress_embedding=use_future_progress_embedding,
enable_rtc=enable_rtc,
action_exec_horizon=action_exec_horizon,
rtc_max_delay=rtc_max_delay,
num_inference_steps=int(args.num_inference_steps),
target_frames=target_frames,
init_abs_state=x0,
)
motion_name = sanitize_motion_name(motion_dir.name)
sample_dir = output_dir / motion_name
sample_dir.mkdir(parents=True, exist_ok=True)
joint_pos = pred_abs[:, :29]
body_pos = pred_abs[:, 29:32]
body_quat = pred_abs[:, 32:36]
dt = 1.0 / float(TARGET_FPS)
joint_vel = finite_difference(joint_pos, dt)
body_lin_vel = finite_difference(body_pos, dt)
body_ang_vel = quaternion_angular_velocity(body_quat, dt)
write_csv(sample_dir / "joint_pos.csv", joint_pos, [f"joint_{i}" for i in range(29)])
write_csv(sample_dir / "joint_vel.csv", joint_vel, [f"joint_vel_{i}" for i in range(29)])
write_csv(sample_dir / "body_pos.csv", body_pos, ["body_0_x", "body_0_y", "body_0_z"])
write_csv(sample_dir / "body_quat.csv", body_quat, ["body_0_w", "body_0_x", "body_0_y", "body_0_z"])
write_csv(sample_dir / "body_lin_vel.csv", body_lin_vel, ["body_0_vel_x", "body_0_vel_y", "body_0_vel_z"])
write_csv(sample_dir / "body_ang_vel.csv", body_ang_vel, ["body_0_angvel_x", "body_0_angvel_y", "body_0_angvel_z"])
np.save(sample_dir / "motion.npy", pred_abs)
if args.export_delta:
write_csv(sample_dir / "joint_pos_delta.csv", pred_delta[:, :29], [f"joint_{i}" for i in range(29)])
write_csv(sample_dir / "body_pos_delta.csv", pred_delta[:, 29:32], ["body_0_x", "body_0_y", "body_0_z"])
write_csv(sample_dir / "body_quat_delta.csv", pred_delta[:, 32:36], ["body_0_w", "body_0_x", "body_0_y", "body_0_z"])
np.save(sample_dir / "motion_delta.npy", pred_delta)
write_metadata(sample_dir / "metadata.txt", motion_name, int(pred_abs.shape[0]))
write_video_info(
sample_dir / "info.txt",
video_decoder=video_decoder,
source_motion_dir=motion_dir,
video_path=video_path,
instruction=instruction,
initial_pose_source=str(motion_dir),
frames=int(pred_abs.shape[0]),
textop_actions=textop_actions,
num_past_frames=int(num_past_frames),
enable_rtc=enable_rtc,
action_exec_horizon=int(action_exec_horizon),
num_inference_steps=int(args.num_inference_steps),
video_sample_fps=float(video_sample_fps),
video_size_pixels=int(video_size_pixels),
video_embed_path=video_embed_path,
source_video_fps=float(source_video_fps),
video_duration_sec=float(video_duration_sec),
full_video=not bool(args.video_clip),
video_config_key=str(args.video_config_key),
video_clip_motion_fps=float(video_clip_motion_fps),
)
for src_name, dst_name in (("info.txt", "info_source.txt"), ("metadata.txt", "metadata_source.txt")):
src_path = motion_dir / src_name
if src_path.exists():
shutil.copy2(src_path, sample_dir / dst_name)
sample_meta = {
"run_dir": str(run_dir),
"ckpt_step": ckpt_step,
"source_motion_dir": str(motion_dir),
"video_path": str(video_path),
"instruction": instruction,
"output_dir": str(sample_dir),
"num_output_frames": int(pred_abs.shape[0]),
"rollout_chunks": int(rollout_chunks),
"textop_actions": textop_actions,
"decode_textop_joint_from_delta": decode_textop_joint_from_delta,
"num_past_frames": int(num_past_frames),
"enable_rtc": enable_rtc,
"action_exec_horizon": int(action_exec_horizon),
"video_sample_fps": float(video_sample_fps),
"video_size_pixels": int(video_size_pixels),
"video_decoder": video_decoder,
"video_embed_path": video_embed_path,
"source_video_fps": float(source_video_fps),
"video_duration_sec": float(video_duration_sec),
"full_video": not bool(args.video_clip),
"video_config_key": str(args.video_config_key),
"video_clip_motion_fps": float(video_clip_motion_fps),
"use_video_instruction": bool(cfg_get(video_cfg, "use_video_instruction", False)),
"use_audio_in_video": False,
}
(sample_dir / "infer_meta.json").write_text(json.dumps(sample_meta, ensure_ascii=False, indent=2), encoding="utf-8")
(output_dir / "manifest.json").write_text(
json.dumps({"samples": [sample_meta]}, ensure_ascii=False, indent=2),
encoding="utf-8",
)
print(f"[OK] video sample -> {sample_dir}")
print(f"[DONE] wrote manifest: {output_dir / 'manifest.json'}")
if __name__ == "__main__":
main()