#!/usr/bin/env python3 from __future__ import annotations import argparse import json import math import random import re import shutil import subprocess from pathlib import Path from typing import List import numpy as np import torch from psi.config.config import LaunchConfig from psi.data.seed.motion_repr import quat_inv_wxyz, quat_mul_wxyz, recover_seed_abs_from_textop_action from psi.data.seed.progress import build_progress_conditions from psi.models.psi0 import Psi0Model from psi.utils import parse_args_to_tyro_config, seed_everything from seed_text_to_motion import ( TARGET_FPS, build_model_states_from_abs, finite_difference, load_pose_from_motion_dir, load_source_frame_count, normalize_quaternion_rows, quaternion_angular_velocity, recover_seed_abs_from_textop_action_with_joint_delta, resolve_ckpt_step, sanitize_motion_name, should_decode_textop_joint_from_delta, write_csv, write_metadata, ) def cfg_get(obj, key: str, default=None): if isinstance(obj, dict): return obj.get(key, default) return getattr(obj, key, default) def audio_data_cfg(launch_config: LaunchConfig, key: str = "audio"): return cfg_get(launch_config.data, key, launch_config.data) def load_audio_array(video_path: Path, sampling_rate: int) -> np.ndarray: cmd = [ "ffmpeg", "-v", "error", "-i", str(video_path), "-ac", "1", "-ar", str(sampling_rate), "-f", "f32le", "-", ] proc = subprocess.run(cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE, check=False) if proc.returncode != 0: msg = proc.stderr.decode("utf-8", errors="ignore") raise RuntimeError(f"ffmpeg failed to decode audio from {video_path}: {msg}") audio = np.frombuffer(proc.stdout, dtype=np.float32).copy() if audio.size == 0: raise RuntimeError(f"No audio samples decoded from {video_path}") return audio def audio_segment(audio: np.ndarray, sampling_rate: int, start_sec: float, duration_sec: float) -> np.ndarray: num_samples = max(1, int(round(duration_sec * sampling_rate))) start_sample = int(round(start_sec * sampling_rate)) end_sample = start_sample + num_samples segment = np.zeros((num_samples,), dtype=np.float32) src_start = max(0, start_sample) src_end = min(audio.shape[0], end_sample) if src_end > src_start: dst_start = src_start - start_sample segment[dst_start : dst_start + (src_end - src_start)] = audio[src_start:src_end] return segment def read_start_hold_seconds(motion_dir: Path, align_start_hold: bool) -> float: if not align_start_hold: return 0.0 info_path = motion_dir / "info.txt" if not info_path.exists(): return 0.0 text = info_path.read_text(encoding="utf-8", errors="ignore") match = re.search(r"start_hold_seconds:\s*([0-9.]+)", text) return float(match.group(1)) if match else 0.0 def resolve_motion_dir(root_dir: Path, motion_dir_arg: str) -> Path: motion_dir = Path(motion_dir_arg).expanduser() if motion_dir.is_absolute(): return motion_dir return root_dir / motion_dir def resolve_video_path(video_dir: Path, motion_dir: Path, video_path_arg: str) -> Path: if video_path_arg: video_path = Path(video_path_arg).expanduser() if video_path.is_absolute(): return video_path return video_dir / video_path return video_dir / f"{motion_dir.name}.mp4" def resolve_audio_path(audio_path: Path, extensions: str = ".wav,.flac,.mp3,.m4a,.aac,.ogg") -> Path: if audio_path.is_file(): return audio_path if not audio_path.is_dir(): return audio_path audio_exts = tuple( ext if ext.startswith(".") else f".{ext}" for ext in [part.strip().lower() for part in str(extensions).split(",")] if ext ) candidates = sorted(p for p in audio_path.iterdir() if p.is_file() and p.suffix.lower() in audio_exts) if not candidates: raise FileNotFoundError(f"No audio files found under {audio_path}") return random.choice(candidates) def normalize_action_for_rtc(maxmin, action: np.ndarray) -> np.ndarray: action_min = np.asarray(maxmin.action_min, dtype=np.float32) action_max = np.asarray(maxmin.action_max, dtype=np.float32) ill_mask = np.abs(action_max - action_min) < 1e-4 * (np.abs(action_max) + np.abs(action_min) + 1e-8) action_max = action_max.copy() action_max[ill_mask] = 1.0 action_norm = np.where(ill_mask, action, (action - action_min) / (action_max - action_min) * 2 - 1) if bool(getattr(maxmin, "use_norm_mask", False)): action_norm_masks = np.asarray(maxmin.action_norm_masks, dtype=bool) action_norm = np.where(action_norm_masks, action_norm, action) return np.clip(action_norm, -1.0, 1.0).astype(np.float32) def rollout_audio_motion( *, model: Psi0Model, maxmin, odim: int, device: str, instruction: str, audio: np.ndarray, audio_sampling_rate: int, audio_window_seconds: float, audio_left_context_seconds: float, full_audio: bool, start_hold_seconds: float, normalize_state: bool, textop_actions: bool, decode_textop_joint_from_delta: bool, num_past_frames: int, use_future_progress_embedding: bool, enable_rtc: bool, action_exec_horizon: int, rtc_max_delay: int, num_inference_steps: int, target_frames: int, init_abs_state: np.ndarray, ) -> tuple[np.ndarray, np.ndarray, int]: horizon = int(model.action_horizon) exec_horizon = int(action_exec_horizon) if enable_rtc else horizon if exec_horizon <= 0 or exec_horizon > horizon: raise ValueError(f"Invalid action_exec_horizon={exec_horizon}, action_horizon={horizon}") rtc_delay = horizon - exec_horizon if enable_rtc and rtc_delay > 0 and rtc_delay >= int(rtc_max_delay): raise ValueError( f"RTC requires action_horizon - action_exec_horizon < max_delay, " f"got {horizon} - {exec_horizon} = {rtc_delay}, max_delay={rtc_max_delay}" ) if not textop_actions: raise ValueError("This audio inference path is for the 39D TextOp action checkpoint.") if target_frames <= 0: target_frames = max(exec_horizon, int(round(audio.shape[0] / float(audio_sampling_rate) * TARGET_FPS))) num_chunks = int(math.ceil(float(target_frames) / float(exec_horizon))) current_abs = np.asarray(init_abs_state, dtype=np.float32).copy() state_horizon = int(num_past_frames) + 1 state_history: List[np.ndarray] = [current_abs.copy() for _ in range(state_horizon)] all_abs: List[np.ndarray] = [] all_delta: List[np.ndarray] = [] prev_actions_norm: np.ndarray | None = None remaining_frames = int(target_frames) for _ in range(num_chunks): frame_cursor = target_frames - remaining_frames states = build_model_states_from_abs( state_history[-state_horizon:], textop_actions=textop_actions, odim=odim, ) if normalize_state: states = maxmin.normalize_state_func(states) states_t = torch.from_numpy(states).unsqueeze(0).to(device) future_progress_t = None if use_future_progress_embedding: _, future_progress = build_progress_conditions( frame_id=frame_cursor, total_frames=target_frames, action_horizon=horizon, ) future_progress_t = torch.from_numpy(future_progress[None]).to(device) # 和训练一致:full_audio=True时每个chunk都看到整段音频embedding,不再给time/progress。 if full_audio: audio_clip = audio.astype(np.float32) else: audio_start_sec = ( frame_cursor / float(TARGET_FPS) - float(start_hold_seconds) - float(audio_left_context_seconds) ) audio_clip = audio_segment(audio, audio_sampling_rate, audio_start_sec, audio_window_seconds) with torch.inference_mode(): if enable_rtc and prev_actions_norm is not None and rtc_delay > 0: prev_rtc = np.concatenate( [ prev_actions_norm[exec_horizon:, :], np.zeros((exec_horizon, prev_actions_norm.shape[1]), dtype=np.float32), ], axis=0, ) prev_actions_t = torch.from_numpy(prev_rtc[None]).to(device) pred_norm = model.predict_action_with_training_rtc_flow( observations=None, states=states_t, instructions=[instruction], num_inference_steps=num_inference_steps, traj2ds=None, prev_actions=prev_actions_t, inference_delay=rtc_delay, max_delay=int(rtc_max_delay), audios=[audio_clip], future_progress=future_progress_t, )[0].float().cpu().numpy() else: pred_norm = model.predict_action( observations=None, states=states_t, instructions=[instruction], num_inference_steps=num_inference_steps, traj2ds=None, audios=[audio_clip], future_progress=future_progress_t, )[0].float().cpu().numpy() pred_norm_clipped = np.clip(np.asarray(pred_norm, dtype=np.float32), -1.0, 1.0) chunk_pred = np.asarray(maxmin.denormalize(pred_norm_clipped), dtype=np.float32) if decode_textop_joint_from_delta: chunk_abs = recover_seed_abs_from_textop_action_with_joint_delta(chunk_pred, current_abs) else: chunk_abs = recover_seed_abs_from_textop_action(chunk_pred, current_abs) prev_abs = np.concatenate([current_abs[None, :], chunk_abs[:-1]], axis=0) joint_delta = chunk_abs[:, :29] - prev_abs[:, :29] root_pos_delta = chunk_abs[:, 29:32] - prev_abs[:, 29:32] root_quat_delta = quat_mul_wxyz(quat_inv_wxyz(prev_abs[:, 32:36]), chunk_abs[:, 32:36]) chunk_delta = np.concatenate([joint_delta, root_pos_delta, root_quat_delta], axis=1) chunk_abs = normalize_quaternion_rows(chunk_abs.astype(np.float32)) if decode_textop_joint_from_delta: chunk_pred_for_rtc = chunk_pred.copy() chunk_pred_for_rtc[:, :29] = chunk_abs[:, :29] prev_actions_norm = normalize_action_for_rtc(maxmin, chunk_pred_for_rtc) else: prev_actions_norm = pred_norm_clipped.copy() take = min(exec_horizon, remaining_frames) if take <= 0: break executed_abs = chunk_abs[:take] executed_delta = chunk_delta[:take].astype(np.float32) current_abs = executed_abs[-1].copy() for frame_abs in executed_abs: state_history.append(frame_abs.astype(np.float32).copy()) state_history = state_history[-state_horizon:] all_abs.append(executed_abs) all_delta.append(executed_delta) remaining_frames -= take pred_abs = np.concatenate(all_abs, axis=0)[:target_frames] pred_delta = np.concatenate(all_delta, axis=0)[:target_frames] pred_abs = normalize_quaternion_rows(pred_abs.astype(np.float32)) return pred_abs, pred_delta, num_chunks def write_audio_info( path: Path, *, audio_config_key: str, source_motion_dir: Path, video_path: Path, initial_pose_source: str, frames: int, textop_actions: bool, num_past_frames: int, enable_rtc: bool, action_exec_horizon: int, num_inference_steps: int, audio_sampling_rate: int, audio_window_seconds: float, full_audio: bool, instruction: str, ) -> None: lines = [ "Motion Information: seed_audio_to_motion_inference", "===================================================", "", f"audio_config_key: {audio_config_key}", f"source_motion_dir: {source_motion_dir}", f"video_path: {video_path}", f"instruction: {instruction}", f"initial_pose_source: {initial_pose_source}", f"target_fps: {TARGET_FPS}", f"num_inference_steps: {num_inference_steps}", f"trained_with_textop_actions: {textop_actions}", f"num_past_frames: {num_past_frames}", f"rtc_enabled: {enable_rtc}", f"action_exec_horizon: {action_exec_horizon}", f"audio_sampling_rate: {audio_sampling_rate}", f"full_audio: {full_audio}", f"audio_window_seconds: {audio_window_seconds}", "output_root_quaternion: wxyz", "output_joint_angles: 29D", "output_root_position: xyz", "", f"frames: {frames}", "", ] path.write_text("\n".join(lines), encoding="utf-8") def parse_args() -> argparse.Namespace: parser = argparse.ArgumentParser(description="Seed audio->motion offline inference exporter") parser.add_argument("--run-dir", type=str, required=True) parser.add_argument("--ckpt-step", type=str, default="latest", help="integer step, 'latest', or 'best'") parser.add_argument("--output-dir", type=str, required=True) parser.add_argument("--motion-dir", type=str, required=True) parser.add_argument("--video-path", type=str, default="") parser.add_argument( "--audio-config-key", type=str, default="audio", choices=("audio", "audio_command"), help="Which launch_config.data audio section to use", ) parser.add_argument("--prompt", type=str, default="", help="Optional text instruction paired with the audio") parser.add_argument("--num-output-frames", type=int, default=0, help="0 means source motion length") parser.add_argument("--num-inference-steps", type=int, default=10) parser.add_argument("--device", type=str, default="cuda:0") parser.add_argument("--seed", type=int, default=42) parser.add_argument("--export-delta", action="store_true", default=False) parser.add_argument("--disable-rtc", action="store_true", default=False) parser.add_argument("--no-align-start-hold", dest="align_start_hold", action="store_false") parser.set_defaults(align_start_hold=True) return parser.parse_args() def main() -> None: args = parse_args() run_dir = Path(args.run_dir) output_dir = Path(args.output_dir) output_dir.mkdir(parents=True, exist_ok=True) cfg_template: LaunchConfig = parse_args_to_tyro_config(run_dir / "argv.txt") # type: ignore launch_config = cfg_template.model_validate_json((run_dir / "run_config.json").read_text()) seed_everything(launch_config.seed or args.seed) ckpt_step = resolve_ckpt_step(run_dir, args.ckpt_step) model = Psi0Model.from_pretrained(run_dir, ckpt_step, launch_config, device=args.device) model.text_only = False model.audio_only = True model.video_only = False model.to(args.device) model.eval() audio_cfg = audio_data_cfg(launch_config, args.audio_config_key) data_root = Path(cfg_get(audio_cfg, "root_dir")) video_root = Path(cfg_get(audio_cfg, "video_dir", "")) motion_dir = resolve_motion_dir(data_root, args.motion_dir) video_path = resolve_audio_path( resolve_video_path(video_root, motion_dir, args.video_path), str(cfg_get(audio_cfg, "audio_file_extensions", ".wav,.flac,.mp3,.m4a,.aac,.ogg")), ) if not motion_dir.exists(): raise FileNotFoundError(f"motion_dir does not exist: {motion_dir}") if not video_path.exists(): raise FileNotFoundError(f"video_path does not exist: {video_path}") maxmin = launch_config.data.transform.field odim = int(launch_config.model.odim) textop_actions = bool(getattr(launch_config.data.transform.repack, "textop_actions", False)) decode_textop_joint_from_delta = textop_actions and should_decode_textop_joint_from_delta(launch_config) num_past_frames = int(getattr(launch_config.data.transform.repack, "num_past_frames", 0)) use_future_progress_embedding = bool(getattr(launch_config.model, "use_future_progress_embedding", False)) enable_rtc = bool(getattr(launch_config.model, "rtc", False)) and (not args.disable_rtc) action_exec_horizon = int(getattr(launch_config.model, "action_exec_horizon", int(model.action_horizon))) rtc_max_delay = int(getattr(launch_config.model, "max_delay", 0)) audio_sampling_rate = int(cfg_get(audio_cfg, "audio_sampling_rate", 16000)) audio_window_seconds = float(cfg_get(audio_cfg, "audio_window_seconds", 1.0)) audio_left_context_seconds = float(cfg_get(audio_cfg, "audio_left_context_seconds", 0.0)) full_audio = bool(cfg_get(audio_cfg, "full_audio", False)) audio = load_audio_array(video_path, audio_sampling_rate) instruction = args.prompt.strip() x0 = load_pose_from_motion_dir(motion_dir) source_frames = load_source_frame_count(motion_dir) target_frames = int(args.num_output_frames) if args.num_output_frames > 0 else int(source_frames or 0) start_hold_seconds = read_start_hold_seconds(motion_dir, bool(args.align_start_hold)) pred_abs, pred_delta, rollout_chunks = rollout_audio_motion( model=model, maxmin=maxmin, odim=odim, device=args.device, instruction=instruction, audio=audio, audio_sampling_rate=audio_sampling_rate, audio_window_seconds=audio_window_seconds, audio_left_context_seconds=audio_left_context_seconds, full_audio=full_audio, start_hold_seconds=start_hold_seconds, normalize_state=bool(getattr(maxmin, "normalize_state", False)), textop_actions=textop_actions, decode_textop_joint_from_delta=decode_textop_joint_from_delta, num_past_frames=num_past_frames, use_future_progress_embedding=use_future_progress_embedding, enable_rtc=enable_rtc, action_exec_horizon=action_exec_horizon, rtc_max_delay=rtc_max_delay, num_inference_steps=int(args.num_inference_steps), target_frames=target_frames, init_abs_state=x0, ) motion_name = sanitize_motion_name(motion_dir.name) sample_dir = output_dir / motion_name sample_dir.mkdir(parents=True, exist_ok=True) joint_pos = pred_abs[:, :29] body_pos = pred_abs[:, 29:32] body_quat = pred_abs[:, 32:36] dt = 1.0 / float(TARGET_FPS) joint_vel = finite_difference(joint_pos, dt) body_lin_vel = finite_difference(body_pos, dt) body_ang_vel = quaternion_angular_velocity(body_quat, dt) write_csv(sample_dir / "joint_pos.csv", joint_pos, [f"joint_{i}" for i in range(29)]) write_csv(sample_dir / "joint_vel.csv", joint_vel, [f"joint_vel_{i}" for i in range(29)]) write_csv(sample_dir / "body_pos.csv", body_pos, ["body_0_x", "body_0_y", "body_0_z"]) write_csv(sample_dir / "body_quat.csv", body_quat, ["body_0_w", "body_0_x", "body_0_y", "body_0_z"]) write_csv(sample_dir / "body_lin_vel.csv", body_lin_vel, ["body_0_vel_x", "body_0_vel_y", "body_0_vel_z"]) write_csv(sample_dir / "body_ang_vel.csv", body_ang_vel, ["body_0_angvel_x", "body_0_angvel_y", "body_0_angvel_z"]) np.save(sample_dir / "motion.npy", pred_abs) if args.export_delta: write_csv(sample_dir / "joint_pos_delta.csv", pred_delta[:, :29], [f"joint_{i}" for i in range(29)]) write_csv(sample_dir / "body_pos_delta.csv", pred_delta[:, 29:32], ["body_0_x", "body_0_y", "body_0_z"]) write_csv(sample_dir / "body_quat_delta.csv", pred_delta[:, 32:36], ["body_0_w", "body_0_x", "body_0_y", "body_0_z"]) np.save(sample_dir / "motion_delta.npy", pred_delta) write_metadata(sample_dir / "metadata.txt", motion_name, int(pred_abs.shape[0])) write_audio_info( sample_dir / "info.txt", audio_config_key=str(args.audio_config_key), source_motion_dir=motion_dir, video_path=video_path, initial_pose_source=str(motion_dir), frames=int(pred_abs.shape[0]), textop_actions=textop_actions, num_past_frames=int(num_past_frames), enable_rtc=enable_rtc, action_exec_horizon=int(action_exec_horizon), num_inference_steps=int(args.num_inference_steps), audio_sampling_rate=int(audio_sampling_rate), audio_window_seconds=float(audio_window_seconds), full_audio=bool(full_audio), instruction=instruction, ) for src_name, dst_name in (("info.txt", "info_source.txt"), ("metadata.txt", "metadata_source.txt")): src_path = motion_dir / src_name if src_path.exists(): shutil.copy2(src_path, sample_dir / dst_name) sample_meta = { "run_dir": str(run_dir), "ckpt_step": ckpt_step, "audio_config_key": str(args.audio_config_key), "instruction": instruction, "source_motion_dir": str(motion_dir), "video_path": str(video_path), "output_dir": str(sample_dir), "num_output_frames": int(pred_abs.shape[0]), "rollout_chunks": int(rollout_chunks), "textop_actions": textop_actions, "decode_textop_joint_from_delta": decode_textop_joint_from_delta, "num_past_frames": int(num_past_frames), "enable_rtc": enable_rtc, "action_exec_horizon": int(action_exec_horizon), "audio_sampling_rate": int(audio_sampling_rate), "full_audio": bool(full_audio), "audio_window_seconds": float(audio_window_seconds), "audio_left_context_seconds": float(audio_left_context_seconds), "start_hold_seconds": float(start_hold_seconds), } (sample_dir / "infer_meta.json").write_text(json.dumps(sample_meta, ensure_ascii=False, indent=2), encoding="utf-8") (output_dir / "manifest.json").write_text( json.dumps({"samples": [sample_meta]}, ensure_ascii=False, indent=2), encoding="utf-8", ) print(f"[OK] audio sample -> {sample_dir}") print(f"[DONE] wrote manifest: {output_dir / 'manifest.json'}") if __name__ == "__main__": main()