code-Psi0-yth-v21 / scripts /infer /seed_audio_to_motion.py
Tinghong-Ye22's picture
Add files using upload-large-folder tool
c4f8e4a verified
Raw History Blame Contribute Delete
22.3 kB
#!/usr/bin/env python3
from __future__ import annotations
import argparse
import json
import math
import random
import re
import shutil
import subprocess
from pathlib import Path
from typing import List
import numpy as np
import torch
from psi.config.config import LaunchConfig
from psi.data.seed.motion_repr import quat_inv_wxyz, quat_mul_wxyz, recover_seed_abs_from_textop_action
from psi.data.seed.progress import build_progress_conditions
from psi.models.psi0 import Psi0Model
from psi.utils import parse_args_to_tyro_config, seed_everything
from seed_text_to_motion import (
TARGET_FPS,
build_model_states_from_abs,
finite_difference,
load_pose_from_motion_dir,
load_source_frame_count,
normalize_quaternion_rows,
quaternion_angular_velocity,
recover_seed_abs_from_textop_action_with_joint_delta,
resolve_ckpt_step,
sanitize_motion_name,
should_decode_textop_joint_from_delta,
write_csv,
write_metadata,
)
def cfg_get(obj, key: str, default=None):
if isinstance(obj, dict):
return obj.get(key, default)
return getattr(obj, key, default)
def audio_data_cfg(launch_config: LaunchConfig, key: str = "audio"):
return cfg_get(launch_config.data, key, launch_config.data)
def load_audio_array(video_path: Path, sampling_rate: int) -> np.ndarray:
cmd = [
"ffmpeg",
"-v",
"error",
"-i",
str(video_path),
"-ac",
"1",
"-ar",
str(sampling_rate),
"-f",
"f32le",
"-",
]
proc = subprocess.run(cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE, check=False)
if proc.returncode != 0:
msg = proc.stderr.decode("utf-8", errors="ignore")
raise RuntimeError(f"ffmpeg failed to decode audio from {video_path}: {msg}")
audio = np.frombuffer(proc.stdout, dtype=np.float32).copy()
if audio.size == 0:
raise RuntimeError(f"No audio samples decoded from {video_path}")
return audio
def audio_segment(audio: np.ndarray, sampling_rate: int, start_sec: float, duration_sec: float) -> np.ndarray:
num_samples = max(1, int(round(duration_sec * sampling_rate)))
start_sample = int(round(start_sec * sampling_rate))
end_sample = start_sample + num_samples
segment = np.zeros((num_samples,), dtype=np.float32)
src_start = max(0, start_sample)
src_end = min(audio.shape[0], end_sample)
if src_end > src_start:
dst_start = src_start - start_sample
segment[dst_start : dst_start + (src_end - src_start)] = audio[src_start:src_end]
return segment
def read_start_hold_seconds(motion_dir: Path, align_start_hold: bool) -> float:
if not align_start_hold:
return 0.0
info_path = motion_dir / "info.txt"
if not info_path.exists():
return 0.0
text = info_path.read_text(encoding="utf-8", errors="ignore")
match = re.search(r"start_hold_seconds:\s*([0-9.]+)", text)
return float(match.group(1)) if match else 0.0
def resolve_motion_dir(root_dir: Path, motion_dir_arg: str) -> Path:
motion_dir = Path(motion_dir_arg).expanduser()
if motion_dir.is_absolute():
return motion_dir
return root_dir / motion_dir
def resolve_video_path(video_dir: Path, motion_dir: Path, video_path_arg: str) -> Path:
if video_path_arg:
video_path = Path(video_path_arg).expanduser()
if video_path.is_absolute():
return video_path
return video_dir / video_path
return video_dir / f"{motion_dir.name}.mp4"
def resolve_audio_path(audio_path: Path, extensions: str = ".wav,.flac,.mp3,.m4a,.aac,.ogg") -> Path:
if audio_path.is_file():
return audio_path
if not audio_path.is_dir():
return audio_path
audio_exts = tuple(
ext if ext.startswith(".") else f".{ext}"
for ext in [part.strip().lower() for part in str(extensions).split(",")]
if ext
)
candidates = sorted(p for p in audio_path.iterdir() if p.is_file() and p.suffix.lower() in audio_exts)
if not candidates:
raise FileNotFoundError(f"No audio files found under {audio_path}")
return random.choice(candidates)
def normalize_action_for_rtc(maxmin, action: np.ndarray) -> np.ndarray:
action_min = np.asarray(maxmin.action_min, dtype=np.float32)
action_max = np.asarray(maxmin.action_max, dtype=np.float32)
ill_mask = np.abs(action_max - action_min) < 1e-4 * (np.abs(action_max) + np.abs(action_min) + 1e-8)
action_max = action_max.copy()
action_max[ill_mask] = 1.0
action_norm = np.where(ill_mask, action, (action - action_min) / (action_max - action_min) * 2 - 1)
if bool(getattr(maxmin, "use_norm_mask", False)):
action_norm_masks = np.asarray(maxmin.action_norm_masks, dtype=bool)
action_norm = np.where(action_norm_masks, action_norm, action)
return np.clip(action_norm, -1.0, 1.0).astype(np.float32)
def rollout_audio_motion(
*,
model: Psi0Model,
maxmin,
odim: int,
device: str,
instruction: str,
audio: np.ndarray,
audio_sampling_rate: int,
audio_window_seconds: float,
audio_left_context_seconds: float,
full_audio: bool,
start_hold_seconds: float,
normalize_state: bool,
textop_actions: bool,
decode_textop_joint_from_delta: bool,
num_past_frames: int,
use_future_progress_embedding: bool,
enable_rtc: bool,
action_exec_horizon: int,
rtc_max_delay: int,
num_inference_steps: int,
target_frames: int,
init_abs_state: np.ndarray,
) -> tuple[np.ndarray, np.ndarray, int]:
horizon = int(model.action_horizon)
exec_horizon = int(action_exec_horizon) if enable_rtc else horizon
if exec_horizon <= 0 or exec_horizon > horizon:
raise ValueError(f"Invalid action_exec_horizon={exec_horizon}, action_horizon={horizon}")
rtc_delay = horizon - exec_horizon
if enable_rtc and rtc_delay > 0 and rtc_delay >= int(rtc_max_delay):
raise ValueError(
f"RTC requires action_horizon - action_exec_horizon < max_delay, "
f"got {horizon} - {exec_horizon} = {rtc_delay}, max_delay={rtc_max_delay}"
)
if not textop_actions:
raise ValueError("This audio inference path is for the 39D TextOp action checkpoint.")
if target_frames <= 0:
target_frames = max(exec_horizon, int(round(audio.shape[0] / float(audio_sampling_rate) * TARGET_FPS)))
num_chunks = int(math.ceil(float(target_frames) / float(exec_horizon)))
current_abs = np.asarray(init_abs_state, dtype=np.float32).copy()
state_horizon = int(num_past_frames) + 1
state_history: List[np.ndarray] = [current_abs.copy() for _ in range(state_horizon)]
all_abs: List[np.ndarray] = []
all_delta: List[np.ndarray] = []
prev_actions_norm: np.ndarray | None = None
remaining_frames = int(target_frames)
for _ in range(num_chunks):
frame_cursor = target_frames - remaining_frames
states = build_model_states_from_abs(
state_history[-state_horizon:],
textop_actions=textop_actions,
odim=odim,
)
if normalize_state:
states = maxmin.normalize_state_func(states)
states_t = torch.from_numpy(states).unsqueeze(0).to(device)
future_progress_t = None
if use_future_progress_embedding:
_, future_progress = build_progress_conditions(
frame_id=frame_cursor,
total_frames=target_frames,
action_horizon=horizon,
)
future_progress_t = torch.from_numpy(future_progress[None]).to(device)
# <modify_full_audio> 和训练一致:full_audio=True时每个chunk都看到整段音频embedding,不再给time/progress。
if full_audio:
audio_clip = audio.astype(np.float32)
else:
audio_start_sec = (
frame_cursor / float(TARGET_FPS)
- float(start_hold_seconds)
- float(audio_left_context_seconds)
)
audio_clip = audio_segment(audio, audio_sampling_rate, audio_start_sec, audio_window_seconds)
with torch.inference_mode():
if enable_rtc and prev_actions_norm is not None and rtc_delay > 0:
prev_rtc = np.concatenate(
[
prev_actions_norm[exec_horizon:, :],
np.zeros((exec_horizon, prev_actions_norm.shape[1]), dtype=np.float32),
],
axis=0,
)
prev_actions_t = torch.from_numpy(prev_rtc[None]).to(device)
pred_norm = model.predict_action_with_training_rtc_flow(
observations=None,
states=states_t,
instructions=[instruction],
num_inference_steps=num_inference_steps,
traj2ds=None,
prev_actions=prev_actions_t,
inference_delay=rtc_delay,
max_delay=int(rtc_max_delay),
audios=[audio_clip],
future_progress=future_progress_t,
)[0].float().cpu().numpy()
else:
pred_norm = model.predict_action(
observations=None,
states=states_t,
instructions=[instruction],
num_inference_steps=num_inference_steps,
traj2ds=None,
audios=[audio_clip],
future_progress=future_progress_t,
)[0].float().cpu().numpy()
pred_norm_clipped = np.clip(np.asarray(pred_norm, dtype=np.float32), -1.0, 1.0)
chunk_pred = np.asarray(maxmin.denormalize(pred_norm_clipped), dtype=np.float32)
if decode_textop_joint_from_delta:
chunk_abs = recover_seed_abs_from_textop_action_with_joint_delta(chunk_pred, current_abs)
else:
chunk_abs = recover_seed_abs_from_textop_action(chunk_pred, current_abs)
prev_abs = np.concatenate([current_abs[None, :], chunk_abs[:-1]], axis=0)
joint_delta = chunk_abs[:, :29] - prev_abs[:, :29]
root_pos_delta = chunk_abs[:, 29:32] - prev_abs[:, 29:32]
root_quat_delta = quat_mul_wxyz(quat_inv_wxyz(prev_abs[:, 32:36]), chunk_abs[:, 32:36])
chunk_delta = np.concatenate([joint_delta, root_pos_delta, root_quat_delta], axis=1)
chunk_abs = normalize_quaternion_rows(chunk_abs.astype(np.float32))
if decode_textop_joint_from_delta:
chunk_pred_for_rtc = chunk_pred.copy()
chunk_pred_for_rtc[:, :29] = chunk_abs[:, :29]
prev_actions_norm = normalize_action_for_rtc(maxmin, chunk_pred_for_rtc)
else:
prev_actions_norm = pred_norm_clipped.copy()
take = min(exec_horizon, remaining_frames)
if take <= 0:
break
executed_abs = chunk_abs[:take]
executed_delta = chunk_delta[:take].astype(np.float32)
current_abs = executed_abs[-1].copy()
for frame_abs in executed_abs:
state_history.append(frame_abs.astype(np.float32).copy())
state_history = state_history[-state_horizon:]
all_abs.append(executed_abs)
all_delta.append(executed_delta)
remaining_frames -= take
pred_abs = np.concatenate(all_abs, axis=0)[:target_frames]
pred_delta = np.concatenate(all_delta, axis=0)[:target_frames]
pred_abs = normalize_quaternion_rows(pred_abs.astype(np.float32))
return pred_abs, pred_delta, num_chunks
def write_audio_info(
path: Path,
*,
audio_config_key: str,
source_motion_dir: Path,
video_path: Path,
initial_pose_source: str,
frames: int,
textop_actions: bool,
num_past_frames: int,
enable_rtc: bool,
action_exec_horizon: int,
num_inference_steps: int,
audio_sampling_rate: int,
audio_window_seconds: float,
full_audio: bool,
instruction: str,
) -> None:
lines = [
"Motion Information: seed_audio_to_motion_inference",
"===================================================",
"",
f"audio_config_key: {audio_config_key}",
f"source_motion_dir: {source_motion_dir}",
f"video_path: {video_path}",
f"instruction: {instruction}",
f"initial_pose_source: {initial_pose_source}",
f"target_fps: {TARGET_FPS}",
f"num_inference_steps: {num_inference_steps}",
f"trained_with_textop_actions: {textop_actions}",
f"num_past_frames: {num_past_frames}",
f"rtc_enabled: {enable_rtc}",
f"action_exec_horizon: {action_exec_horizon}",
f"audio_sampling_rate: {audio_sampling_rate}",
f"full_audio: {full_audio}",
f"audio_window_seconds: {audio_window_seconds}",
"output_root_quaternion: wxyz",
"output_joint_angles: 29D",
"output_root_position: xyz",
"",
f"frames: {frames}",
"",
]
path.write_text("\n".join(lines), encoding="utf-8")
def parse_args() -> argparse.Namespace:
parser = argparse.ArgumentParser(description="Seed audio->motion offline inference exporter")
parser.add_argument("--run-dir", type=str, required=True)
parser.add_argument("--ckpt-step", type=str, default="latest", help="integer step, 'latest', or 'best'")
parser.add_argument("--output-dir", type=str, required=True)
parser.add_argument("--motion-dir", type=str, required=True)
parser.add_argument("--video-path", type=str, default="")
parser.add_argument(
"--audio-config-key",
type=str,
default="audio",
choices=("audio", "audio_command"),
help="Which launch_config.data audio section to use",
)
parser.add_argument("--prompt", type=str, default="", help="Optional text instruction paired with the audio")
parser.add_argument("--num-output-frames", type=int, default=0, help="0 means source motion length")
parser.add_argument("--num-inference-steps", type=int, default=10)
parser.add_argument("--device", type=str, default="cuda:0")
parser.add_argument("--seed", type=int, default=42)
parser.add_argument("--export-delta", action="store_true", default=False)
parser.add_argument("--disable-rtc", action="store_true", default=False)
parser.add_argument("--no-align-start-hold", dest="align_start_hold", action="store_false")
parser.set_defaults(align_start_hold=True)
return parser.parse_args()
def main() -> None:
args = parse_args()
run_dir = Path(args.run_dir)
output_dir = Path(args.output_dir)
output_dir.mkdir(parents=True, exist_ok=True)
cfg_template: LaunchConfig = parse_args_to_tyro_config(run_dir / "argv.txt") # type: ignore
launch_config = cfg_template.model_validate_json((run_dir / "run_config.json").read_text())
seed_everything(launch_config.seed or args.seed)
ckpt_step = resolve_ckpt_step(run_dir, args.ckpt_step)
model = Psi0Model.from_pretrained(run_dir, ckpt_step, launch_config, device=args.device)
model.text_only = False
model.audio_only = True
model.video_only = False
model.to(args.device)
model.eval()
audio_cfg = audio_data_cfg(launch_config, args.audio_config_key)
data_root = Path(cfg_get(audio_cfg, "root_dir"))
video_root = Path(cfg_get(audio_cfg, "video_dir", ""))
motion_dir = resolve_motion_dir(data_root, args.motion_dir)
video_path = resolve_audio_path(
resolve_video_path(video_root, motion_dir, args.video_path),
str(cfg_get(audio_cfg, "audio_file_extensions", ".wav,.flac,.mp3,.m4a,.aac,.ogg")),
)
if not motion_dir.exists():
raise FileNotFoundError(f"motion_dir does not exist: {motion_dir}")
if not video_path.exists():
raise FileNotFoundError(f"video_path does not exist: {video_path}")
maxmin = launch_config.data.transform.field
odim = int(launch_config.model.odim)
textop_actions = bool(getattr(launch_config.data.transform.repack, "textop_actions", False))
decode_textop_joint_from_delta = textop_actions and should_decode_textop_joint_from_delta(launch_config)
num_past_frames = int(getattr(launch_config.data.transform.repack, "num_past_frames", 0))
use_future_progress_embedding = bool(getattr(launch_config.model, "use_future_progress_embedding", False))
enable_rtc = bool(getattr(launch_config.model, "rtc", False)) and (not args.disable_rtc)
action_exec_horizon = int(getattr(launch_config.model, "action_exec_horizon", int(model.action_horizon)))
rtc_max_delay = int(getattr(launch_config.model, "max_delay", 0))
audio_sampling_rate = int(cfg_get(audio_cfg, "audio_sampling_rate", 16000))
audio_window_seconds = float(cfg_get(audio_cfg, "audio_window_seconds", 1.0))
audio_left_context_seconds = float(cfg_get(audio_cfg, "audio_left_context_seconds", 0.0))
full_audio = bool(cfg_get(audio_cfg, "full_audio", False))
audio = load_audio_array(video_path, audio_sampling_rate)
instruction = args.prompt.strip()
x0 = load_pose_from_motion_dir(motion_dir)
source_frames = load_source_frame_count(motion_dir)
target_frames = int(args.num_output_frames) if args.num_output_frames > 0 else int(source_frames or 0)
start_hold_seconds = read_start_hold_seconds(motion_dir, bool(args.align_start_hold))
pred_abs, pred_delta, rollout_chunks = rollout_audio_motion(
model=model,
maxmin=maxmin,
odim=odim,
device=args.device,
instruction=instruction,
audio=audio,
audio_sampling_rate=audio_sampling_rate,
audio_window_seconds=audio_window_seconds,
audio_left_context_seconds=audio_left_context_seconds,
full_audio=full_audio,
start_hold_seconds=start_hold_seconds,
normalize_state=bool(getattr(maxmin, "normalize_state", False)),
textop_actions=textop_actions,
decode_textop_joint_from_delta=decode_textop_joint_from_delta,
num_past_frames=num_past_frames,
use_future_progress_embedding=use_future_progress_embedding,
enable_rtc=enable_rtc,
action_exec_horizon=action_exec_horizon,
rtc_max_delay=rtc_max_delay,
num_inference_steps=int(args.num_inference_steps),
target_frames=target_frames,
init_abs_state=x0,
)
motion_name = sanitize_motion_name(motion_dir.name)
sample_dir = output_dir / motion_name
sample_dir.mkdir(parents=True, exist_ok=True)
joint_pos = pred_abs[:, :29]
body_pos = pred_abs[:, 29:32]
body_quat = pred_abs[:, 32:36]
dt = 1.0 / float(TARGET_FPS)
joint_vel = finite_difference(joint_pos, dt)
body_lin_vel = finite_difference(body_pos, dt)
body_ang_vel = quaternion_angular_velocity(body_quat, dt)
write_csv(sample_dir / "joint_pos.csv", joint_pos, [f"joint_{i}" for i in range(29)])
write_csv(sample_dir / "joint_vel.csv", joint_vel, [f"joint_vel_{i}" for i in range(29)])
write_csv(sample_dir / "body_pos.csv", body_pos, ["body_0_x", "body_0_y", "body_0_z"])
write_csv(sample_dir / "body_quat.csv", body_quat, ["body_0_w", "body_0_x", "body_0_y", "body_0_z"])
write_csv(sample_dir / "body_lin_vel.csv", body_lin_vel, ["body_0_vel_x", "body_0_vel_y", "body_0_vel_z"])
write_csv(sample_dir / "body_ang_vel.csv", body_ang_vel, ["body_0_angvel_x", "body_0_angvel_y", "body_0_angvel_z"])
np.save(sample_dir / "motion.npy", pred_abs)
if args.export_delta:
write_csv(sample_dir / "joint_pos_delta.csv", pred_delta[:, :29], [f"joint_{i}" for i in range(29)])
write_csv(sample_dir / "body_pos_delta.csv", pred_delta[:, 29:32], ["body_0_x", "body_0_y", "body_0_z"])
write_csv(sample_dir / "body_quat_delta.csv", pred_delta[:, 32:36], ["body_0_w", "body_0_x", "body_0_y", "body_0_z"])
np.save(sample_dir / "motion_delta.npy", pred_delta)
write_metadata(sample_dir / "metadata.txt", motion_name, int(pred_abs.shape[0]))
write_audio_info(
sample_dir / "info.txt",
audio_config_key=str(args.audio_config_key),
source_motion_dir=motion_dir,
video_path=video_path,
initial_pose_source=str(motion_dir),
frames=int(pred_abs.shape[0]),
textop_actions=textop_actions,
num_past_frames=int(num_past_frames),
enable_rtc=enable_rtc,
action_exec_horizon=int(action_exec_horizon),
num_inference_steps=int(args.num_inference_steps),
audio_sampling_rate=int(audio_sampling_rate),
audio_window_seconds=float(audio_window_seconds),
full_audio=bool(full_audio),
instruction=instruction,
)
for src_name, dst_name in (("info.txt", "info_source.txt"), ("metadata.txt", "metadata_source.txt")):
src_path = motion_dir / src_name
if src_path.exists():
shutil.copy2(src_path, sample_dir / dst_name)
sample_meta = {
"run_dir": str(run_dir),
"ckpt_step": ckpt_step,
"audio_config_key": str(args.audio_config_key),
"instruction": instruction,
"source_motion_dir": str(motion_dir),
"video_path": str(video_path),
"output_dir": str(sample_dir),
"num_output_frames": int(pred_abs.shape[0]),
"rollout_chunks": int(rollout_chunks),
"textop_actions": textop_actions,
"decode_textop_joint_from_delta": decode_textop_joint_from_delta,
"num_past_frames": int(num_past_frames),
"enable_rtc": enable_rtc,
"action_exec_horizon": int(action_exec_horizon),
"audio_sampling_rate": int(audio_sampling_rate),
"full_audio": bool(full_audio),
"audio_window_seconds": float(audio_window_seconds),
"audio_left_context_seconds": float(audio_left_context_seconds),
"start_hold_seconds": float(start_hold_seconds),
}
(sample_dir / "infer_meta.json").write_text(json.dumps(sample_meta, ensure_ascii=False, indent=2), encoding="utf-8")
(output_dir / "manifest.json").write_text(
json.dumps({"samples": [sample_meta]}, ensure_ascii=False, indent=2),
encoding="utf-8",
)
print(f"[OK] audio sample -> {sample_dir}")
print(f"[DONE] wrote manifest: {output_dir / 'manifest.json'}")
if __name__ == "__main__":
main()