Download scripts/infer/seed_text_to_motion.py from Tinghong-Ye22/code-Psi0-yth-v21: direct link, hf CLI and curl.
- Browser
- Download file 38.7 kB
-
https://huggingface.co/Tinghong-Ye22/code-Psi0-yth-v21/resolve/main/scripts/infer/seed_text_to_motion.py
- Command line
-
hf download hf://Tinghong-Ye22/code-Psi0-yth-v21/scripts/infer/seed_text_to_motion.py
-
curl -L -o seed_text_to_motion.py https://huggingface.co/Tinghong-Ye22/code-Psi0-yth-v21/resolve/main/scripts/infer/seed_text_to_motion.py
38.7 kB
| #!/usr/bin/env python3 | |
| from __future__ import annotations | |
| import argparse | |
| import json | |
| import math | |
| import shutil | |
| from pathlib import Path | |
| from typing import List | |
| import numpy as np | |
| import pandas as pd | |
| import torch | |
| from psi.config.config import LaunchConfig | |
| from psi.data.seed.motion_repr import ( | |
| SEED_ABS_DIM, | |
| SEED_TEXTOP_ACTION_DIM, | |
| TEXTOP_JOINT_DELTA, | |
| TEXTOP_ROOT_DELTA_LOCAL, | |
| TEXTOP_ROOT_HEIGHT, | |
| TEXTOP_ROOT_ROT6D_DELTA, | |
| quat_inv_wxyz, | |
| quat_mul_wxyz, | |
| build_seed_textop_state, | |
| recover_seed_abs_from_textop_action, | |
| rot6d_delta_to_quat, | |
| world_delta_from_local, | |
| ) | |
| from psi.data.seed.progress import build_progress_conditions | |
| from psi.data.seed.text_prompt import ( | |
| DEFAULT_SEED_TEXT_FIELDS, | |
| compose_seed_motion_prompt, | |
| compose_seed_motion_prompt_from_row, | |
| ) | |
| from psi.models.psi0 import Psi0Model | |
| from psi.utils import parse_args_to_tyro_config, seed_everything | |
| DEFAULT_TEXT_FIELDS: List[str] = list(DEFAULT_SEED_TEXT_FIELDS) | |
| TARGET_FPS = 50 | |
| def compose_instruction(row: pd.Series, text_fields: List[str]) -> str: | |
| return compose_seed_motion_prompt_from_row(row, text_fields) | |
| def map_motion_dir(root_dir: Path, move_g1_path: str) -> Path: | |
| rel = str(move_g1_path).strip().replace("\\", "/") | |
| rel = rel.replace("g1/csv/", "g1/csv_sonic/") | |
| rel = rel.replace("/csv/", "/csv_sonic/") | |
| if rel.endswith(".csv"): | |
| rel = rel[:-4] | |
| if rel.startswith("dataset/seed/"): | |
| rel = rel[len("dataset/seed/") :] | |
| if rel.startswith("seed/"): | |
| rel = rel[len("seed/") :] | |
| p = Path(rel) | |
| if p.is_absolute(): | |
| return p | |
| return root_dir / p | |
| def sanitize_motion_name(name: str) -> str: | |
| out = "".join(ch if (ch.isalnum() or ch in {"_", "-", "."}) else "_" for ch in name.strip()) | |
| out = out.strip("._") | |
| return out or "manual_prompt" | |
| def normalize_quaternion_rows(motion_abs: np.ndarray) -> np.ndarray: | |
| if motion_abs.shape[1] < 36: | |
| return motion_abs | |
| q = motion_abs[:, 32:36] | |
| norm = np.linalg.norm(q, axis=1, keepdims=True) | |
| norm = np.where(norm < 1e-8, 1.0, norm) | |
| motion_abs[:, 32:36] = q / norm | |
| return motion_abs | |
| def quat_multiply(q1: np.ndarray, q2: np.ndarray) -> np.ndarray: | |
| w1, x1, y1, z1 = q1 | |
| w2, x2, y2, z2 = q2 | |
| return np.array( | |
| [ | |
| w1 * w2 - x1 * x2 - y1 * y2 - z1 * z2, | |
| w1 * x2 + x1 * w2 + y1 * z2 - z1 * y2, | |
| w1 * y2 - x1 * z2 + y1 * w2 + z1 * x2, | |
| w1 * z2 + x1 * y2 - y1 * x2 + z1 * w2, | |
| ], | |
| dtype=np.float64, | |
| ) | |
| def quat_conjugate(q: np.ndarray) -> np.ndarray: | |
| return np.array([q[0], -q[1], -q[2], -q[3]], dtype=np.float64) | |
| def quat_to_rotvec(q: np.ndarray) -> np.ndarray: | |
| q = q / max(np.linalg.norm(q), 1e-12) | |
| if q[0] < 0.0: | |
| q = -q | |
| vec = q[1:] | |
| sin_half = np.linalg.norm(vec) | |
| if sin_half < 1e-10: | |
| return np.zeros(3, dtype=np.float64) | |
| axis = vec / sin_half | |
| angle = 2.0 * np.arctan2(sin_half, q[0]) | |
| return axis * angle | |
| def finite_difference(data: np.ndarray, dt: float) -> np.ndarray: | |
| if data.shape[0] < 2: | |
| return np.zeros_like(data, dtype=np.float64) | |
| edge_order = 2 if data.shape[0] > 2 else 1 | |
| return np.gradient(data, dt, axis=0, edge_order=edge_order) | |
| def quaternion_angular_velocity(quat_wxyz: np.ndarray, dt: float) -> np.ndarray: | |
| ang_vel = np.zeros((quat_wxyz.shape[0], 3), dtype=np.float64) | |
| if quat_wxyz.shape[0] < 2: | |
| return ang_vel | |
| for i in range(quat_wxyz.shape[0] - 1): | |
| dq = quat_multiply(quat_wxyz[i + 1], quat_conjugate(quat_wxyz[i])) | |
| ang_vel[i] = quat_to_rotvec(dq) / dt | |
| ang_vel[-1] = ang_vel[-2] | |
| return ang_vel | |
| def write_csv(path: Path, array: np.ndarray, headers: List[str]) -> None: | |
| np.savetxt( | |
| path, | |
| np.asarray(array, dtype=np.float64), | |
| delimiter=",", | |
| header=",".join(headers), | |
| comments="", | |
| fmt="%.6f", | |
| ) | |
| def write_metadata(path: Path, motion_name: str, timesteps: int) -> None: | |
| path.write_text( | |
| "\n".join( | |
| [ | |
| f"Metadata for: {motion_name}", | |
| "==============================", | |
| "", | |
| "Body part indexes:", | |
| "[0]", | |
| "", | |
| f"Total timesteps: {timesteps}", | |
| "", | |
| ] | |
| ), | |
| encoding="utf-8", | |
| ) | |
| def write_info( | |
| path: Path, | |
| *, | |
| source_path: str, | |
| initial_pose_source: str, | |
| frames: int, | |
| use_delta_actions: bool, | |
| root_delta_actions: bool, | |
| textop_actions: bool, | |
| num_past_frames: int, | |
| enable_rtc: bool, | |
| action_exec_horizon: int, | |
| num_inference_steps: int, | |
| ) -> None: | |
| lines = [ | |
| "Motion Information: seed_text_to_motion_inference", | |
| "==================================================", | |
| "", | |
| f"source_path: {source_path}", | |
| f"initial_pose_source: {initial_pose_source}", | |
| f"target_fps: {TARGET_FPS}", | |
| f"num_inference_steps: {num_inference_steps}", | |
| f"trained_with_delta_actions: {use_delta_actions}", | |
| f"trained_with_root_delta_actions: {root_delta_actions}", | |
| f"trained_with_textop_actions: {textop_actions}", | |
| f"num_past_frames: {num_past_frames}", | |
| f"rtc_enabled: {enable_rtc}", | |
| f"action_exec_horizon: {action_exec_horizon}", | |
| "output_root_quaternion: wxyz", | |
| "output_joint_angles: 29D", | |
| "output_root_position: xyz", | |
| "", | |
| f"frames: {frames}", | |
| "", | |
| ] | |
| path.write_text("\n".join(lines), encoding="utf-8") | |
| def load_pose_from_motion_dir(motion_dir: Path) -> np.ndarray: | |
| joint = pd.read_csv(motion_dir / "joint_pos.csv").to_numpy(dtype=np.float32) | |
| body_pos = pd.read_csv(motion_dir / "body_pos.csv").to_numpy(dtype=np.float32) | |
| body_quat = pd.read_csv(motion_dir / "body_quat.csv").to_numpy(dtype=np.float32) | |
| return np.concatenate([joint[0], body_pos[0], body_quat[0]], axis=0).astype(np.float32) | |
| def load_pose_from_npy(path: Path) -> np.ndarray: | |
| pose = np.load(path).astype(np.float32) | |
| if pose.ndim == 2: | |
| pose = pose[0] | |
| if pose.shape != (36,): | |
| raise ValueError(f"Expected init pose shape (36,) or (T,36), got {pose.shape} from {path}") | |
| return pose.astype(np.float32) | |
| def resolve_motion_dir(data_root: Path, motion_dir_arg: str) -> Path: | |
| motion_dir = Path(motion_dir_arg).expanduser() | |
| if motion_dir.is_absolute(): | |
| return motion_dir | |
| return data_root / motion_dir | |
| def load_initial_pose(root_dir: Path, row: pd.Series, restore_from_data: bool, strict: bool = False) -> np.ndarray: | |
| if not restore_from_data: | |
| if strict: | |
| # <modify_jerk> root-delta也必须有真实首帧,否则root轨迹会从zero累加跑偏。 | |
| raise ValueError("Delta/root-delta inference requires a real initial pose. Do not use --no-restore-from-data.") | |
| return np.zeros((36,), dtype=np.float32) | |
| try: | |
| motion_dir = map_motion_dir(root_dir, str(row["move_g1_path"])) | |
| return load_pose_from_motion_dir(motion_dir) | |
| except Exception as exc: | |
| if strict: | |
| raise RuntimeError(f"Failed to load initial pose from metadata row: {exc}") from exc | |
| return np.zeros((36,), dtype=np.float32) | |
| def load_source_frame_count(source_motion_dir: Path) -> int | None: | |
| try: | |
| joint = pd.read_csv(source_motion_dir / "joint_pos.csv") | |
| frames = int(len(joint)) | |
| if frames > 0: | |
| return frames | |
| except Exception: | |
| pass | |
| return None | |
| def build_model_states_from_abs( | |
| state_history_abs: List[np.ndarray], | |
| *, | |
| textop_actions: bool, | |
| odim: int, | |
| ) -> np.ndarray: | |
| states_abs = np.stack(state_history_abs, axis=0).astype(np.float32) | |
| states = build_seed_textop_state(states_abs[:, :SEED_ABS_DIM]) if textop_actions else states_abs | |
| if states.shape[-1] < odim: | |
| pad = np.zeros((states.shape[0], odim - states.shape[-1]), dtype=np.float32) | |
| states = np.concatenate([states, pad], axis=-1) | |
| if states.shape[-1] != odim: | |
| raise ValueError(f"Infer state dim mismatch: built {states.shape[-1]}D, model odim={odim}") | |
| return states.astype(np.float32) | |
| def should_decode_textop_joint_from_delta(launch_config: LaunchConfig) -> bool: | |
| if SEED_TEXTOP_ACTION_DIM == 39: | |
| return False | |
| group_weight = getattr(getattr(launch_config, "train", None), "loss_group_weight", None) | |
| if group_weight is None: | |
| return False | |
| joint_abs_weight = float(getattr(group_weight, "joint_abs", 1.0)) | |
| joint_delta_weight = float(getattr(group_weight, "joint_delta", 1.0)) | |
| return joint_abs_weight == 0.0 and joint_delta_weight != 0.0 | |
| def recover_seed_abs_from_textop_action_with_joint_delta( | |
| features: np.ndarray, | |
| current_abs: np.ndarray, | |
| ) -> np.ndarray: | |
| # breakpoint() | |
| # 当joint_abs不参与训练时,推理侧也改成用joint_delta递推关节,避免使用未监督的joint_abs通道。 | |
| features = np.asarray(features, dtype=np.float32) | |
| current_abs = np.asarray(current_abs, dtype=np.float32).copy() | |
| if features.shape[-1] != SEED_TEXTOP_ACTION_DIM: | |
| raise ValueError(f"Expected textop action dim {SEED_TEXTOP_ACTION_DIM}, got {features.shape[-1]}") | |
| if current_abs.shape[-1] != SEED_ABS_DIM: | |
| raise ValueError(f"Expected current_abs dim {SEED_ABS_DIM}, got {current_abs.shape[-1]}") | |
| if TEXTOP_JOINT_DELTA.stop == TEXTOP_JOINT_DELTA.start: | |
| return recover_seed_abs_from_textop_action(features, current_abs) | |
| frames = [] | |
| prev_abs = current_abs.copy() | |
| for feat in features: | |
| joint_delta = feat[TEXTOP_JOINT_DELTA] | |
| root_delta_local = feat[TEXTOP_ROOT_DELTA_LOCAL] | |
| root_rot_delta = feat[TEXTOP_ROOT_ROT6D_DELTA] | |
| root_height = feat[TEXTOP_ROOT_HEIGHT][0] | |
| joint_abs = prev_abs[:29] + joint_delta | |
| root_pos = prev_abs[29:32] + world_delta_from_local(prev_abs[32:36], root_delta_local) | |
| root_pos[2] = root_height | |
| rel_quat = rot6d_delta_to_quat(root_rot_delta) | |
| root_quat = quat_mul_wxyz(prev_abs[32:36], rel_quat) | |
| frame = np.concatenate([joint_abs, root_pos.astype(np.float32), root_quat.astype(np.float32)], axis=0) | |
| frame = frame.astype(np.float32) | |
| frames.append(frame) | |
| prev_abs = frame | |
| return np.stack(frames, axis=0).astype(np.float32) | |
| def resolve_metadata_csv(data_root: Path, launch_config: LaunchConfig, metadata_csv_arg: str) -> Path | None: | |
| if metadata_csv_arg: | |
| metadata_csv = Path(metadata_csv_arg) | |
| elif hasattr(launch_config.data, "metadata_csv"): | |
| metadata_csv = Path(launch_config.data.metadata_csv) | |
| elif hasattr(launch_config.data, "text"): | |
| text_cfg = launch_config.data.text | |
| metadata_csv_value = text_cfg.get("metadata_csv") if isinstance(text_cfg, dict) else getattr(text_cfg, "metadata_csv", None) | |
| if metadata_csv_value is None: | |
| return None | |
| metadata_csv = Path(metadata_csv_value) | |
| else: | |
| return None | |
| if not metadata_csv.is_absolute(): | |
| metadata_csv = data_root / metadata_csv | |
| return metadata_csv | |
| def resolve_text_data_root(launch_config: LaunchConfig) -> Path: | |
| if hasattr(launch_config.data, "root_dir"): | |
| return Path(launch_config.data.root_dir) | |
| if hasattr(launch_config.data, "text"): | |
| text_cfg = launch_config.data.text | |
| root_dir = text_cfg.get("root_dir") if isinstance(text_cfg, dict) else getattr(text_cfg, "root_dir", None) | |
| if root_dir: | |
| return Path(root_dir) | |
| raise AttributeError("run config has neither data.root_dir nor data.text.root_dir") | |
| def load_prompt_initial_pose( | |
| *, | |
| args: argparse.Namespace, | |
| data_root: Path, | |
| metadata_csv: Path | None, | |
| use_delta_actions: bool, | |
| root_delta_actions: bool, | |
| textop_actions: bool, | |
| ) -> tuple[np.ndarray, str]: | |
| if args.init_pose_npy: | |
| # <modify> 允许prompt模式显式指定36D初始姿态,避免delta模型从zero起步 | |
| init_path = Path(args.init_pose_npy).expanduser() | |
| if not init_path.is_absolute(): | |
| cwd_path = Path.cwd() / init_path | |
| data_path = data_root / init_path | |
| init_path = cwd_path if cwd_path.exists() else data_path | |
| return load_pose_from_npy(init_path), str(init_path) | |
| if args.init_motion_dir: | |
| # <modify> 允许prompt模式显式指定SEED动作目录,用该动作第一帧作为初始姿态 | |
| motion_dir = resolve_motion_dir(data_root, args.init_motion_dir) | |
| return load_pose_from_motion_dir(motion_dir), str(motion_dir) | |
| if args.restore_from_data: | |
| # <modify> 默认从metadata的start-index动作取真实首帧,不再静默使用zero | |
| if metadata_csv is None: | |
| raise ValueError( | |
| "Prompt inference needs an initial pose. Use --init-motion-dir or --init-pose-npy " | |
| "when the run config has no metadata_csv." | |
| ) | |
| prompt_metadata = pd.read_csv(metadata_csv) | |
| if len(prompt_metadata) == 0: | |
| raise ValueError(f"Empty metadata csv: {metadata_csv}") | |
| ref_index = min(max(0, int(args.start_index)), len(prompt_metadata) - 1) | |
| ref_row = prompt_metadata.iloc[ref_index] | |
| motion_dir = map_motion_dir(data_root, str(ref_row["move_g1_path"])) | |
| return load_pose_from_motion_dir(motion_dir), f"{motion_dir} (metadata index {ref_index})" | |
| # <modify_jerk> hybrid root-delta prompt推理同样需要真实初始root。 | |
| if use_delta_actions or root_delta_actions or textop_actions: | |
| raise ValueError( | |
| "Delta/root-delta/TextOp prompt inference requires a real initial pose. " | |
| "Use --init-motion-dir, --init-pose-npy, or keep --restore-from-data." | |
| ) | |
| return np.zeros((36,), dtype=np.float32), "zero" | |
| def rollout_motion( | |
| *, | |
| model: Psi0Model, | |
| maxmin, | |
| odim: int, | |
| device: str, | |
| instruction: str, | |
| text_only: bool, | |
| normalize_state: bool, | |
| use_delta_actions: bool, | |
| root_delta_actions: bool, | |
| textop_actions: bool, | |
| decode_textop_joint_from_delta: bool, | |
| num_past_frames: int, | |
| use_future_progress_embedding: bool, | |
| enable_rtc: bool, | |
| action_exec_horizon: int, | |
| rtc_max_delay: int, | |
| num_inference_steps: int, | |
| target_frames: int, | |
| init_abs_state: np.ndarray, | |
| ) -> tuple[np.ndarray, np.ndarray, int]: | |
| horizon = int(model.action_horizon) | |
| exec_horizon = int(action_exec_horizon) if enable_rtc else horizon | |
| if exec_horizon <= 0 or exec_horizon > horizon: | |
| raise ValueError(f"Invalid action_exec_horizon={exec_horizon}, action_horizon={horizon}") | |
| rtc_delay = horizon - exec_horizon | |
| if enable_rtc and rtc_delay > 0 and rtc_delay >= int(rtc_max_delay): | |
| raise ValueError( | |
| f"RTC requires action_horizon - action_exec_horizon < max_delay, " | |
| f"got {horizon} - {exec_horizon} = {rtc_delay}, max_delay={rtc_max_delay}" | |
| ) | |
| if target_frames <= 0: | |
| target_frames = exec_horizon | |
| # <modify> RTC推理按exec_horizon滚动;普通推理仍按完整action_horizon滚动 | |
| # 原逻辑:num_chunks = int(math.ceil(float(target_frames) / float(horizon))) | |
| num_chunks = int(math.ceil(float(target_frames) / float(exec_horizon))) | |
| current_abs = np.asarray(init_abs_state, dtype=np.float32).copy() | |
| # <modify_states> 推理时维护和训练一致的多帧state历史,首个chunk用初始姿态重复填充。 | |
| state_horizon = int(num_past_frames) + 1 | |
| state_history: List[np.ndarray] = [current_abs.copy() for _ in range(state_horizon)] | |
| all_abs: List[np.ndarray] = [] | |
| all_delta: List[np.ndarray] = [] | |
| prev_actions_norm: np.ndarray | None = None | |
| remaining_frames = int(target_frames) | |
| for _ in range(num_chunks): | |
| generated_frames = int(target_frames - remaining_frames) | |
| # <modify_states> 使用最近To帧state作为输入;保持shape(B, To, Ds)。 | |
| # 原逻辑: | |
| # states = current_abs.reshape(1, odim).astype(np.float32) | |
| states = build_model_states_from_abs( | |
| state_history[-state_horizon:], | |
| textop_actions=textop_actions, | |
| odim=odim, | |
| ) | |
| if normalize_state: | |
| states = maxmin.normalize_state_func(states) | |
| states_t = torch.from_numpy(states).unsqueeze(0).to(device) | |
| future_progress_t = None | |
| if use_future_progress_embedding: | |
| _, future_progress = build_progress_conditions( | |
| frame_id=generated_frames, | |
| total_frames=target_frames, | |
| action_horizon=horizon, | |
| ) | |
| future_progress_t = torch.from_numpy(future_progress[None]).to(device) | |
| with torch.inference_mode(): | |
| if enable_rtc and prev_actions_norm is not None and rtc_delay > 0: | |
| # <modify> RTC推理把上一chunk尚未执行的尾部动作作为下一chunk的prefix条件 | |
| prev_rtc = np.concatenate( | |
| [ | |
| prev_actions_norm[exec_horizon:, :], | |
| np.zeros((exec_horizon, prev_actions_norm.shape[1]), dtype=np.float32), | |
| ], | |
| axis=0, | |
| ) | |
| prev_actions_t = torch.from_numpy(prev_rtc[None]).to(device) | |
| pred_norm = model.predict_action_with_training_rtc_flow( | |
| observations=None if text_only else [[]], | |
| states=states_t, | |
| instructions=[instruction], | |
| num_inference_steps=num_inference_steps, | |
| traj2ds=None, | |
| prev_actions=prev_actions_t, | |
| inference_delay=rtc_delay, | |
| max_delay=int(rtc_max_delay), | |
| future_progress=future_progress_t, | |
| )[0].float().cpu().numpy() | |
| else: | |
| pred_norm = model.predict_action( | |
| observations=None if text_only else [[]], | |
| states=states_t, | |
| instructions=[instruction], | |
| num_inference_steps=num_inference_steps, | |
| traj2ds=None, | |
| future_progress=future_progress_t, | |
| )[0].float().cpu().numpy() | |
| # <modify> 训练侧ActionStateTransform会把normalized action裁剪到[-1,1],推理反归一化前也要保持同一范围。 | |
| # 原逻辑:chunk_pred = np.asarray(maxmin.denormalize(pred_norm), dtype=np.float32) | |
| pred_norm_clipped = np.clip(np.asarray(pred_norm, dtype=np.float32), -1.0, 1.0) | |
| chunk_pred = np.asarray(maxmin.denormalize(pred_norm_clipped), dtype=np.float32) | |
| # <modify_textop_repr> TextOp训练目标先恢复到36D绝对姿态;root旋转delta在helper里按相对旋转6D解码。 | |
| if textop_actions: | |
| if decode_textop_joint_from_delta: | |
| chunk_abs = recover_seed_abs_from_textop_action_with_joint_delta(chunk_pred, current_abs) | |
| else: | |
| chunk_abs = recover_seed_abs_from_textop_action(chunk_pred, current_abs) | |
| prev_abs = np.concatenate([current_abs[None, :], chunk_abs[:-1]], axis=0) | |
| joint_delta = chunk_abs[:, :29] - prev_abs[:, :29] | |
| root_pos_delta = chunk_abs[:, 29:32] - prev_abs[:, 29:32] | |
| root_quat_delta = quat_mul_wxyz(quat_inv_wxyz(prev_abs[:, 32:36]), chunk_abs[:, 32:36]) | |
| chunk_delta = np.concatenate([joint_delta, root_pos_delta, root_quat_delta], axis=1) | |
| elif use_delta_actions: | |
| # <modify> delta action需要从current_abs累加成绝对姿态,支持多chunk滚动 | |
| # 原逻辑:把chunk_pred直接当绝对姿态,和delta训练目标不一致 | |
| chunk_delta = chunk_pred | |
| chunk_abs = np.cumsum(chunk_delta, axis=0) + current_abs[None, :] | |
| elif root_delta_actions: | |
| # <modify_jerk> hybrid action:关节直接用绝对预测,root 7维从当前姿态逐帧累加。 | |
| chunk_abs = chunk_pred.copy() | |
| root_abs = current_abs[29:36].astype(np.float32).copy() | |
| root_abs_seq = [] | |
| for root_delta in chunk_pred[:, 29:36]: | |
| root_abs = root_abs + root_delta | |
| quat_norm = np.linalg.norm(root_abs[3:7]) | |
| if quat_norm > 1e-8: | |
| root_abs[3:7] = root_abs[3:7] / quat_norm | |
| root_abs_seq.append(root_abs.copy()) | |
| chunk_abs[:, 29:36] = np.stack(root_abs_seq, axis=0) | |
| first_joint_delta = chunk_abs[0, :29] - current_abs[:29] | |
| if chunk_abs.shape[0] > 1: | |
| rest_joint_delta = chunk_abs[1:, :29] - chunk_abs[:-1, :29] | |
| joint_delta = np.concatenate([first_joint_delta[None, :], rest_joint_delta], axis=0) | |
| else: | |
| joint_delta = first_joint_delta[None, :] | |
| chunk_delta = np.concatenate([joint_delta, chunk_pred[:, 29:36]], axis=1) | |
| else: | |
| chunk_abs = chunk_pred | |
| first_delta = chunk_abs[0] - current_abs | |
| if chunk_abs.shape[0] > 1: | |
| rest_delta = chunk_abs[1:] - chunk_abs[:-1] | |
| chunk_delta = np.concatenate([first_delta[None, :], rest_delta], axis=0) | |
| else: | |
| chunk_delta = first_delta[None, :] | |
| chunk_abs = normalize_quaternion_rows(chunk_abs.astype(np.float32)) | |
| # <modify> TextOp且joint_abs未监督时,RTC prefix不能继续使用模型预测的joint_abs通道。 | |
| # 原逻辑:prev_actions_norm = np.asarray(pred_norm, dtype=np.float32).copy() | |
| # 这里用joint_delta还原后的关节绝对值回填0:29,再重新normalize,避免坏joint_abs污染下一chunk。 | |
| if textop_actions and decode_textop_joint_from_delta: | |
| chunk_pred_for_rtc = chunk_pred.copy() | |
| chunk_pred_for_rtc[:, :29] = chunk_abs[:, :29] | |
| # <modify> 这里不能直接调用maxmin.normalize,因为当前配置normalize_state=True时会要求data里有states。 | |
| action_min = np.asarray(maxmin.action_min, dtype=np.float32) | |
| action_max = np.asarray(maxmin.action_max, dtype=np.float32) | |
| ill_mask = np.abs(action_max - action_min) < 1e-4 * (np.abs(action_max) + np.abs(action_min) + 1e-8) | |
| action_max = action_max.copy() | |
| action_max[ill_mask] = 1.0 | |
| chunk_pred_for_rtc_norm = np.where( | |
| ill_mask, | |
| chunk_pred_for_rtc, | |
| (chunk_pred_for_rtc - action_min) / (action_max - action_min) * 2 - 1, | |
| ) | |
| if bool(getattr(maxmin, "use_norm_mask", False)): | |
| action_norm_masks = np.asarray(maxmin.action_norm_masks, dtype=bool) | |
| chunk_pred_for_rtc_norm = np.where(action_norm_masks, chunk_pred_for_rtc_norm, chunk_pred_for_rtc) | |
| prev_actions_norm = np.clip(chunk_pred_for_rtc_norm, -1, 1).astype(np.float32).copy() | |
| else: | |
| # <modify> RTC prefix也使用裁剪后的normalized action,避免把越界预测作为clean prefix条件。 | |
| # 原逻辑:prev_actions_norm = np.asarray(pred_norm, dtype=np.float32).copy() | |
| prev_actions_norm = pred_norm_clipped.copy() | |
| # <modify> RTC只导出本轮实际执行的前exec_horizon帧;普通模式exec_horizon等于horizon | |
| take = min(exec_horizon, remaining_frames) | |
| if take <= 0: | |
| break | |
| executed_abs = chunk_abs[:take] | |
| executed_delta = chunk_delta[:take].astype(np.float32) | |
| current_abs = executed_abs[-1].copy() | |
| # <modify_states> 每执行一步都推进state历史,下一chunk看到的是真实rollout历史而不是单帧。 | |
| for frame_abs in executed_abs: | |
| state_history.append(frame_abs.astype(np.float32).copy()) | |
| state_history = state_history[-state_horizon:] | |
| all_abs.append(executed_abs) | |
| all_delta.append(executed_delta) | |
| remaining_frames -= take | |
| pred_abs = np.concatenate(all_abs, axis=0)[:target_frames] | |
| pred_delta = np.concatenate(all_delta, axis=0)[:target_frames] | |
| pred_abs = normalize_quaternion_rows(pred_abs.astype(np.float32)) | |
| return pred_abs, pred_delta, num_chunks | |
| def resolve_ckpt_step(run_dir: Path, ckpt_step: str) -> int | str: | |
| if ckpt_step == "best": | |
| # <modify> 支持--ckpt-step best,直接读取checkpoints/best_ckpt目录。 | |
| best_ckpt = run_dir / "checkpoints" / "best_ckpt" | |
| if not best_ckpt.exists(): | |
| raise RuntimeError(f"No best_ckpt found under {run_dir / 'checkpoints'}") | |
| return "best_ckpt" | |
| if ckpt_step != "latest": | |
| return int(ckpt_step) | |
| ckpt_root = run_dir / "checkpoints" | |
| candidates = [] | |
| for p in ckpt_root.iterdir(): | |
| if p.is_dir() and p.name.startswith("ckpt_"): | |
| try: | |
| candidates.append(int(p.name.split("_")[-1])) | |
| except ValueError: | |
| continue | |
| if not candidates: | |
| raise RuntimeError(f"No ckpt_* found under {ckpt_root}") | |
| return max(candidates) | |
| def parse_args() -> argparse.Namespace: | |
| parser = argparse.ArgumentParser(description="Seed text->motion offline inference exporter") | |
| parser.add_argument("--run-dir", type=str, required=True) | |
| parser.add_argument("--ckpt-step", type=str, default="latest", help="integer step, 'latest', or 'best'") | |
| parser.add_argument("--output-dir", type=str, required=True) | |
| parser.add_argument("--num-samples", type=int, default=1) | |
| parser.add_argument("--start-index", type=int, default=0) | |
| parser.add_argument("--num-inference-steps", type=int, default=10) | |
| parser.add_argument("--device", type=str, default="cuda:0") | |
| parser.add_argument("--metadata-csv", type=str, default="") | |
| parser.add_argument("--prompt", type=str, default="", help="Direct text prompt for manual inference") | |
| parser.add_argument("--prompt-motion-name", type=str, default="manual_prompt") | |
| parser.add_argument("--init-motion-dir", type=str, default="", help="SEED csv_sonic motion dir used for prompt initial pose") | |
| parser.add_argument("--init-pose-npy", type=str, default="", help="36D pose npy, or Tx36 motion npy, used for prompt initial pose") | |
| parser.add_argument("--num-output-frames", type=int, default=0, help="0 means auto length") | |
| parser.add_argument("--seed", type=int, default=42) | |
| parser.add_argument("--restore-from-data", action="store_true", default=True) | |
| parser.add_argument("--no-restore-from-data", dest="restore_from_data", action="store_false") | |
| parser.add_argument("--export-delta", action="store_true", default=False) | |
| parser.add_argument("--disable-rtc", action="store_true", default=False) | |
| parser.add_argument( | |
| "--force-text-only", | |
| action="store_true", | |
| default=False, | |
| help="Use the same checkpoint but force text-only VLM inputs, even if run_config is audio-only.", | |
| ) | |
| return parser.parse_args() | |
| def main() -> None: | |
| args = parse_args() | |
| run_dir = Path(args.run_dir) | |
| output_dir = Path(args.output_dir) | |
| output_dir.mkdir(parents=True, exist_ok=True) | |
| cfg_template: LaunchConfig = parse_args_to_tyro_config(run_dir / "argv.txt") # type: ignore | |
| launch_config = cfg_template.model_validate_json((run_dir / "run_config.json").read_text()) | |
| seed_everything(launch_config.seed or args.seed) | |
| # 找到指定ckpt | |
| ckpt_step = resolve_ckpt_step(run_dir, args.ckpt_step) | |
| model = Psi0Model.from_pretrained(run_dir, ckpt_step, launch_config, device=args.device) | |
| if args.force_text_only: | |
| model.text_only = True | |
| model.audio_only = False | |
| model.video_only = False | |
| model.to(args.device) | |
| model.eval() | |
| data_root = resolve_text_data_root(launch_config) | |
| prompt_mode = bool(args.prompt.strip()) | |
| use_delta_actions = bool(getattr(launch_config.data.transform.repack, "use_delta_actions", False)) | |
| # <modify_jerk> run_config里记录hybrid action模式:关节绝对,root delta。 | |
| root_delta_actions = bool(getattr(launch_config.data.transform.repack, "root_delta_actions", False)) | |
| # TextOp模式下训练目标是39D action/state;推理端用同一个decoder恢复为36D绝对动作。 | |
| textop_actions = bool(getattr(launch_config.data.transform.repack, "textop_actions", False)) | |
| decode_textop_joint_from_delta = textop_actions and should_decode_textop_joint_from_delta(launch_config) | |
| # <modify_states> 从run_config读取SEED多帧state配置;旧checkpoint默认退化为单帧state。 | |
| num_past_frames = int(getattr(launch_config.data.transform.repack, "num_past_frames", 0)) | |
| # <modify> run_config里model.rtc=True时默认使用RTC滚动推理;可用--disable-rtc关闭 | |
| enable_rtc = bool(getattr(launch_config.model, "rtc", False)) and (not args.disable_rtc) | |
| use_future_progress_embedding = bool(getattr(launch_config.model, "use_future_progress_embedding", False)) | |
| action_exec_horizon = int(getattr(launch_config.model, "action_exec_horizon", int(model.action_horizon))) | |
| rtc_max_delay = int(getattr(launch_config.model, "max_delay", 0)) | |
| metadata_csv = resolve_metadata_csv(data_root, launch_config, args.metadata_csv) | |
| metadata = None | |
| if not prompt_mode: | |
| metadata = pd.read_csv(metadata_csv) | |
| start = max(0, args.start_index) | |
| end = min(len(metadata), start + max(0, args.num_samples)) | |
| if start >= end: | |
| raise ValueError(f"Empty sampling range: start={start}, end={end}, metadata_size={len(metadata)}") | |
| sample_indices = list(range(start, end)) | |
| else: | |
| sample_indices = [0] | |
| text_only = bool(args.force_text_only) or bool(getattr(launch_config.data.transform.model, "text_only", False)) | |
| maxmin = launch_config.data.transform.field | |
| odim = int(launch_config.model.odim) | |
| action_dim = int(getattr(launch_config.model, "action_dim", getattr(model, "action_dim", SEED_TEXTOP_ACTION_DIM))) | |
| if textop_actions and (action_dim != SEED_TEXTOP_ACTION_DIM or odim != SEED_TEXTOP_ACTION_DIM): | |
| raise ValueError( | |
| f"TextOp infer/train dim mismatch: action_dim={action_dim}, odim={odim}, expected={SEED_TEXTOP_ACTION_DIM}" | |
| ) | |
| manifest = { | |
| "run_dir": str(run_dir), | |
| "ckpt_step": ckpt_step, | |
| "metadata_csv": str(metadata_csv) if metadata_csv is not None else "", | |
| "mode": "prompt" if prompt_mode else "metadata", | |
| "text_only": text_only, | |
| "action_dim": int(action_dim), | |
| "state_dim": int(odim), | |
| "use_delta_actions": use_delta_actions, | |
| "root_delta_actions": root_delta_actions, | |
| "textop_actions": textop_actions, | |
| "decode_textop_joint_from_delta": decode_textop_joint_from_delta, | |
| "num_past_frames": int(num_past_frames), | |
| "use_future_progress_embedding": bool(use_future_progress_embedding), | |
| "enable_rtc": enable_rtc, | |
| "action_exec_horizon": int(action_exec_horizon), | |
| "rtc_max_delay": int(rtc_max_delay), | |
| "num_inference_steps": int(args.num_inference_steps), | |
| "num_output_frames": int(args.num_output_frames), | |
| "export_delta": bool(args.export_delta), | |
| "samples": [], | |
| } | |
| for idx in sample_indices: | |
| if prompt_mode: | |
| raw_instruction = args.prompt.strip() | |
| instruction = compose_seed_motion_prompt(raw_instruction) | |
| source_motion_dir = Path("manual_prompt") | |
| # <modify> prompt模式下delta推理不能再用zero初始姿态,默认从metadata的start-index动作取真实首帧 | |
| # 原逻辑:x0 = np.zeros((36,), dtype=np.float32) | |
| x0, initial_pose_source = load_prompt_initial_pose( | |
| args=args, | |
| data_root=data_root, | |
| metadata_csv=metadata_csv, | |
| use_delta_actions=use_delta_actions, | |
| root_delta_actions=root_delta_actions, | |
| textop_actions=textop_actions, | |
| ) | |
| auto_frames = max(int(model.action_horizon), 300) | |
| target_frames = int(args.num_output_frames) if args.num_output_frames > 0 else auto_frames | |
| motion_name = sanitize_motion_name(args.prompt_motion_name) | |
| sample_dir = output_dir / "g1" / "csv_sonic" / "manual" / motion_name | |
| else: | |
| assert metadata is not None | |
| row = metadata.iloc[idx] | |
| raw_instruction = "" | |
| instruction = compose_instruction(row, DEFAULT_TEXT_FIELDS) | |
| source_motion_dir = map_motion_dir(data_root, str(row.get("move_g1_path", ""))) | |
| # <modify> delta推理加载初始姿态失败时直接报错,避免静默退回zero | |
| x0 = load_initial_pose( | |
| data_root, | |
| row, | |
| restore_from_data=args.restore_from_data, | |
| strict=(use_delta_actions or root_delta_actions or textop_actions), | |
| ) | |
| initial_pose_source = str(source_motion_dir) if args.restore_from_data else "zero" | |
| source_frames = load_source_frame_count(source_motion_dir) | |
| if args.num_output_frames > 0: | |
| target_frames = int(args.num_output_frames) | |
| elif source_frames is not None: | |
| target_frames = source_frames | |
| else: | |
| target_frames = int(model.action_horizon) | |
| # Output folder keeps the same relative hierarchy as csv_sonic | |
| try: | |
| rel_dir = source_motion_dir.relative_to(data_root) | |
| except Exception: | |
| rel_dir = Path(f"sample_{idx:04d}") | |
| sample_dir = output_dir / rel_dir | |
| motion_name = source_motion_dir.name if source_motion_dir.name else f"sample_{idx:04d}" | |
| pred_abs, pred_delta, rollout_chunks = rollout_motion( | |
| model=model, | |
| maxmin=maxmin, | |
| odim=odim, | |
| device=args.device, | |
| instruction=instruction, | |
| text_only=text_only, | |
| normalize_state=bool(getattr(maxmin, "normalize_state", False)), | |
| use_delta_actions=use_delta_actions, | |
| root_delta_actions=root_delta_actions, | |
| textop_actions=textop_actions, | |
| decode_textop_joint_from_delta=decode_textop_joint_from_delta, | |
| num_past_frames=num_past_frames, | |
| use_future_progress_embedding=use_future_progress_embedding, | |
| enable_rtc=enable_rtc, | |
| action_exec_horizon=action_exec_horizon, | |
| rtc_max_delay=rtc_max_delay, | |
| num_inference_steps=int(args.num_inference_steps), | |
| target_frames=target_frames, | |
| init_abs_state=x0, | |
| ) | |
| sample_dir.mkdir(parents=True, exist_ok=True) | |
| joint_pos = pred_abs[:, :29] | |
| body_pos = pred_abs[:, 29:32] | |
| body_quat = pred_abs[:, 32:36] | |
| dt = 1.0 / float(TARGET_FPS) | |
| joint_vel = finite_difference(joint_pos, dt) | |
| body_lin_vel = finite_difference(body_pos, dt) | |
| body_ang_vel = quaternion_angular_velocity(body_quat, dt) | |
| # Main outputs: match csv_sonic naming/headers | |
| write_csv(sample_dir / "joint_pos.csv", joint_pos, [f"joint_{i}" for i in range(29)]) | |
| write_csv(sample_dir / "joint_vel.csv", joint_vel, [f"joint_vel_{i}" for i in range(29)]) | |
| write_csv(sample_dir / "body_pos.csv", body_pos, ["body_0_x", "body_0_y", "body_0_z"]) | |
| write_csv(sample_dir / "body_quat.csv", body_quat, ["body_0_w", "body_0_x", "body_0_y", "body_0_z"]) | |
| write_csv(sample_dir / "body_lin_vel.csv", body_lin_vel, ["body_0_vel_x", "body_0_vel_y", "body_0_vel_z"]) | |
| write_csv(sample_dir / "body_ang_vel.csv", body_ang_vel, ["body_0_angvel_x", "body_0_angvel_y", "body_0_angvel_z"]) | |
| np.save(sample_dir / "motion.npy", pred_abs) | |
| # Optional delta outputs | |
| if args.export_delta: | |
| write_csv(sample_dir / "joint_pos_delta.csv", pred_delta[:, :29], [f"joint_{i}" for i in range(29)]) | |
| write_csv(sample_dir / "body_pos_delta.csv", pred_delta[:, 29:32], ["body_0_x", "body_0_y", "body_0_z"]) | |
| write_csv(sample_dir / "body_quat_delta.csv", pred_delta[:, 32:36], ["body_0_w", "body_0_x", "body_0_y", "body_0_z"]) | |
| np.save(sample_dir / "motion_delta.npy", pred_delta) | |
| write_metadata(sample_dir / "metadata.txt", motion_name, int(pred_abs.shape[0])) | |
| write_info( | |
| sample_dir / "info.txt", | |
| source_path=str(source_motion_dir), | |
| initial_pose_source=initial_pose_source, | |
| frames=int(pred_abs.shape[0]), | |
| use_delta_actions=use_delta_actions, | |
| root_delta_actions=root_delta_actions, | |
| textop_actions=textop_actions, | |
| num_past_frames=int(num_past_frames), | |
| enable_rtc=enable_rtc, | |
| action_exec_horizon=int(action_exec_horizon), | |
| num_inference_steps=int(args.num_inference_steps), | |
| ) | |
| # Keep source references if existing | |
| if not prompt_mode: | |
| src_info = source_motion_dir / "info.txt" | |
| src_meta = source_motion_dir / "metadata.txt" | |
| if src_info.exists(): | |
| shutil.copy2(src_info, sample_dir / "info_source.txt") | |
| if src_meta.exists(): | |
| shutil.copy2(src_meta, sample_dir / "metadata_source.txt") | |
| sample_meta = { | |
| "sample_index": int(idx), | |
| "instruction": instruction, | |
| "raw_instruction": raw_instruction, | |
| "move_g1_path": "" if prompt_mode else str(metadata.iloc[idx].get("move_g1_path", "")), | |
| "source_motion_dir": str(source_motion_dir), | |
| "initial_pose_source": initial_pose_source, | |
| "num_past_frames": int(num_past_frames), | |
| "root_delta_actions": root_delta_actions, | |
| "textop_actions": textop_actions, | |
| "decode_textop_joint_from_delta": decode_textop_joint_from_delta, | |
| "enable_rtc": enable_rtc, | |
| "action_exec_horizon": int(action_exec_horizon), | |
| "output_dir": str(sample_dir), | |
| "num_output_frames": int(pred_abs.shape[0]), | |
| "rollout_chunks": int(rollout_chunks), | |
| } | |
| (sample_dir / "infer_meta.json").write_text(json.dumps(sample_meta, ensure_ascii=False, indent=2)) | |
| manifest["samples"].append(sample_meta) | |
| print(f"[OK] sample={idx} -> {sample_dir}") | |
| (output_dir / "manifest.json").write_text(json.dumps(manifest, ensure_ascii=False, indent=2)) | |
| print(f"[DONE] wrote manifest: {output_dir / 'manifest.json'}") | |
| if __name__ == "__main__": | |
| main() | |