code-HFD / scripts /infer /full_duplex_audio_to_motion.py
Tinghong-Ye22's picture
Add files using upload-large-folder tool
06f9561 verified
Raw History Blame Contribute Delete
107 kB
#!/usr/bin/env python3 from __future__ import annotations import argparse import contextlib import hashlib import json import math import os import shutil import subprocess import sys import wave from pathlib import Path from typing import Any, Sequence import numpy as np import torch PROJECT_ROOT = Path(__file__).resolve().parents[2] DATA_SCRIPT_DIR = PROJECT_ROOT / "scripts" / "data" INFER_SCRIPT_DIR = PROJECT_ROOT / "scripts" / "infer" if str(DATA_SCRIPT_DIR) not in sys.path: sys.path.insert(0, str(DATA_SCRIPT_DIR)) if str(INFER_SCRIPT_DIR) not in sys.path: sys.path.insert(0, str(INFER_SCRIPT_DIR)) from cache_omni_hidden_states import ( # noqa: E402 load_audio_array as load_omni_audio_array, ) from psi.config.config import LaunchConfig # noqa: E402 from psi.data.seed.full_duplex_audio_contract import ( # noqa: E402 AUDIO_COMMAND_STREAMING_PROMPT, ) from psi.data.seed.full_duplex_audio_command_dataset import ( # noqa: E402 _read_resampled_speech_float, detect_clean_speech_onset_sample, ) from psi.data.seed.qwen_streaming_audio import ( # noqa: E402 continuous_audio_production_identity, hfdv2_train_infer_contract, prepare_qwen_streaming_audio_window, ) from psi.data.seed.storage_paths import ( # noqa: E402 relocate_hfd_storage_path, ) from psi.data.seed.motion_repr import ( # noqa: E402 build_seed_textop_action, recover_seed_abs_from_textop_action, ) from psi.data.seed.full_duplex_timing import ( # noqa: E402 ACTION_CHUNK_FRAMES, ACTION_EXEC_FRAMES, ACTION_SAMPLE_STEP_FRAMES, AUDIO_RATE, AUDIO_STRIDE_FRAMES, AUDIO_WINDOW_SECONDS, MOTION_FPS, RTC_PREFIX_FRAMES, complete_audio_clip_starts, ) from psi.models.full_duplex_psi0 import ( # noqa: E402 FullDuplexPsi0Model, FullDuplexStreamCondition, ) from psi.utils import parse_args_to_tyro_config, seed_everything # noqa: E402 from psi.runtime.motion_rollout import _flow_action, _stationary_future_progress # noqa: E402 from psi.runtime.motion_rollout import decode_action_chunk, next_rtc_prefix from seed_audio_to_motion import normalize_action_for_rtc # noqa: E402 from seed_text_to_motion import ( # noqa: E402 TARGET_FPS, build_model_states_from_abs, finite_difference, load_pose_from_motion_dir, normalize_quaternion_rows, quaternion_angular_velocity, resolve_ckpt_step, write_csv, write_metadata, ) LISTEN_TOKEN = "</listen>" MOTION_TOKEN = "</motion>" IDLE_TOKEN = "</idle>" CONTROL_TOKENS = (LISTEN_TOKEN, MOTION_TOKEN, IDLE_TOKEN) DEFAULT_COMMAND_PROMPT = AUDIO_COMMAND_STREAMING_PROMPT EXPORTED_AUDIO_STRIDE_FRAMES = 20 class FullDuplexControlTimeout(RuntimeError): """A strict rollout ended because the model never selected motion.""" def __init__(self, message: str, *, diagnostics: dict[str, Any]) -> None: super().__init__(message) self.diagnostics = diagnostics def _expected_stream_cache_tokens( consumed_windows: int, max_cache_tokens: int, ) -> int: """Return the visible cache length after one-source causal rolling.""" consumed = max(int(consumed_windows), 0) limit = int(max_cache_tokens) return consumed if limit <= 0 else min(consumed, limit) def _control_decision( *, predicted_id: int, respect_control_token: bool, ) -> str: """Choose this window's execution directly from this window's token.""" token_id = int(predicted_id) if token_id < 0 or token_id >= len(CONTROL_TOKENS): raise ValueError(f"Invalid full-duplex control id: {token_id}") if not bool(respect_control_token): return "motion" return ("listen", "motion", "idle")[token_id] def _rtc_prefix_after_control_decision( known_prefix: np.ndarray | None, *, previous_control: str | None, current_control: str, semantics_version: int, ) -> np.ndarray | None: """Apply the checkpoint-versioned RTC policy at a control boundary.""" if ( int(semantics_version) in {12, 13, 14, 15, 16, 17, 18} and previous_control is not None and str(current_control) != str(previous_control) ): return None return known_prefix def _autocast(device: torch.device): if device.type == "cuda": return torch.autocast("cuda", dtype=torch.bfloat16) return contextlib.nullcontext() def _normalise_audio(source: Path, destination: Path) -> np.ndarray: destination.parent.mkdir(parents=True, exist_ok=True) command = [ "ffmpeg", "-v", "error", "-y", "-i", str(source), "-vn", "-ar", str(AUDIO_RATE), "-ac", "1", "-c:a", "pcm_f32le", str(destination), ] result = subprocess.run(command, capture_output=True, check=False) if result.returncode != 0: raise RuntimeError( f"ffmpeg failed to normalize {source}: " f"{result.stderr.decode('utf-8', errors='ignore')}" ) model_waveform = load_omni_audio_array(source, AUDIO_RATE) rendered_waveform = load_omni_audio_array(destination, AUDIO_RATE) if not np.array_equal(model_waveform, rendered_waveform): max_abs = ( float(np.max(np.abs(model_waveform - rendered_waveform))) if model_waveform.shape == rendered_waveform.shape else float("inf") ) raise RuntimeError( "Normalized inference WAV differs from the waveform used by the " "HFDv2 online Qwen input path: " f"source_shape={model_waveform.shape}, " f"normalized_shape={rendered_waveform.shape}, max_abs={max_abs:g}" ) return model_waveform def resolve_prompt(raw_prompt: str, audio_path: Path, launch_config: LaunchConfig) -> str: del audio_path, launch_config if str(raw_prompt).strip().lower() != "auto": return str(raw_prompt).strip() return DEFAULT_COMMAND_PROMPT def validate_online_prompt(prompt: str) -> None: if str(prompt) != AUDIO_COMMAND_STREAMING_PROMPT: raise RuntimeError( "HFDv2 inference prefix differs from training. Use --prompt auto; " "the spoken waveform, not an extra text command, carries intent." ) def validate_contract(launch_config: LaunchConfig) -> None: model = launch_config.model data = launch_config.data repack = data.transform.repack root40 = getattr(model, "action_representation", "textop39") == "root40_v1" if root40 != bool(getattr(repack, "root40_actions", False)): raise RuntimeError("Model and dataset action representations disagree") semantics_version = int( getattr(launch_config.train, "full_duplex_semantics_version", 0) ) checks = { "model.action_chunk_size": (int(model.action_chunk_size), ACTION_CHUNK_FRAMES), "model.action_exec_horizon": (int(model.action_exec_horizon), ACTION_EXEC_FRAMES), "model.max_delay": (int(model.max_delay), RTC_PREFIX_FRAMES + 1), "data.action_exec_horizon": (int(data.action_exec_horizon), ACTION_EXEC_FRAMES), "data.rtc_prefix_frames": (int(repack.rtc_prefix_frames), RTC_PREFIX_FRAMES), "model.action_dim": (int(model.action_dim), 40 if root40 else 39), "model.odim": (int(model.odim), 40 if root40 else 39), "model.view_feature_dim": (int(model.view_feature_dim), 2048), "data.motion_fps": (int(getattr(data, "motion_fps", MOTION_FPS)), MOTION_FPS), "data.audio_sampling_rate": ( int(getattr(data, "audio_sampling_rate", AUDIO_RATE)), AUDIO_RATE, ), "data.audio_sample_step_frames": ( int(getattr(data, "audio_sample_step_frames", AUDIO_STRIDE_FRAMES)), AUDIO_STRIDE_FRAMES, ), "data.action_sample_step_frames": ( int( getattr( data, "action_sample_step_frames", ACTION_SAMPLE_STEP_FRAMES, ) ), ACTION_SAMPLE_STEP_FRAMES, ), } failed = [f"{name}={actual}, expected={expected}" for name, (actual, expected) in checks.items() if actual != expected] if semantics_version not in {9, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24}: failed.append( "train.full_duplex_semantics_version=" f"{semantics_version}, expected 9, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23 or 24" ) window = float(getattr(data, "audio_window_seconds", AUDIO_WINDOW_SECONDS)) if not math.isclose(window, AUDIO_WINDOW_SECONDS, rel_tol=0.0, abs_tol=1e-9): failed.append(f"data.audio_window_seconds={window}, expected={AUDIO_WINDOW_SECONDS}") if not bool(model.rtc): failed.append("model.rtc=False, expected=True") if not bool(repack.textop_actions): failed.append("data.transform.repack.textop_actions=False, expected=True") if not bool(model.use_future_progress_embedding): failed.append("model.use_future_progress_embedding=False, expected=True") if not bool(getattr(model, "streaming_qwen_enabled", False)): failed.append("model.streaming_qwen_enabled=False, expected=True") if not bool(getattr(model, "tune_vlm", False)): failed.append("model.tune_vlm=False, expected=True") if bool(getattr(model, "gradient_checkpointing", True)): failed.append("model.gradient_checkpointing=True, expected=False") if not bool(getattr(data, "online_audio", False)): failed.append("data.online_audio=False, expected=True") if ( bool(getattr(data, "stationary_heading_alignment", False)) != (semantics_version in {15, 16, 17, 18, 19, 20, 21, 22, 23, 24}) ): failed.append( "data.stationary_heading_alignment differs from semantics v15-v24" ) if bool(getattr(model, "predict_motion_duration", False)) != ( semantics_version in {16, 17, 18, 19, 20, 21, 22, 23, 24} ): failed.append( "model.predict_motion_duration differs from semantics v16-v24" ) session_voice_variants = int( getattr(data, "session_voice_variants", 0) ) if semantics_version in {17, 18, 19, 20, 21, 22, 23, 24}: if session_voice_variants < 2: failed.append( "semantics v17-v24 requires data.session_voice_variants>=2" ) elif session_voice_variants != 0: failed.append( "historical semantics require data.session_voice_variants=0" ) interruption_sessions = bool( getattr(data, "interruption_sessions", False) ) if interruption_sessions != (semantics_version in {20, 21, 22, 23, 24}): failed.append( "data.interruption_sessions differs from semantics v20-v24" ) state_aware_control = bool( getattr(model, "state_aware_serial_control", False) ) if state_aware_control != (semantics_version == 21): failed.append( "model.state_aware_serial_control differs from versioned policy" ) independent_task_pools = bool( getattr(data, "independent_task_pools", False) ) if independent_task_pools != (semantics_version in {21, 22, 23, 24}): failed.append( "data.independent_task_pools differs from semantics v21-v24" ) idle_residual_retargeting = bool( getattr(data, "stationary_idle_residual_retargeting", False) ) if idle_residual_retargeting != (semantics_version == 24): failed.append( "data.stationary_idle_residual_retargeting differs from " "semantics v24" ) implicit_repair_ratio = float( getattr(data, "self_repair_implicit_ratio", 0.0) ) if semantics_version == 24: if not 0.0 < implicit_repair_ratio < 1.0: failed.append( "semantics v24 requires 0 < " "data.self_repair_implicit_ratio < 1" ) elif implicit_repair_ratio != 0.0: failed.append( "historical semantics require data.self_repair_implicit_ratio=0" ) if failed: raise ValueError("Full-duplex train/infer contract mismatch: " + "; ".join(failed)) def validate_checkpoint_encoder_contract( run_dir: Path, ckpt_step: int | str, current_contract: dict[str, Any], *, expected_global_step: int | None = None, ) -> dict[str, Any]: checkpoint_name = str(ckpt_step) if checkpoint_name in {"best", "best_ckpt"}: checkpoint_dir = run_dir / "checkpoints" / "best_ckpt" elif checkpoint_name.startswith("ckpt_"): checkpoint_dir = run_dir / "checkpoints" / checkpoint_name else: checkpoint_dir = run_dir / "checkpoints" / f"ckpt_{checkpoint_name}" marker_path = checkpoint_dir / "full_duplex_source_boundary.json" if not marker_path.is_file(): raise RuntimeError( "This checkpoint has no HFDv2 source-boundary/contract marker" ) marker = json.loads(marker_path.read_text(encoding="utf-8")) try: global_step = int(marker["global_step"]) except (KeyError, TypeError, ValueError) as error: raise RuntimeError( f"Checkpoint has no valid global_step marker: {marker_path}" ) from error if global_step <= 0: raise RuntimeError( f"Checkpoint has invalid global_step={global_step}: {marker_path}" ) if expected_global_step is not None and global_step != int( expected_global_step ): raise RuntimeError( "Checkpoint changed while automatic inference was starting: " f"expected global_step={expected_global_step}, observed={global_step}. " "Retry so one output directory cannot mix two best_ckpt versions." ) if checkpoint_name not in {"best", "best_ckpt"}: directory_step = int(checkpoint_name.removeprefix("ckpt_")) if global_step != directory_step: raise RuntimeError( "Immutable checkpoint directory/marker mismatch: " f"directory={checkpoint_name}, marker global_step={global_step}" ) if ( int(marker.get("full_duplex_semantics_version", 0)) not in {9, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24} or marker.get("architecture") != "hfd_qwen_native_kv_serial_control" ): raise RuntimeError( "Refusing to run HFDv2 inference on a legacy/cached-hidden checkpoint" ) checkpoint_contract = marker.get("train_infer_contract") if checkpoint_contract != current_contract: mismatch_paths = _contract_mismatch_paths( checkpoint_contract, current_contract, ) mismatch_summary = ", ".join(mismatch_paths[:8]) or "unknown" raise RuntimeError( "Checkpoint HFDv2 contract differs from active base Qwen, " "prompt, raw data, tail tuning, KV horizon, effective batch, " "or RTC geometry; differing fields: " f"{mismatch_summary}" ) return marker def _contract_mismatch_paths( checkpoint_value: Any, active_value: Any, prefix: str = "", ) -> list[str]: if isinstance(checkpoint_value, dict) and isinstance(active_value, dict): paths: list[str] = [] for key in sorted(set(checkpoint_value) | set(active_value)): child_prefix = f"{prefix}.{key}" if prefix else str(key) if key not in checkpoint_value or key not in active_value: paths.append(child_prefix) continue paths.extend( _contract_mismatch_paths( checkpoint_value[key], active_value[key], child_prefix, ) ) return paths if checkpoint_value != active_value: return [prefix or "<root>"] return [] def _build_active_hfdv2_contract( launch_config: LaunchConfig, *, semantics_version: int, backbone: Path, continuous_audio_root: Path, background_assignment_path: Path, ) -> dict[str, Any]: from psi.models.relative_instruction_memory import architecture_contract return hfdv2_train_infer_contract( architecture_options=architecture_contract(launch_config.model), full_duplex_semantics_version=semantics_version, backbone=backbone, continuous_audio_root=continuous_audio_root, background_assignment_path=background_assignment_path, qwen_trainable_last_layers=( launch_config.model.streaming_qwen_trainable_last_layers ), qwen_train_final_norm=launch_config.model.streaming_qwen_train_final_norm, qwen_kv_gradient_horizon_windows=( launch_config.train.qwen_kv_gradient_horizon_windows ), dense_window_accumulation_steps=int( getattr( launch_config.train, "dense_window_accumulation_steps", 1, ) ), train_microbatch_per_gpu=launch_config.train.train_batch_size, source_accumulation_groups=( launch_config.train.source_accumulation_groups ), audio_sampling_rate=launch_config.data.audio_sampling_rate, audio_window_seconds=launch_config.data.audio_window_seconds, audio_stride_frames=launch_config.data.audio_sample_step_frames, action_sample_step_frames=( launch_config.data.action_sample_step_frames ), motion_fps=launch_config.data.motion_fps, action_chunk_frames=launch_config.model.action_chunk_size, action_execute_frames=launch_config.model.action_exec_horizon, control_condition_train_mode=( launch_config.model.control_condition_train_mode ), control_condition_inference_mode=( launch_config.model.control_condition_inference_mode ), control_condition_temperature=( launch_config.model.control_condition_temperature ), joint_velocity_loss_weight=( launch_config.train.joint_velocity_loss_weight ), joint_acceleration_loss_weight=( launch_config.train.joint_acceleration_loss_weight ), root_continuity_loss_weight=( launch_config.train.root_continuity_loss_weight ), stationary_root_height_loss_weight=( launch_config.train.stationary_root_height_loss_weight ), session_min_turns=getattr(launch_config.data, "session_min_turns", 1), session_max_turns=getattr(launch_config.data, "session_max_turns", 1), session_voice_variants=getattr( launch_config.data, "session_voice_variants", 0, ), interruption_sessions=getattr( launch_config.data, "interruption_sessions", False, ), interruption_ratio=getattr( launch_config.data, "interruption_ratio", 0.0, ), interruption_min_start_seconds=getattr( launch_config.data, "interruption_min_start_seconds", 2.0, ), interruption_max_start_seconds=getattr( launch_config.data, "interruption_max_start_seconds", 6.0, ), interruption_min_remaining_seconds=getattr( launch_config.data, "interruption_min_remaining_seconds", 1.5, ), initial_idle_max_steps=getattr( launch_config.data, "initial_idle_max_steps", 0, ), stationary_blend_frames=getattr( launch_config.data, "stationary_blend_frames", ACTION_EXEC_FRAMES, ), turn_transition_frames=getattr( launch_config.data, "turn_transition_frames", ACTION_EXEC_FRAMES, ), causal_listen_timing=getattr( launch_config.data, "causal_listen_timing", False, ), max_speech_onset_steps=getattr( launch_config.data, "max_speech_onset_steps", 0, ), listen_reaction_steps=getattr( launch_config.data, "listen_reaction_steps", 0, ), motion_duration_prediction=bool( getattr(launch_config.model, "predict_motion_duration", False) ), motion_duration_default_seconds=float( getattr( launch_config.model, "motion_duration_default_seconds", 8.0, ) ), motion_duration_distribution_bins=int( getattr( launch_config.model, "motion_duration_distribution_bins", 1, ) ), motion_duration_min_seconds=float( getattr(launch_config.model, "motion_duration_min_seconds", 0.5) ), motion_duration_max_seconds=float( getattr(launch_config.model, "motion_duration_max_seconds", 180.0) ), motion_duration_loss_weight=float( getattr( launch_config.train, "motion_duration_loss_weight", 0.0, ) ), latest_instruction_loss_weight=float( getattr( launch_config.train, "latest_instruction_loss_weight", 0.0, ) ), latest_instruction_metadata_path=getattr( launch_config.data, "metadata_jsonl", None, ), instruction_recency_slots=int( getattr( launch_config.model, "instruction_recency_slots", 0, ) ), instruction_memory_loss_weight=float( getattr( launch_config.train, "instruction_memory_loss_weight", 0.0, ) ), max_stream_steps=int( getattr(launch_config.data, "max_stream_steps", 1923) ), state_aware_serial_control=bool( getattr(launch_config.model, "state_aware_serial_control", False) ), independent_task_pools=bool( getattr(launch_config.data, "independent_task_pools", False) ), task_session_variants={ "turn-taking": int( getattr( launch_config.data, "turn_taking_session_variants", 0, ) ), "interruption": int( getattr( launch_config.data, "command_interruption_session_variants", 0, ) ), "emergency-stop": int( getattr( launch_config.data, "emergency_stop_session_variants", 0, ) ), "safe-hold": int( getattr( launch_config.data, "safe_hold_session_variants", 0, ) ), "backchannel": int( getattr( launch_config.data, "backchannel_session_variants", 0, ) ), "hesitation": int( getattr( launch_config.data, "hesitation_session_variants", 0, ) ), "pause": int( getattr( launch_config.data, "pause_session_variants", 0, ) ), }, interruption_task_session_ratio=float( getattr( launch_config.data, "interruption_task_session_ratio", 0.0, ) ), control_task_session_ratio=float( getattr( launch_config.data, "control_task_session_ratio", 0.0, ) ), backchannel_task_session_ratio=float( getattr( launch_config.data, "backchannel_task_session_ratio", 0.0, ) ), hesitation_task_window_ratio=float( getattr( launch_config.data, "hesitation_task_window_ratio", 0.0, ) ), pause_task_window_ratio=float( getattr(launch_config.data, "pause_task_window_ratio", 0.0) ), control_followup_ratio=float( getattr(launch_config.data, "control_followup_ratio", 0.0) ), control_post_idle_min_steps=int( getattr( launch_config.data, "control_post_idle_min_steps", 1, ) ), control_post_idle_max_steps=int( getattr( launch_config.data, "control_post_idle_max_steps", 0, ) ), control_post_idle_sampling=str( getattr( launch_config.data, "control_post_idle_sampling", "legacy_uniform", ) ), control_min_start_seconds=float( getattr( launch_config.data, "control_min_start_seconds", 2.0, ) ), control_max_start_seconds=float( getattr( launch_config.data, "control_max_start_seconds", 6.0, ) ), control_min_remaining_seconds=float( getattr( launch_config.data, "control_min_remaining_seconds", 0.4, ) ), control_activity_aware_timing=bool( getattr( launch_config.data, "control_activity_aware_timing", False, ) ), backchannel_command_manifest=getattr( launch_config.data, "backchannel_command_manifest", None, ), hesitation_pause_manifest=getattr( launch_config.data, "hesitation_pause_manifest", None, ), new_task_max_rows_per_instruction_train=int( getattr( launch_config.data, "new_task_max_rows_per_instruction_train", 32, ) ), new_task_max_rows_per_instruction_val=int( getattr( launch_config.data, "new_task_max_rows_per_instruction_val", 8, ) ), hesitation_filled_ratio=float( getattr(launch_config.data, "hesitation_filled_ratio", 0.60) ), self_repair_implicit_ratio=float( getattr(launch_config.data, "self_repair_implicit_ratio", 0.0) ), backchannel_min_start_seconds=float( getattr( launch_config.data, "backchannel_min_start_seconds", 1.2, ) ), backchannel_max_start_seconds=float( getattr( launch_config.data, "backchannel_max_start_seconds", 6.0, ) ), backchannel_min_remaining_seconds=float( getattr( launch_config.data, "backchannel_min_remaining_seconds", 1.2, ) ), filter_known_tts_translation_mismatches=bool( getattr( launch_config.data, "filter_known_tts_translation_mismatches", False, ) ), canonical_motion_manifest=getattr( launch_config.data, "canonical_motion_manifest", "", ), control_command_manifest=getattr( launch_config.data, "control_command_manifest", None, ), safe_hold_stability_window_frames=int( getattr( launch_config.data, "safe_hold_stability_window_frames", 20, ) ), safe_stationary_anchors=bool( getattr(launch_config.data, "safe_stationary_anchors", False) ), stationary_idle_residual_retargeting=bool( getattr( launch_config.data, "stationary_idle_residual_retargeting", False, ) ), dynamic_task_entry_blend=bool( getattr(launch_config.data, "dynamic_task_entry_blend", False) ), stationary_listen_hold=bool(getattr(launch_config.data, "stationary_listen_hold", False)), long_session_drop_ratio=float( getattr(launch_config.data, "long_session_drop_ratio", 0.0) ), stop_enter_loss_weight=float( getattr(launch_config.train, "stop_enter_loss_weight", 0.0) ), stop_stay_loss_weight=float( getattr(launch_config.train, "stop_stay_loss_weight", 0.0) ), stop_release_loss_weight=float( getattr(launch_config.train, "stop_release_loss_weight", 0.0) ), ) def _continuous_window_count(sample_count: int) -> int: window_samples = int(round(AUDIO_WINDOW_SECONDS * AUDIO_RATE)) stride_samples = int( round(AUDIO_STRIDE_FRAMES / float(MOTION_FPS) * AUDIO_RATE) ) prefix_samples = window_samples - stride_samples return int(math.ceil((prefix_samples + int(sample_count)) / stride_samples)) def _exported_window_count(sample_count: int) -> int: window_samples = int(round(AUDIO_WINDOW_SECONDS * AUDIO_RATE)) stride_samples = int( round(EXPORTED_AUDIO_STRIDE_FRAMES / float(MOTION_FPS) * AUDIO_RATE) ) prefix_samples = window_samples - stride_samples return int(math.ceil((prefix_samples + int(sample_count)) / stride_samples)) def _file_sha256(path: Path) -> str: digest = hashlib.sha256() with path.open("rb") as handle: for chunk in iter(lambda: handle.read(1024 * 1024), b""): digest.update(chunk) return digest.hexdigest() def _build_continuous_mixed_waveform( *, speech_waveform: np.ndarray, background_waveform: np.ndarray, listen_start_index: int, ) -> tuple[np.ndarray, float]: """Reproduce the training waveform mix and its two exact RTC overlaps.""" speech = np.asarray(speech_waveform, dtype=np.float32).reshape(-1) background = np.asarray(background_waveform, dtype=np.float32).reshape(-1) window_samples = int(round(AUDIO_WINDOW_SECONDS * AUDIO_RATE)) stride_samples = int( round(AUDIO_STRIDE_FRAMES / float(MOTION_FPS) * AUDIO_RATE) ) prefix_samples = window_samples - stride_samples listen_windows = _continuous_window_count(speech.size) mixed_samples = (listen_windows - 1) * stride_samples + window_samples background_start = int(listen_start_index) * stride_samples background_end = background_start + mixed_samples if background_start < 0 or background_end > background.size: raise RuntimeError( "Selected continuous background cannot cover inference speech: " f"start={listen_start_index} samples={mixed_samples} " f"background_samples={background.size}" ) mixed = background[background_start:background_end].copy() speech_peak = float(np.max(np.abs(speech), initial=0.0)) peak_limit = 10.0 ** (-3.0 / 20.0) speech_scale = ( min(1.0, peak_limit / speech_peak) if speech_peak > 0.0 else 1.0 ) speech_start = prefix_samples speech_end = speech_start + speech.size mixed[speech_start:speech_end] += speech * np.float32(speech_scale) if float(np.max(np.abs(mixed), initial=0.0)) >= 1.0: raise RuntimeError( "Continuous inference mix clips; the production background " "contract requires its original quiet level" ) # Training writes PCM16 before Qwen export. Quantize here too so online # inference sees the same waveform domain instead of a subtly different # pre-quantization float sum. mixed = ( np.rint(mixed * 32768.0) .clip(-32768, 32767) .astype("<i2") .astype(np.float32) / np.float32(32768.0) ) # The last prefix interval must already be pure background so the next # motion window shares the exact same PCM overlap. next_start = (int(listen_start_index) + listen_windows) * stride_samples if not np.array_equal( mixed[-prefix_samples:], background[next_start : next_start + prefix_samples], ): raise RuntimeError("Mixed/background RTC overlap is not continuous") return np.ascontiguousarray(mixed, dtype=np.float32), float(speech_scale) def _build_continuous_multiturn_waveform( *, speech_waveforms: Sequence[np.ndarray], speech_scales: Sequence[float], background_waveform: np.ndarray, listen_start_index: int, inter_turn_background_steps: int | Sequence[int], ) -> tuple[np.ndarray, list[int], list[int]]: """Place several commands on one uninterrupted inference background.""" if len(speech_waveforms) < 2: raise ValueError("Multi-turn inference requires at least two commands") if len(speech_waveforms) != len(speech_scales): raise ValueError("Speech waveform/scale count mismatch") gap_count = len(speech_waveforms) - 1 if isinstance(inter_turn_background_steps, (int, np.integer)): gap_steps = [int(inter_turn_background_steps)] * gap_count else: gap_steps = [int(value) for value in inter_turn_background_steps] if len(gap_steps) != gap_count: raise ValueError( "Multi-turn inference needs one background gap per adjacent " f"turn pair: turns={len(speech_waveforms)} gaps={gap_steps}" ) if any(value < 0 for value in gap_steps): raise ValueError( "Multi-turn inference background gaps cannot be negative: " f"{gap_steps}" ) speeches = [ np.asarray(value, dtype=np.float32).reshape(-1) for value in speech_waveforms ] listen_counts = [_continuous_window_count(value.size) for value in speeches] turn_window_starts: list[int] = [] cursor = 0 for turn_index, count in enumerate(listen_counts): turn_window_starts.append(cursor) cursor += count if turn_index + 1 < len(listen_counts): cursor += gap_steps[turn_index] total_windows = cursor window_samples = int(round(AUDIO_WINDOW_SECONDS * AUDIO_RATE)) stride_samples = int( round(AUDIO_STRIDE_FRAMES / float(MOTION_FPS) * AUDIO_RATE) ) prefix_samples = window_samples - stride_samples flat_count = (total_windows - 1) * stride_samples + window_samples background_start = int(listen_start_index) * stride_samples background_end = background_start + flat_count background = np.asarray(background_waveform, dtype=np.float32).reshape(-1) if background_start < 0 or background_end > background.size: raise RuntimeError( "Selected background cannot cover multi-turn inference: " f"start={listen_start_index} windows={total_windows}" ) mixed = background[background_start:background_end].copy() previous_end = -1 for speech, scale, window_start in zip( speeches, speech_scales, turn_window_starts, ): start = prefix_samples + window_start * stride_samples end = start + speech.size if start < previous_end: raise RuntimeError( "Two inference commands overlap; interruption allows speech " "over motion, not simultaneous command waveforms" ) mixed[start:end] += speech * np.float32(scale) previous_end = end if float(np.max(np.abs(mixed), initial=0.0)) >= 1.0: raise RuntimeError("Multi-turn inference speech/background mix clips") mixed = ( np.rint(mixed * 32768.0) .clip(-32768, 32767) .astype("<i2") .astype(np.float32) / np.float32(32768.0) ) return ( np.ascontiguousarray(mixed, dtype=np.float32), turn_window_starts, listen_counts, ) def _read_background_sessions( continuous_audio_root: Path, ) -> dict[str, dict[str, Any]]: manifest = ( continuous_audio_root / "background" / "sessions" / "session_manifest.json" ) payload = json.loads(manifest.read_text(encoding="utf-8")) if payload.get("format") != "hfd_continuous_background_sessions_v1": raise RuntimeError(f"Unsupported background session manifest: {manifest}") result = { str(row["key"]): dict(row) for row in payload.get("sessions", []) } if not result: raise RuntimeError(f"No continuous background sessions in {manifest}") return result def _read_training_background_assignment( assignment_path: Path, audio_path: Path, ) -> dict[str, Any] | None: # The production manifest contains tens of thousands of rows on CPFS. # Calling Path.resolve() for every row performs a stat() on every path # component and turns a missing lookup (for example a newly generated Stop # command) into minutes of metadata I/O. Manifest paths are already # absolute; compare lexical, normalized keys and resolve only the one query # path so symlinked caller paths keep working. def path_key(value: str | os.PathLike[str]) -> str: relocated = relocate_hfd_storage_path(value) return os.path.normcase( os.path.abspath( os.path.normpath(os.fspath(relocated)) ) ) expanded = audio_path.expanduser() wanted = {path_key(expanded), path_key(expanded.resolve())} with assignment_path.open("r", encoding="utf-8") as handle: for line in handle: if not line.strip(): continue row = json.loads(line) if path_key(str(row["original_audio_path"])) in wanted: return row return None def _choose_continuous_background( *, sessions: dict[str, dict[str, Any]], speech_samples: int, motion_frames: int, initial_idle_steps: int, max_listen_tail_steps: int, seed_key: str, required_session_windows: int = 0, terminal_background_steps: int = 0, ) -> tuple[str, int]: listen_windows = ( int(required_session_windows) if int(required_session_windows) > 0 else _continuous_window_count(speech_samples) ) motion_windows = int( math.ceil(max(int(motion_frames), 1) / float(ACTION_EXEC_FRAMES)) ) required_after = listen_windows + motion_windows + max( int(max_listen_tail_steps), 1 ) + max(int(terminal_background_steps), 0) choices: list[tuple[str, int, int]] = [] total = 0 for key, row in sorted(sessions.items()): first = max(int(initial_idle_steps), 0) sample_count = int(row["sample_count"]) window_samples = int(round(AUDIO_WINDOW_SECONDS * AUDIO_RATE)) stride_samples = int( round(AUDIO_STRIDE_FRAMES / float(MOTION_FPS) * AUDIO_RATE) ) runtime_windows = ( 0 if sample_count < window_samples else (sample_count - window_samples) // stride_samples + 1 ) slots = runtime_windows - required_after - first + 1 if slots <= 0: continue choices.append((key, first, slots)) total += slots if total <= 0: raise RuntimeError( "No one-piece background session is long enough for inference" ) digest = hashlib.sha256(seed_key.encode("utf-8")).digest() slot = int.from_bytes(digest[:8], "big") % total for key, first, count in choices: if slot < count: return key, first + slot slot -= count raise AssertionError("Continuous background selection fell through") def _control_record(condition: FullDuplexStreamCondition) -> dict[str, Any]: probabilities = condition.control_probabilities.float()[0] if int(probabilities.numel()) != len(CONTROL_TOKENS): raise RuntimeError( "Full-duplex control head must predict listen/motion/idle: " f"logits={tuple(condition.control_logits.shape)}" ) token_id = int(probabilities.argmax().item()) record = { "predicted_id": token_id, "predicted_token": CONTROL_TOKENS[token_id], "listen_probability": float(probabilities[0].item()), "motion_probability": float(probabilities[1].item()), "idle_probability": float(probabilities[2].item()), "condition_weights": [ float(value) for value in condition.control_condition_weights.float()[0].tolist() ], "condition_mode": str(condition.control_condition_mode), "control_entropy": float(condition.control_entropy.float()[0].item()), "cache_tokens": int(condition.kv_cache.sequence_length), } duration_prediction = getattr( condition, "motion_duration_log1p_seconds", None, ) if duration_prediction is not None: log_duration = float( duration_prediction.float()[0].item() ) record["predicted_motion_duration_seconds"] = float( math.expm1(min(max(log_duration, 0.0), math.log1p(3600.0))) ) return record def _motion_duration_frames_from_condition( condition: FullDuplexStreamCondition, *, motion_fps: int, max_frames: int, ) -> int | None: """Decode one causal duration prediction into raw primitive frames.""" prediction = getattr(condition, "motion_duration_log1p_seconds", None) if prediction is None: return None fps = max(int(motion_fps), 1) upper = max(int(max_frames), 1) log_duration = float(prediction.float()[0].item()) seconds = math.expm1( min(max(log_duration, 0.0), math.log1p(upper / float(fps))) ) return min(max(int(round(seconds * fps)), 1), upper) def _future_progress( frame_id: int, emitted_motion_frames: int, transition_frames: int, *, inplace: bool = False, ) -> np.ndarray: """Match training's transition-plus-original-motion progress exactly. Training emits ``transition_frames`` from the stationary anchor to the original task's first pose, then appends original frames ``1..N-1``. Thus an emitted sequence of length ``M`` represents ``M-(transition-1)`` original task frames. The complete transition is progress zero. """ if inplace: from psi.runtime.swa_rollout import inplace_future_progress return inplace_future_progress(frame_id, emitted_motion_frames, ACTION_CHUNK_FRAMES) emitted = int(emitted_motion_frames) transition = int(transition_frames) if transition < 2: raise ValueError(f"transition_frames must be at least 2, got {transition}") original_total = emitted - (transition - 1) if original_total <= 0: raise ValueError( "Motion budget must include the stationary transition and at least " f"one original task frame: emitted={emitted}, transition={transition}" ) denominator = float(max(original_total - 1, 1)) rows = [] # ``frame_id`` is the number of already executed action frames, hence it # indexes the current observed state. The action token at position zero # is the *next* state (t+1), exactly as in the v26 dataset. RTC's ten # known positions are still part of this complete 30-frame timeline; they # must not shift the progress labels backwards by ten frames. for frame in range(frame_id + 1, frame_id + ACTION_CHUNK_FRAMES + 1): original_frame = max(frame - (transition - 1), 0) clipped = min(original_frame, original_total - 1) progress = float(clipped) / denominator rows.append( [ progress, max(0.0, 1.0 - progress), progress, math.sin(2.0 * math.pi * progress), math.cos(2.0 * math.pi * progress), ] ) return np.asarray(rows, dtype=np.float32) def assemble_stream_audio( *, speech_waveform: np.ndarray, white_noise_waveforms: Sequence[np.ndarray], control_events: list[dict[str, Any]], total_motion_frames: int, ) -> np.ndarray: """Overlap-average the exact 0.6 s windows consumed by streaming inference.""" total_samples = int( round(int(total_motion_frames) / float(MOTION_FPS) * AUDIO_RATE) ) if total_samples <= 0: raise ValueError( f"total_motion_frames must be positive, got {total_motion_frames}" ) window_samples = int(round(AUDIO_WINDOW_SECONDS * AUDIO_RATE)) mixed = np.zeros(total_samples, dtype=np.float64) weights = np.zeros(total_samples, dtype=np.float64) speech = np.asarray(speech_waveform, dtype=np.float32).reshape(-1) if not white_noise_waveforms: raise ValueError("At least one white-noise waveform is required") for event in control_events: input_kind = str(event["input_kind"]) if input_kind == "wav_audio": source_start = int(event["input_start_sample"]) audio_window = speech[source_start : source_start + window_samples] elif input_kind in { "initial_background", "post_wav_background", "terminal_background", }: noise_index = int(event["white_noise_index"]) audio_window = np.asarray( white_noise_waveforms[noise_index % len(white_noise_waveforms)], dtype=np.float32, ).reshape(-1) else: raise ValueError(f"Unsupported streaming input kind: {input_kind!r}") if audio_window.shape != (window_samples,): raise RuntimeError( "Streaming audio event does not contain one complete window: " f"kind={input_kind} shape={audio_window.shape}" ) output_frame = int(event["output_frame"]) if output_frame < 0: raise ValueError(f"Negative output_frame in audio event: {event}") destination_start = int(round(output_frame / float(MOTION_FPS) * AUDIO_RATE)) destination_end = min(destination_start + window_samples, total_samples) if destination_end <= destination_start: continue take = destination_end - destination_start mixed[destination_start:destination_end] += audio_window[:take] weights[destination_start:destination_end] += 1.0 valid = weights > 0 if not np.all(valid): missing = np.flatnonzero(~valid) raise RuntimeError( "Streaming control events do not cover the complete rendered audio " f"timeline: missing_samples={missing.size}, first={int(missing[0])}" ) mixed[valid] /= weights[valid] return np.clip(mixed, -1.0, 1.0).astype(np.float32) def write_pcm16_wav(path: Path, waveform: np.ndarray) -> None: values = np.asarray(waveform, dtype=np.float32).reshape(-1) pcm = np.round(np.clip(values, -1.0, 1.0) * 32767.0).astype("<i2") path.parent.mkdir(parents=True, exist_ok=True) temporary = path.with_name(f".{path.name}.tmp") with wave.open(str(temporary), "wb") as handle: handle.setnchannels(1) handle.setsampwidth(2) handle.setframerate(AUDIO_RATE) handle.writeframes(pcm.tobytes()) temporary.replace(path) def rollout_full_duplex( *, model: FullDuplexPsi0Model, background_waveforms: Sequence[np.ndarray], waveform: np.ndarray, prompt: str, source_id: str, init_abs: np.ndarray, launch_config: LaunchConfig, motion_frames: int, num_inference_steps: int, respect_control_token: bool, max_listen_tail_steps: int, initial_idle_steps: int, seed: int, white_noise_start_index: int = 0, continuous_background_listen_start_index: int | None = None, fixed_post_wav_background_steps: int = 0, terminal_background_steps: int = 0, ) -> tuple[np.ndarray, dict[str, Any]]: if motion_frames <= 0: raise ValueError("motion_frames must be positive") device = model.device starts = complete_audio_clip_starts(len(waveform)) if not starts: raise ValueError( f"Audio must contain at least one complete 0.6 s clip; got {len(waveform)} samples" ) initial_idle_steps = max(int(initial_idle_steps), 0) fixed_post_wav_background_steps = max( int(fixed_post_wav_background_steps), 0, ) terminal_background_steps = max(int(terminal_background_steps), 0) rollout_semantics_version = int( getattr( getattr(launch_config, "train", None), "full_duplex_semantics_version", 11, ) ) inplace_motion = bool(getattr(launch_config.data, "dynamic_task_entry_blend", False)) live_overlay = bool(getattr(launch_config.data, "multimodal_overlay_manifests", ())) first_transition_frames = int( getattr( launch_config.data, "turn_transition_frames" if rollout_semantics_version in {14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24} else "stationary_blend_frames", ACTION_EXEC_FRAMES, ) ) later_transition_frames = ( int( getattr( launch_config.data, "turn_transition_frames", first_transition_frames, ) ) if rollout_semantics_version in {13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24} else first_transition_frames ) if not inplace_motion and int(motion_frames) <= max( first_transition_frames, later_transition_frames, ) - 1: raise ValueError( "motion_frames must include the complete stationary-to-task " "transition: " f"motion_frames={motion_frames}, first={first_transition_frames}, " f"later={later_transition_frames}" ) cache = model.begin_source([source_id]) control_events: list[dict[str, Any]] = [] window_samples = int(round(AUDIO_WINDOW_SECONDS * AUDIO_RATE)) maxmin = launch_config.data.transform.field current_abs = np.asarray(init_abs, dtype=np.float32).copy() # The first source window has no previous executed action. It predicts a # complete 30-frame chunk; only subsequent windows use the previous # chunk's final 10 frames as the fixed RTC prefix. known_prefix_np: np.ndarray | None = None last_action_norm = None last_action_origin = None previous_executed_control: str | None = None timeline_generated: list[np.ndarray] = [] timeline_frame_count = 0 post_wav_motion_frames = 0 post_wav_background_steps = 0 task_motion_progress = 0 motion_turn_index = -1 active_transition_frames = first_transition_frames active_motion_frames = int(motion_frames) used_transition_frames: list[int] = [] predicted_motion_raw_frames_by_turn: list[int] = [] initial_input_motion_frames = 0 in_wav_motion_frames = 0 if not background_waveforms: raise ValueError("At least one raw background window is required") continuous_background = ( continuous_background_listen_start_index is not None ) if continuous_background: white_queries = ( int(continuous_background_listen_start_index) - initial_idle_steps ) if white_queries < 0: raise RuntimeError( "Continuous background has no room for requested initial idle" ) else: white_queries = int(white_noise_start_index) % len(background_waveforms) initial_white_noise_index = int(white_queries) listen_tail_steps = 0 first_any_motion_frame: int | None = None post_wav_motion_start_frame: int | None = None stream_cache_updates = 0 max_cache_tokens = 0 generator = torch.Generator(device=device) generator.manual_seed(int(seed)) def background_window(index: int) -> tuple[np.ndarray, int]: resolved = int(index) if continuous_background: if resolved < 0 or resolved >= len(background_waveforms): raise RuntimeError( "Continuous background window is outside its source: " f"index={resolved} count={len(background_waveforms)}" ) else: resolved %= len(background_waveforms) return np.asarray( background_waveforms[resolved], dtype=np.float32, ), resolved def encode_raw_window(audio_window: np.ndarray) -> FullDuplexStreamCondition: state_aware_control = bool( getattr( launch_config.model, "state_aware_serial_control", False, ) ) control_context_kwargs: dict[str, torch.Tensor] = {} if state_aware_control: states_np = build_model_states_from_abs( [current_abs], textop_actions=True, odim=int(launch_config.model.odim), ) if bool(maxmin.normalize_state): states_np = maxmin.normalize_state_func(states_np) states = torch.from_numpy(states_np).unsqueeze(0).to( device=device ) rtc_prefix_np = ( np.zeros( ( RTC_PREFIX_FRAMES, int( getattr( launch_config.model, "action_dim", model.action_dim, ) ), ), dtype=np.float32, ) if known_prefix_np is None else np.asarray(known_prefix_np, dtype=np.float32) ) rtc_mask_np = np.full( RTC_PREFIX_FRAMES, known_prefix_np is not None, dtype=np.bool_, ) previous_ids = { None: 3, "listen": 0, "motion": 1, "idle": 2, } if previous_executed_control == "motion": original_total = max( int(active_motion_frames) - (int(active_transition_frames) - 1), 1, ) original_frame = max( int(task_motion_progress) - (int(active_transition_frames) - 1), 0, ) progress_value = float( min(original_frame, original_total - 1) ) / float(max(original_total - 1, 1)) control_progress_np = np.asarray( [ progress_value, max(0.0, 1.0 - progress_value), progress_value, math.sin(2.0 * math.pi * progress_value), math.cos(2.0 * math.pi * progress_value), ], dtype=np.float32, ) else: control_progress_np = np.asarray( [0.0, 1.0, 0.0, 0.0, 1.0], dtype=np.float32, ) control_context_kwargs = { "states": states, "control_progress": torch.from_numpy(control_progress_np) .unsqueeze(0) .to(device=device), "previous_control_ids": torch.tensor( [previous_ids[previous_executed_control]], device=device, dtype=torch.long, ), "control_rtc_prefix": torch.from_numpy(rtc_prefix_np) .unsqueeze(0) .to(device=device), "control_rtc_mask": torch.from_numpy(rtc_mask_np) .unsqueeze(0) .to(device=device), } prepared = prepare_qwen_streaming_audio_window( model.vlm_processor, np.asarray(audio_window, dtype=np.float32), first_window=stream_cache_updates == 0, sampling_rate=AUDIO_RATE, ) if live_overlay: # Absence of a camera/question is a trained overlay mode, not a # switch back to the legacy audio prompt/token packing. from psi.data.seed.live_multimodal_overlay import live_audio_packet model.swa_packets = [live_audio_packet( prepared["stream_input_features"], prepared["stream_feature_attention_mask"], stream_cache_updates, )] with torch.inference_mode(), _autocast(device): return model.encode_stream_input( source_ids=[source_id], kv_cache=cache, update_semantic_cache=True, vlm_hidden_states=None, attention_mask=prepared["stream_attention_mask"] .unsqueeze(0) .to(device=device), stream_input_ids=prepared["stream_input_ids"] .unsqueeze(0) .to(device=device), stream_input_features=prepared["stream_input_features"] .unsqueeze(0) .to(device=device), stream_feature_attention_mask=prepared[ "stream_feature_attention_mask" ] .unsqueeze(0) .to(device=device), **control_context_kwargs, ) def accept_stream_condition( condition: FullDuplexStreamCondition, ) -> FullDuplexStreamCondition: nonlocal cache, stream_cache_updates stream_cache_updates += 1 expected = _expected_stream_cache_tokens( stream_cache_updates, max_cache_tokens, ) actual = int(condition.kv_cache.sequence_length) if actual != expected: raise RuntimeError( "Continuous source cache length mismatch: " f"updates={stream_cache_updates} max={max_cache_tokens} " f"actual={actual} expected={expected}" ) cache = condition.kv_cache return condition def append_timeline(chunk_abs: np.ndarray) -> None: nonlocal timeline_frame_count chunk = np.asarray(chunk_abs, dtype=np.float32) if chunk.ndim != 2 or chunk.shape[1] != 36: raise ValueError(f"Invalid generated motion chunk: {chunk.shape}") if not len(chunk): return timeline_generated.append(chunk) timeline_frame_count += int(chunk.shape[0]) def generate_action( condition: FullDuplexStreamCondition, *, task_frame: int | None, ) -> np.ndarray: nonlocal known_prefix_np, last_action_norm, last_action_origin states_np = build_model_states_from_abs( [current_abs], textop_actions=True, odim=int(launch_config.model.odim), ) if bool(maxmin.normalize_state): states_np = maxmin.normalize_state_func(states_np) states = torch.from_numpy(states_np).unsqueeze(0).to(device=device) known_prefix = ( None if known_prefix_np is None else torch.from_numpy(known_prefix_np).unsqueeze(0).to(device=device) ) progress_np = ( _stationary_future_progress(int(motion_frames)) if task_frame is None else _future_progress( int(task_frame), int(active_motion_frames), active_transition_frames, inplace=inplace_motion, ) ) future_progress = torch.from_numpy(progress_np).unsqueeze(0).to( device=device ) with torch.inference_mode(): predicted_norm = _flow_action( model, condition, known_prefix=known_prefix, states=states, future_progress=future_progress, num_inference_steps=num_inference_steps, generator=generator, )[0].cpu().numpy() predicted_norm = np.clip( predicted_norm.astype(np.float32), -1.0, 1.0, ) if known_prefix_np is not None and predicted_norm.shape[-1] == 40: predicted_norm[:RTC_PREFIX_FRAMES] = known_prefix_np chunk_abs = decode_action_chunk(predicted_norm, current_abs, maxmin) last_action_norm, last_action_origin = predicted_norm.copy(), current_abs.copy() known_prefix_np = next_rtc_prefix(predicted_norm, current_abs, chunk_abs[ACTION_EXEC_FRAMES-1], maxmin) return chunk_abs def execute_prediction( condition: FullDuplexStreamCondition, *, predicted_id: int, motion_budget: int | None = None, ) -> tuple[str, int]: """Execute this window's predicted state and close RTC on that output.""" nonlocal first_any_motion_frame, task_motion_progress, current_abs nonlocal motion_turn_index, active_transition_frames nonlocal active_motion_frames nonlocal known_prefix_np, previous_executed_control executed = _control_decision( predicted_id=int(predicted_id), respect_control_token=respect_control_token, ) active_train_config = getattr(launch_config, "train", None) known_prefix_np = _rtc_prefix_after_control_decision( known_prefix_np, previous_control=previous_executed_control, current_control=executed, semantics_version=int( getattr( active_train_config, "full_duplex_semantics_version", 11, ) ), ) entering_new_motion = ( executed == "motion" and previous_executed_control != "motion" ) if entering_new_motion: # Every listen -> motion decision begins a new primitive task in # v13. Its future-progress clock must restart instead of carrying # the previous turn's completed value into the next command. task_motion_progress = 0 motion_turn_index += 1 active_transition_frames = ( first_transition_frames if motion_turn_index == 0 else later_transition_frames ) used_transition_frames.append(active_transition_frames) predicted_raw_frames = _motion_duration_frames_from_condition( condition, motion_fps=MOTION_FPS, max_frames=( max( int(getattr(launch_config.data, "max_stream_steps", 1923)), 1, ) * ACTION_EXEC_FRAMES ), ) if predicted_raw_frames is None: active_motion_frames = int(motion_frames) else: # Training keeps the known stationary join at phase zero and # measures progress over the raw primitive. Rebuild that exact # emitted budget and freeze it for this complete motion turn. active_motion_frames = ( int(predicted_raw_frames) + (0 if inplace_motion else active_transition_frames - 1) ) predicted_motion_raw_frames_by_turn.append( int(predicted_raw_frames) ) previous_executed_control = executed output_frame = timeline_frame_count task_frame = task_motion_progress if executed == "motion" else None chunk_abs = generate_action(condition, task_frame=task_frame) # RTC always advances by the deployment horizon. The first model # call returns a full 30-frame chunk so its trailing 10 frames can be # reused as the next window's known prefix; they must not be emitted # twice here. take = min(int(chunk_abs.shape[0]), ACTION_EXEC_FRAMES) if executed == "motion" and motion_budget is not None: take = min(take, max(int(motion_budget), 0)) if take <= 0: return executed, 0 append_timeline(chunk_abs[:take]) # The next window observes the last frame that was actually executed, # never the unexecuted RTC tail. The latter is retained separately in # ``known_prefix_np``. This also handles the terminal partial chunk. current_abs = chunk_abs[take - 1].copy() if int(model.action_dim) == 40 and take != ACTION_EXEC_FRAMES: known_prefix_np = next_rtc_prefix(last_action_norm, last_action_origin, current_abs, maxmin, executed_frames=take) if executed == "motion": if first_any_motion_frame is None: first_any_motion_frame = output_frame task_motion_progress += take return executed, take # Before speech, stream synthetic background through the same cache and let # the model decide the output state; input kind never hard-codes execution. for idle_index in range(initial_idle_steps): audio_window, background_index = background_window(white_queries) condition = encode_raw_window(audio_window) condition = accept_stream_condition(condition) output_frame = timeline_frame_count control = _control_record(condition) executed, executed_frames = execute_prediction( condition, predicted_id=int(control["predicted_id"]), ) if executed == "motion": initial_input_motion_frames += executed_frames event = { "input_kind": "initial_background", "idle_index": idle_index, "white_noise_index": background_index, "output_frame": output_frame, "executed": executed, **control, } control_events.append(event) white_queries += 1 # Speech windows continue updating the same source cache. The control head # is authoritative on every window instead of being overwritten as listen. for clip_index, start_sample in enumerate(starts): condition = encode_raw_window( waveform[start_sample : start_sample + window_samples] ) condition = accept_stream_condition(condition) output_frame = timeline_frame_count control = _control_record(condition) executed, executed_frames = execute_prediction( condition, predicted_id=int(control["predicted_id"]), ) if executed == "motion": in_wav_motion_frames += executed_frames control_events.append( { "input_kind": "wav_audio", "clip_index": clip_index, "input_start_sample": start_sample, "input_end_sample": start_sample + window_samples, "input_start_seconds": start_sample / float(AUDIO_RATE), "input_end_seconds": (start_sample + window_samples) / float(AUDIO_RATE), "output_frame": output_frame, "executed": executed, **control, } ) speech_cache_tokens = int(cache.sequence_length) speech_qwen_cache_tokens = int(getattr(cache, "token_count", 0)) # Do not insert an unobserved pose hold here. Training advances once per # 0.4 s execution stride, so an extra non-modelled tail is a direct # train/infer mismatch and a visible source of stop-start motion. speech_tail_frames = 0 if continuous_background: white_queries = ( int(continuous_background_listen_start_index) + len(starts) ) def post_wav_complete() -> bool: if fixed_post_wav_background_steps > 0: return post_wav_background_steps >= fixed_post_wav_background_steps return post_wav_motion_frames >= int(motion_frames) while not post_wav_complete(): audio_window, background_index = background_window(white_queries) # Background audio is still a real observation and is appended to the # exact same native Qwen KV as idle and speech windows. condition = encode_raw_window(audio_window) condition = accept_stream_condition(condition) event = { "input_kind": "post_wav_background", "white_noise_index": background_index, "output_frame": timeline_frame_count, **_control_record(condition), } white_queries += 1 post_wav_background_steps += 1 executed, executed_frames = execute_prediction( condition, predicted_id=int(event["predicted_id"]), motion_budget=( None if fixed_post_wav_background_steps > 0 else int(motion_frames) - post_wav_motion_frames ), ) if executed != "motion": listen_tail_steps += 1 event["executed"] = executed control_events.append(event) if ( fixed_post_wav_background_steps <= 0 and listen_tail_steps >= max(int(max_listen_tail_steps), 1) ): post_wav_events = [ item for item in control_events if item["input_kind"] == "post_wav_background" ] token_counts = { token: sum( item["predicted_token"] == token for item in post_wav_events ) for token in CONTROL_TOKENS } last_event = post_wav_events[-1] message = ( "The model did not produce enough </motion> windows before " f"the post-WAV non-motion limit ({max_listen_tail_steps}). " f"Post-WAV predictions: {token_counts}; last probabilities: " f"listen={last_event['listen_probability']:.6f}, " f"motion={last_event['motion_probability']:.6f}, " f"idle={last_event['idle_probability']:.6f}. " "Inference will not falsify its control token by forcing motion." ) raise FullDuplexControlTimeout( message, diagnostics={ "error_type": "full_duplex_control_timeout", "message": message, "source_id": source_id, "audio_samples": int(len(waveform)), "audio_duration_seconds": len(waveform) / float(AUDIO_RATE), "speech_clip_count": len(starts), "initial_idle_steps": int(initial_idle_steps), "listen_tail_steps": int(listen_tail_steps), "post_wav_prediction_counts": token_counts, "stream_cache_updates": int(stream_cache_updates), "final_cache_tokens": int(cache.sequence_length), "streaming_max_cache_tokens": int(max_cache_tokens), "generated_frames_before_failure": int( timeline_frame_count ), "control_events": control_events, }, ) continue if post_wav_motion_start_frame is None: post_wav_motion_start_frame = int(event["output_frame"]) event["executed"] = "motion" control_events.append(event) post_wav_motion_frames += executed_frames # A multi-turn deployment must not end on the final motion frame. Continue # streaming the same physical background, state, RTC history and native # Qwen KV so the video exposes whether the model naturally returns to idle. # These windows are never forced to idle: every action and token remains the # model's own closed-loop prediction. terminal_events: list[dict[str, Any]] = [] for terminal_index in range(terminal_background_steps): audio_window, background_index = background_window(white_queries) condition = encode_raw_window(audio_window) condition = accept_stream_condition(condition) event = { "input_kind": "terminal_background", "terminal_index": terminal_index, "white_noise_index": background_index, "output_frame": timeline_frame_count, **_control_record(condition), } white_queries += 1 executed, _ = execute_prediction( condition, predicted_id=int(event["predicted_id"]), ) event["executed"] = executed control_events.append(event) terminal_events.append(event) complete_motion = np.concatenate(timeline_generated, axis=0) if ( fixed_post_wav_background_steps <= 0 and post_wav_motion_start_frame is None ): raise RuntimeError("Full-duplex rollout completed without a motion window") post_wav_prediction_counts = { token: sum( event["predicted_token"] == token for event in control_events if event["input_kind"] == "post_wav_background" ) for token in CONTROL_TOKENS } terminal_prediction_counts = { token: sum( event["predicted_token"] == token for event in terminal_events ) for token in CONTROL_TOKENS } terminal_idle_streak = 0 for event in reversed(terminal_events): if event["executed"] != "idle": break terminal_idle_streak += 1 metadata = { "source_id": source_id, "audio_samples": int(len(waveform)), "audio_duration_seconds": len(waveform) / float(AUDIO_RATE), "white_noise_start_index": initial_white_noise_index, "continuous_background": bool(continuous_background), "background_listen_start_index": ( int(continuous_background_listen_start_index) if continuous_background else None ), "initial_idle_steps": int(initial_idle_steps), "initial_idle_frames": int(initial_idle_steps * ACTION_EXEC_FRAMES), "initial_idle_seconds": initial_idle_steps * ACTION_EXEC_FRAMES / float(MOTION_FPS), "speech_start_frame_50fps": int( initial_idle_steps * ACTION_EXEC_FRAMES ), "speech_start_seconds": initial_idle_steps * ACTION_EXEC_FRAMES / float(MOTION_FPS), "speech_clip_count": len(starts), "speech_clip_start_samples": starts, "speech_cache_tokens": speech_cache_tokens, "final_cache_tokens": int(cache.sequence_length), "speech_qwen_cache_tokens": speech_qwen_cache_tokens, "final_qwen_cache_tokens": int(getattr(cache, "token_count", 0)), "stream_cache_updates": int(stream_cache_updates), "streaming_max_cache_tokens": int(max_cache_tokens), "speech_tail_hold_frames": int(speech_tail_frames), "listen_tail_steps": int(listen_tail_steps), "fixed_post_wav_background_steps": int( fixed_post_wav_background_steps ), "post_wav_background_steps": int(post_wav_background_steps), "post_wav_prediction_counts": post_wav_prediction_counts, "terminal_background_steps": int(terminal_background_steps), "terminal_prediction_counts": terminal_prediction_counts, "terminal_idle_streak": int(terminal_idle_streak), "terminal_idle_ratio": ( terminal_prediction_counts[IDLE_TOKEN] / float(terminal_background_steps) if terminal_background_steps > 0 else None ), "forced_motion_transition": False, "forced_motion_transition_count": 0, "initial_input_motion_frames": int(initial_input_motion_frames), "in_wav_motion_frames": int(in_wav_motion_frames), "first_any_motion_frame_50fps": ( int(first_any_motion_frame) if first_any_motion_frame is not None else None ), "post_wav_motion_start_frame_50fps": ( int(post_wav_motion_start_frame) if post_wav_motion_start_frame is not None else None ), # Backward-compatible keys retain their historical post-WAV meaning. "motion_start_frame_50fps": ( int(post_wav_motion_start_frame) if post_wav_motion_start_frame is not None else None ), "motion_start_seconds": ( post_wav_motion_start_frame / float(MOTION_FPS) if post_wav_motion_start_frame is not None else None ), "pretask_motion_frames": ( int(post_wav_motion_start_frame) if post_wav_motion_start_frame is not None else None ), "motion_frames": int(post_wav_motion_frames), "task_motion_progress_frames": int(task_motion_progress), "task_transition_frames": first_transition_frames, "task_transition_frames_by_turn": used_transition_frames, "predicted_motion_raw_frames_by_turn": ( predicted_motion_raw_frames_by_turn ), "predicted_motion_seconds_by_turn": [ frames / float(MOTION_FPS) for frames in predicted_motion_raw_frames_by_turn ], "future_progress_original_frames": int( active_motion_frames - (0 if inplace_motion else active_transition_frames - 1) ), "future_progress_geometry": "inplace" if inplace_motion else "prefix", "semantic_input_format": "live_av_audio_only" if live_overlay else "legacy_audio", "total_frames": int(complete_motion.shape[0]), "control_events": control_events, } return complete_motion, metadata def export_motion(output_dir: Path, motion: np.ndarray) -> None: output_dir.mkdir(parents=True, exist_ok=True) joint_pos = motion[:, :29] body_pos = motion[:, 29:32] body_quat = normalize_quaternion_rows(motion[:, 32:36]) dt = 1.0 / float(TARGET_FPS) write_csv(output_dir / "joint_pos.csv", joint_pos, [f"joint_{index}" for index in range(29)]) write_csv(output_dir / "joint_vel.csv", finite_difference(joint_pos, dt), [f"joint_vel_{index}" for index in range(29)]) write_csv(output_dir / "body_pos.csv", body_pos, ["body_0_x", "body_0_y", "body_0_z"]) write_csv(output_dir / "body_quat.csv", body_quat, ["body_0_w", "body_0_x", "body_0_y", "body_0_z"]) write_csv(output_dir / "body_lin_vel.csv", finite_difference(body_pos, dt), ["body_0_vel_x", "body_0_vel_y", "body_0_vel_z"]) write_csv(output_dir / "body_ang_vel.csv", quaternion_angular_velocity(body_quat, dt), ["body_0_angvel_x", "body_0_angvel_y", "body_0_angvel_z"]) np.save(output_dir / "motion.npy", motion.astype(np.float32)) write_metadata(output_dir / "metadata.txt", output_dir.name, int(motion.shape[0])) def parse_args() -> argparse.Namespace: parser = argparse.ArgumentParser( description="HFD source-scoped full-duplex streaming audio inference" ) parser.add_argument("--run-dir", required=True) parser.add_argument("--ckpt-step", default="latest") parser.add_argument("--output-dir", required=True) parser.add_argument("--audio-path", required=True) parser.add_argument( "--additional-audio-path", action="append", default=[], help=( "Additional command WAVs in the same v13 KV session. The cache, " "background, robot state and RTC history are not reset." ), ) parser.add_argument("--init-motion-dir", required=True) parser.add_argument("--motion-frames", type=int, default=300) parser.add_argument( "--inter-turn-background-steps", type=int, default=0, help=( "Continuous 0.4 s background windows between command WAVs. Zero " "covers the prior task plus its longest aligned settling join." ), ) parser.add_argument( "--inter-turn-background-steps-by-turn", default="", help=( "Optional comma-separated background-window counts for adjacent " "turn pairs. This makes a deterministic multi-turn evaluation " "exercise different waiting durations without resetting KV." ), ) parser.add_argument( "--terminal-background-steps", type=int, default=0, help=( "After the final requested motion budget, continue this many " "0.4 s background windows without forcing a control token." ), ) parser.add_argument("--num-inference-steps", type=int, default=10) parser.add_argument("--prompt", default="auto") parser.add_argument("--backbone", default="") parser.add_argument( "--white-noise-audio-dir", default="", help=( "Legacy argument retained for CLI compatibility; HFDv2 reads the " "continuous raw background session bound by the run contract." ), ) parser.add_argument( "--white-noise-start-index", type=int, default=0, help="First cyclic white-noise clip index for this source session.", ) parser.add_argument( "--expected-checkpoint-global-step", type=int, default=0, help=( "Optional immutable checkpoint identity captured by the automatic " "watcher. A mismatch aborts instead of reusing a stale best output." ), ) parser.add_argument("--device", default="cuda:0") parser.add_argument("--seed", type=int, default=42) parser.add_argument("--ignore-control-token", action="store_true") parser.add_argument( "--control-condition-mode", choices=("hard", "soft"), default="", help=( "Optional explicit serial-control ablation. Empty uses the " "checkpoint's inference mode (production: hard)." ), ) parser.add_argument( "--max-listen-tail-steps", type=int, default=500, help=( "Maximum post-WAV non-motion windows before inference fails " "without overriding the model token." ), ) parser.add_argument( "--initial-idle-steps", type=int, default=0, help="Number of 0.4 s synthetic background windows before the WAV.", ) return parser.parse_args() def main() -> None: args = parse_args() run_dir = relocate_hfd_storage_path(args.run_dir).resolve() output_dir = relocate_hfd_storage_path(args.output_dir).resolve() audio_path = relocate_hfd_storage_path(args.audio_path).resolve() audio_paths = [audio_path] + [ relocate_hfd_storage_path(value).resolve() for value in args.additional_audio_path ] init_motion_dir = relocate_hfd_storage_path(args.init_motion_dir).resolve() missing_audio = [path for path in audio_paths if not path.is_file()] if missing_audio: raise FileNotFoundError(f"Audio does not exist: {missing_audio[0]}") if not init_motion_dir.is_dir(): raise FileNotFoundError(f"Initial motion does not exist: {init_motion_dir}") output_dir.mkdir(parents=True, exist_ok=True) cfg_template: LaunchConfig = parse_args_to_tyro_config(run_dir / "argv.txt") # type: ignore raw_run_config = json.loads( (run_dir / "run_config.json").read_text(encoding="utf-8") ) saved_semantics_version = ( raw_run_config.get("train", {}).get("full_duplex_semantics_version") if isinstance(raw_run_config, dict) else None ) if saved_semantics_version not in {9, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24}: raise RuntimeError( "Refusing fused Qwen-KV/serial-control inference on a legacy " "checkpoint" ) semantics_version = int(saved_semantics_version) launch_config = cfg_template.model_validate(raw_run_config) if int(getattr(launch_config.train, "stage", 0)) == 1: raise RuntimeError("Stage 1 trains semantic/control only; use a stage-2 checkpoint for motion inference") validate_contract(launch_config) seed_everything(launch_config.seed or args.seed) device = torch.device(args.device) ckpt_step = resolve_ckpt_step(run_dir, args.ckpt_step) backbone = ( str(args.backbone).strip() or str(Path(str(launch_config.model.model_name_or_path)).expanduser()) ) if not (Path(backbone) / "config.json").is_file(): raise FileNotFoundError(f"Invalid local Qwen backbone: {backbone}") continuous_root = relocate_hfd_storage_path( str(launch_config.data.continuous_audio_root) ) background_assignment_path = relocate_hfd_storage_path( str(launch_config.data.background_assignment_path) ) if not continuous_root.is_absolute(): continuous_root = PROJECT_ROOT / continuous_root if not background_assignment_path.is_absolute(): background_assignment_path = ( PROJECT_ROOT / background_assignment_path ) continuous_audio_production_identity( continuous_root, assignment_path=background_assignment_path, ) active_contract = _build_active_hfdv2_contract( launch_config, semantics_version=semantics_version, backbone=backbone, continuous_audio_root=continuous_root, background_assignment_path=background_assignment_path, ) if getattr(launch_config.model, "interaction_architecture", "legacy") == "a1_c_deferred": from psi.models.interaction_arch.runtime import ARCH_KEY, ArchitectureSpec, architecture_contract active_contract[ARCH_KEY] = architecture_contract(ArchitectureSpec("a1", interaction="cross")) expected_checkpoint_step = ( int(args.expected_checkpoint_global_step) if int(args.expected_checkpoint_global_step) > 0 else None ) checkpoint_marker = validate_checkpoint_encoder_contract( run_dir, ckpt_step, active_contract, expected_global_step=expected_checkpoint_step, ) os.environ["PSI_PRETRAIN_BACKBONE_PATH"] = backbone model_class = FullDuplexPsi0Model if bool(getattr(launch_config.model, "swa_visual_qa_enabled", False)): from psi.models.full_duplex_swa import FullDuplexSwaModel from types import SimpleNamespace from psi.models.interaction_arch.loading import load_swa_checkpoint model_class = SimpleNamespace(from_streaming_checkpoint=load_swa_checkpoint) model = model_class.from_streaming_checkpoint( run_dir, ckpt_step, launch_config, device ) control_condition_mode = ( str(args.control_condition_mode).strip() or str(launch_config.model.control_condition_inference_mode) ) model.set_control_condition_inference_mode(control_condition_mode) marker_after_load = validate_checkpoint_encoder_contract( run_dir, ckpt_step, active_contract, expected_global_step=int(checkpoint_marker["global_step"]), ) if marker_after_load != checkpoint_marker: raise RuntimeError( "Checkpoint metadata changed while model weights were loading; retry " "inference to avoid an ambiguous mutable best_ckpt snapshot." ) normalized_audio_paths: list[Path] = [] clean_waveforms: list[np.ndarray] = [] assignments: list[dict[str, Any] | None] = [] for turn_index, turn_audio_path in enumerate(audio_paths): normalized_path = ( output_dir / "input_audio_16k_mono.wav" if turn_index == 0 else output_dir / f"input_audio_turn_{turn_index + 1:02d}_16k_mono.wav" ) clean = _normalise_audio(turn_audio_path, normalized_path) assignment = _read_training_background_assignment( background_assignment_path, turn_audio_path, ) if assignment is not None: # The offline assignment builder used scipy.resample_poly for the # 24 kHz TTS corpus. Reuse it for every known training command. assigned = _read_resampled_speech_float( turn_audio_path, AUDIO_RATE, ) if assigned.size != int(assignment["speech_samples"]): raise RuntimeError( "Training speech length changed after assignment: " f"path={turn_audio_path} actual={assigned.size} " f"expected={assignment['speech_samples']}" ) clean = assigned normalized_audio_paths.append(normalized_path) clean_waveforms.append(clean) assignments.append(assignment) normalized_audio_path = normalized_audio_paths[0] clean_waveform = clean_waveforms[0] waveform = clean_waveform continuous_background_listen_start_index: int | None = None sessions = _read_background_sessions(continuous_root) assignment = assignments[0] is_multiturn = len(audio_paths) > 1 if is_multiturn and semantics_version not in {13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24}: raise RuntimeError( "Multiple command WAVs require a multi-turn semantics-v13+ checkpoint" ) # A single-turn benchmark historically waits until the model has emitted # the requested number of motion frames. That stopping rule is invalid # for a multi-turn session: earlier commands already execute inside the # inter-turn gaps, and the final command may naturally return to idle. # Give the final command one fixed wall-clock action horizon and render # every predicted state instead of waiting for extra </motion> tokens. fixed_post_wav_background_steps = ( int( math.ceil( int(args.motion_frames) / float(ACTION_EXEC_FRAMES) ) ) if is_multiturn else 0 ) default_inter_turn_background_steps = ( max(int(args.inter_turn_background_steps), 1) if int(args.inter_turn_background_steps) > 0 else int( math.ceil( ( int(args.motion_frames) + int( getattr( launch_config.data, "turn_transition_frames", ACTION_EXEC_FRAMES, ) ) + ACTION_EXEC_FRAMES - 1 ) / float(ACTION_EXEC_FRAMES) ) ) ) inter_turn_background_steps_by_turn: list[int] = [] if is_multiturn: raw_gap_steps = str(args.inter_turn_background_steps_by_turn).strip() if raw_gap_steps: try: inter_turn_background_steps_by_turn = [ int(value.strip()) for value in raw_gap_steps.split(",") if value.strip() ] except ValueError as error: raise ValueError( "--inter-turn-background-steps-by-turn must be a " f"comma-separated integer list, got {raw_gap_steps!r}" ) from error expected_gaps = len(clean_waveforms) - 1 if len(inter_turn_background_steps_by_turn) != expected_gaps: raise ValueError( "--inter-turn-background-steps-by-turn needs one value " f"per adjacent pair: turns={len(clean_waveforms)} " f"gaps={inter_turn_background_steps_by_turn}" ) if any(value < 0 for value in inter_turn_background_steps_by_turn): raise ValueError( "--inter-turn-background-steps-by-turn cannot contain " f"negative values: {inter_turn_background_steps_by_turn}" ) else: inter_turn_background_steps_by_turn = [ default_inter_turn_background_steps ] * (len(clean_waveforms) - 1) expected_exported_listen_windows = _exported_window_count( clean_waveform.size ) if ( not is_multiturn and assignment is not None and int(assignment["listen_window_count"]) == expected_exported_listen_windows ): background_key = str(assignment["background_key"]) continuous_background_listen_start_index = ( int(assignment["background_listen_start_index"]) * EXPORTED_AUDIO_STRIDE_FRAMES // AUDIO_STRIDE_FRAMES ) selection = "exact_training_assignment" else: session_window_count = ( sum(_continuous_window_count(value.size) for value in clean_waveforms) + sum(inter_turn_background_steps_by_turn) ) ( background_key, continuous_background_listen_start_index, ) = _choose_continuous_background( sessions=sessions, speech_samples=clean_waveform.size, motion_frames=int(args.motion_frames), initial_idle_steps=max(int(args.initial_idle_steps), 0), max_listen_tail_steps=max( int(args.max_listen_tail_steps), 1 ), seed_key=( "::".join(str(path) for path in audio_paths) + f"::{int(args.seed)}" ), required_session_windows=( session_window_count if is_multiturn else 0 ), terminal_background_steps=max( int(args.terminal_background_steps), 0 ), ) selection = ( "deterministic_multiturn_assignment" if is_multiturn else "deterministic_inference_assignment" ) session = sessions.get(background_key) if session is None: raise RuntimeError( f"Assignment names an unknown background: {background_key}" ) background_audio_path = Path(str(session["path"])) if _file_sha256(background_audio_path) != str(session["sha256"]): raise RuntimeError( f"Continuous background WAV changed after production: " f"{background_audio_path}" ) background_waveform = load_omni_audio_array( background_audio_path, AUDIO_RATE, ) turn_window_starts = [0] turn_listen_counts = [_continuous_window_count(clean_waveform.size)] if is_multiturn: peak_limit = 10.0 ** (-3.0 / 20.0) speech_scales = [] for clean, turn_assignment in zip(clean_waveforms, assignments): if turn_assignment is not None: speech_scales.append(float(turn_assignment["speech_scale"])) continue peak = float(np.max(np.abs(clean), initial=0.0)) speech_scales.append( min(1.0, peak_limit / peak) if peak > 0.0 else 1.0 ) ( waveform, turn_window_starts, turn_listen_counts, ) = _build_continuous_multiturn_waveform( speech_waveforms=clean_waveforms, speech_scales=speech_scales, background_waveform=background_waveform, listen_start_index=continuous_background_listen_start_index, inter_turn_background_steps=inter_turn_background_steps_by_turn, ) speech_scale: float | list[float] = speech_scales elif selection == "exact_training_assignment": waveform, speech_scale = _build_continuous_mixed_waveform( speech_waveform=clean_waveform, background_waveform=background_waveform, listen_start_index=continuous_background_listen_start_index, ) if not math.isclose( speech_scale, float(assignment["speech_scale"]), rel_tol=0.0, abs_tol=1e-8, ): raise RuntimeError("Runtime speech scale differs from training assignment") if ( _continuous_window_count(clean_waveform.size) != len(complete_audio_clip_starts(len(waveform))) ): raise RuntimeError( "Exact training mixed WAV no longer matches its assignment: " f"{audio_path}" ) else: waveform, speech_scale = _build_continuous_mixed_waveform( speech_waveform=clean_waveform, background_waveform=background_waveform, listen_start_index=continuous_background_listen_start_index, ) mixed_audio_path = output_dir / "input_audio_mixed_16k_mono.wav" write_pcm16_wav(mixed_audio_path, waveform) window_samples = int(round(AUDIO_WINDOW_SECONDS * AUDIO_RATE)) background_starts = complete_audio_clip_starts( len(background_waveform) ) white_noise_audio = [ background_waveform[ start : start + window_samples ] for start in background_starts ] if any( window.shape != (window_samples,) for window in white_noise_audio ): raise RuntimeError( f"Background WAV/index count mismatch: {background_key}" ) background_metadata = { "semantics_version": semantics_version, "mode": "continuous_source_local", "selection": selection, "background_key": background_key, "background_listen_start_index": int( continuous_background_listen_start_index ), "background_window_count": len(white_noise_audio), "speech_scale": speech_scale, "turn_window_starts": turn_window_starts, "turn_listen_window_counts": turn_listen_counts, "inter_turn_background_steps": ( default_inter_turn_background_steps if is_multiturn else 0 ), "inter_turn_background_steps_by_turn": ( inter_turn_background_steps_by_turn if is_multiturn else [] ), "fixed_post_wav_background_steps": int( fixed_post_wav_background_steps ), "mixed_audio_path": str(mixed_audio_path), } prompt = resolve_prompt(args.prompt, audio_path, launch_config) validate_online_prompt(prompt) init_abs = load_pose_from_motion_dir(init_motion_dir) audio_prefix_seconds = AUDIO_WINDOW_SECONDS - ( AUDIO_STRIDE_FRAMES / float(MOTION_FPS) ) audible_speech_onset_seconds = [ max(int(args.initial_idle_steps), 0) * AUDIO_STRIDE_FRAMES / float(MOTION_FPS) + int(window_start) * AUDIO_STRIDE_FRAMES / float(MOTION_FPS) + audio_prefix_seconds + detect_clean_speech_onset_sample(clean, AUDIO_RATE) / float(AUDIO_RATE) for clean, window_start in zip(clean_waveforms, turn_window_starts) ] try: motion, metadata = rollout_full_duplex( model=model, background_waveforms=white_noise_audio, waveform=waveform, prompt=prompt, source_id="::".join(str(path) for path in audio_paths), init_abs=init_abs, launch_config=launch_config, motion_frames=int(args.motion_frames), num_inference_steps=int(args.num_inference_steps), respect_control_token=not bool(args.ignore_control_token), max_listen_tail_steps=max(int(args.max_listen_tail_steps), 1), initial_idle_steps=max(int(args.initial_idle_steps), 0), seed=int(args.seed), white_noise_start_index=int(args.white_noise_start_index), continuous_background_listen_start_index=( continuous_background_listen_start_index ), fixed_post_wav_background_steps=int( fixed_post_wav_background_steps ), terminal_background_steps=max( int(args.terminal_background_steps), 0 ), ) except FullDuplexControlTimeout as error: failure = dict(error.diagnostics) failure.update( { "run_dir": str(run_dir), "ckpt_step": ckpt_step, "audio_path": str(audio_path), "audio_paths": [str(path) for path in audio_paths], "normalized_audio_path": str(normalized_audio_path), "normalized_audio_paths": [ str(path) for path in normalized_audio_paths ], "init_motion_dir": str(init_motion_dir), "prompt": prompt, "respect_control_token": not bool( args.ignore_control_token ), "initial_idle_steps_requested": max( int(args.initial_idle_steps), 0, ), "num_inference_steps": int(args.num_inference_steps), "white_noise_start_index": int(args.white_noise_start_index), "terminal_background_steps_requested": max( int(args.terminal_background_steps), 0 ), "fixed_post_wav_background_steps": int( fixed_post_wav_background_steps ), } ) (output_dir / "infer_failure.json").write_text( json.dumps(failure, ensure_ascii=False, indent=2) + "\n", encoding="utf-8", ) shutil.copy2( run_dir / "run_config.json", output_dir / "run_config.snapshot.json", ) raise export_motion(output_dir, motion) assembled_audio_path = output_dir / "assembled_input_audio_16k.wav" assembled_audio = assemble_stream_audio( speech_waveform=waveform, white_noise_waveforms=white_noise_audio, control_events=metadata["control_events"], total_motion_frames=int(metadata["total_frames"]), ) write_pcm16_wav(assembled_audio_path, assembled_audio) metadata.update( { "run_dir": str(run_dir), "ckpt_step": ckpt_step, "audio_path": str(audio_path), "audio_paths": [str(path) for path in audio_paths], "normalized_audio_path": str(normalized_audio_path), "normalized_audio_paths": [ str(path) for path in normalized_audio_paths ], "assembled_input_audio_path": str(assembled_audio_path), "background_audio": background_metadata, "init_motion_dir": str(init_motion_dir), "prompt": prompt, "respect_control_token": not bool(args.ignore_control_token), "initial_idle_steps_requested": max( int(args.initial_idle_steps), 0, ), "terminal_background_steps_requested": max( int(args.terminal_background_steps), 0 ), "fixed_post_wav_background_steps_requested": int( fixed_post_wav_background_steps ), "num_inference_steps": int(args.num_inference_steps), "checkpoint_global_step": int(checkpoint_marker["global_step"]), "hfdv2_train_infer_contract": active_contract, "control_condition_mode": control_condition_mode, "white_noise_start_index_requested": int( args.white_noise_start_index ), "audio_sampling_rate": AUDIO_RATE, "motion_fps": MOTION_FPS, "audio_window_seconds": AUDIO_WINDOW_SECONDS, "audio_stride_frames": AUDIO_STRIDE_FRAMES, "action_chunk_frames": ACTION_CHUNK_FRAMES, "action_exec_frames": ACTION_EXEC_FRAMES, "rtc_prefix_frames": RTC_PREFIX_FRAMES, "speech_start_seconds": float(audible_speech_onset_seconds[0]), "speech_start_frame_50fps": int( round(audible_speech_onset_seconds[0] * MOTION_FPS) ), "turn_audible_speech_onset_seconds": [ float(value) for value in audible_speech_onset_seconds ], "audio_timeline": { "format": "overlap_average_stream_windows_v1", "sample_rate": AUDIO_RATE, "window_samples": int( round(AUDIO_WINDOW_SECONDS * AUDIO_RATE) ), "stride_samples": int( round(AUDIO_STRIDE_FRAMES / MOTION_FPS * AUDIO_RATE) ), "samples": int(assembled_audio.shape[0]), "includes_initial_background": True, "includes_speech": True, "includes_post_wav_background": True, "includes_terminal_background": bool( int(args.terminal_background_steps) > 0 ), }, } ) (output_dir / "infer_meta.json").write_text( json.dumps(metadata, ensure_ascii=False, indent=2) + "\n", encoding="utf-8", ) shutil.copy2(run_dir / "run_config.json", output_dir / "run_config.snapshot.json") motion_start_seconds = metadata["motion_start_seconds"] motion_start_text = ( f"{float(motion_start_seconds):.3f}s" if motion_start_seconds is not None else "none" ) print( f"[done] clips={metadata['speech_clip_count']} " f"cache={metadata['speech_cache_tokens']}->{metadata['final_cache_tokens']} " f"updates={metadata['stream_cache_updates']} " f"motion_start={motion_start_text} frames={metadata['total_frames']} " f"output={output_dir}", flush=True, ) if __name__ == "__main__": main()
#!/usr/bin/env python3
from __future__ import annotations
import argparse
import contextlib
import hashlib
import json
import math
import os
import shutil
import subprocess
import sys
import wave
from pathlib import Path
from typing import Any, Sequence
import numpy as np
import torch
PROJECT_ROOT = Path(__file__).resolve().parents[2]
DATA_SCRIPT_DIR = PROJECT_ROOT / "scripts" / "data"
INFER_SCRIPT_DIR = PROJECT_ROOT / "scripts" / "infer"
if str(DATA_SCRIPT_DIR) not in sys.path:
sys.path.insert(0, str(DATA_SCRIPT_DIR))
if str(INFER_SCRIPT_DIR) not in sys.path:
sys.path.insert(0, str(INFER_SCRIPT_DIR))
from cache_omni_hidden_states import ( # noqa: E402
load_audio_array as load_omni_audio_array,
)
from psi.config.config import LaunchConfig # noqa: E402
from psi.data.seed.full_duplex_audio_contract import ( # noqa: E402
AUDIO_COMMAND_STREAMING_PROMPT,
)
from psi.data.seed.full_duplex_audio_command_dataset import ( # noqa: E402
_read_resampled_speech_float,
detect_clean_speech_onset_sample,
)
from psi.data.seed.qwen_streaming_audio import ( # noqa: E402
continuous_audio_production_identity,
hfdv2_train_infer_contract,
prepare_qwen_streaming_audio_window,
)
from psi.data.seed.storage_paths import ( # noqa: E402
relocate_hfd_storage_path,
)
from psi.data.seed.motion_repr import ( # noqa: E402
build_seed_textop_action,
recover_seed_abs_from_textop_action,
)
from psi.data.seed.full_duplex_timing import ( # noqa: E402
ACTION_CHUNK_FRAMES,
ACTION_EXEC_FRAMES,
ACTION_SAMPLE_STEP_FRAMES,
AUDIO_RATE,
AUDIO_STRIDE_FRAMES,
AUDIO_WINDOW_SECONDS,
MOTION_FPS,
RTC_PREFIX_FRAMES,
complete_audio_clip_starts,
)
from psi.models.full_duplex_psi0 import ( # noqa: E402
FullDuplexPsi0Model,
FullDuplexStreamCondition,
)
from psi.utils import parse_args_to_tyro_config, seed_everything # noqa: E402
from psi.runtime.motion_rollout import _flow_action, _stationary_future_progress # noqa: E402
from psi.runtime.motion_rollout import decode_action_chunk, next_rtc_prefix
from seed_audio_to_motion import normalize_action_for_rtc # noqa: E402
from seed_text_to_motion import ( # noqa: E402
TARGET_FPS,
build_model_states_from_abs,
finite_difference,
load_pose_from_motion_dir,
normalize_quaternion_rows,
quaternion_angular_velocity,
resolve_ckpt_step,
write_csv,
write_metadata,
)
LISTEN_TOKEN = "</listen>"
MOTION_TOKEN = "</motion>"
IDLE_TOKEN = "</idle>"
CONTROL_TOKENS = (LISTEN_TOKEN, MOTION_TOKEN, IDLE_TOKEN)
DEFAULT_COMMAND_PROMPT = AUDIO_COMMAND_STREAMING_PROMPT
EXPORTED_AUDIO_STRIDE_FRAMES = 20
class FullDuplexControlTimeout(RuntimeError):
"""A strict rollout ended because the model never selected motion."""
def __init__(self, message: str, *, diagnostics: dict[str, Any]) -> None:
super().__init__(message)
self.diagnostics = diagnostics