Download scripts/infer/full_duplex_audio_to_motion.py from Tinghong-Ye22/code-HFD: direct link, hf CLI and curl.
- Browser
- Download file 107 kB
-
https://huggingface.co/Tinghong-Ye22/code-HFD/resolve/main/scripts/infer/full_duplex_audio_to_motion.py
- Command line
-
hf download hf://Tinghong-Ye22/code-HFD/scripts/infer/full_duplex_audio_to_motion.py
-
curl -L -o full_duplex_audio_to_motion.py https://huggingface.co/Tinghong-Ye22/code-HFD/resolve/main/scripts/infer/full_duplex_audio_to_motion.py
107 kB
#!/usr/bin/env python3
from __future__ import annotations
import argparse
import contextlib
import hashlib
import json
import math
import os
import shutil
import subprocess
import sys
import wave
from pathlib import Path
from typing import Any, Sequence
import numpy as np
import torch
PROJECT_ROOT = Path(__file__).resolve().parents[2]
DATA_SCRIPT_DIR = PROJECT_ROOT / "scripts" / "data"
INFER_SCRIPT_DIR = PROJECT_ROOT / "scripts" / "infer"
if str(DATA_SCRIPT_DIR) not in sys.path:
sys.path.insert(0, str(DATA_SCRIPT_DIR))
if str(INFER_SCRIPT_DIR) not in sys.path:
sys.path.insert(0, str(INFER_SCRIPT_DIR))
from cache_omni_hidden_states import ( # noqa: E402
load_audio_array as load_omni_audio_array,
)
from psi.config.config import LaunchConfig # noqa: E402
from psi.data.seed.full_duplex_audio_contract import ( # noqa: E402
AUDIO_COMMAND_STREAMING_PROMPT,
)
from psi.data.seed.full_duplex_audio_command_dataset import ( # noqa: E402
_read_resampled_speech_float,
detect_clean_speech_onset_sample,
)
from psi.data.seed.qwen_streaming_audio import ( # noqa: E402
continuous_audio_production_identity,
hfdv2_train_infer_contract,
prepare_qwen_streaming_audio_window,
)
from psi.data.seed.storage_paths import ( # noqa: E402
relocate_hfd_storage_path,
)
from psi.data.seed.motion_repr import ( # noqa: E402
build_seed_textop_action,
recover_seed_abs_from_textop_action,
)
from psi.data.seed.full_duplex_timing import ( # noqa: E402
ACTION_CHUNK_FRAMES,
ACTION_EXEC_FRAMES,
ACTION_SAMPLE_STEP_FRAMES,
AUDIO_RATE,
AUDIO_STRIDE_FRAMES,
AUDIO_WINDOW_SECONDS,
MOTION_FPS,
RTC_PREFIX_FRAMES,
complete_audio_clip_starts,
)
from psi.models.full_duplex_psi0 import ( # noqa: E402
FullDuplexPsi0Model,
FullDuplexStreamCondition,
)
from psi.utils import parse_args_to_tyro_config, seed_everything # noqa: E402
from psi.runtime.motion_rollout import _flow_action, _stationary_future_progress # noqa: E402
from psi.runtime.motion_rollout import decode_action_chunk, next_rtc_prefix
from seed_audio_to_motion import normalize_action_for_rtc # noqa: E402
from seed_text_to_motion import ( # noqa: E402
TARGET_FPS,
build_model_states_from_abs,
finite_difference,
load_pose_from_motion_dir,
normalize_quaternion_rows,
quaternion_angular_velocity,
resolve_ckpt_step,
write_csv,
write_metadata,
)
LISTEN_TOKEN = "</listen>"
MOTION_TOKEN = "</motion>"
IDLE_TOKEN = "</idle>"
CONTROL_TOKENS = (LISTEN_TOKEN, MOTION_TOKEN, IDLE_TOKEN)
DEFAULT_COMMAND_PROMPT = AUDIO_COMMAND_STREAMING_PROMPT
EXPORTED_AUDIO_STRIDE_FRAMES = 20
class FullDuplexControlTimeout(RuntimeError):
"""A strict rollout ended because the model never selected motion."""
def __init__(self, message: str, *, diagnostics: dict[str, Any]) -> None:
super().__init__(message)
self.diagnostics = diagnostics
def _expected_stream_cache_tokens(
consumed_windows: int,
max_cache_tokens: int,
) -> int:
"""Return the visible cache length after one-source causal rolling."""
consumed = max(int(consumed_windows), 0)
limit = int(max_cache_tokens)
return consumed if limit <= 0 else min(consumed, limit)
def _control_decision(
*,
predicted_id: int,
respect_control_token: bool,
) -> str:
"""Choose this window's execution directly from this window's token."""
token_id = int(predicted_id)
if token_id < 0 or token_id >= len(CONTROL_TOKENS):
raise ValueError(f"Invalid full-duplex control id: {token_id}")
if not bool(respect_control_token):
return "motion"
return ("listen", "motion", "idle")[token_id]
def _rtc_prefix_after_control_decision(
known_prefix: np.ndarray | None,
*,
previous_control: str | None,
current_control: str,
semantics_version: int,
) -> np.ndarray | None:
"""Apply the checkpoint-versioned RTC policy at a control boundary."""
if (
int(semantics_version) in {12, 13, 14, 15, 16, 17, 18}
and previous_control is not None
and str(current_control) != str(previous_control)
):
return None
return known_prefix
def _autocast(device: torch.device):
if device.type == "cuda":
return torch.autocast("cuda", dtype=torch.bfloat16)
return contextlib.nullcontext()
def _normalise_audio(source: Path, destination: Path) -> np.ndarray:
destination.parent.mkdir(parents=True, exist_ok=True)
command = [
"ffmpeg",
"-v",
"error",
"-y",
"-i",
str(source),
"-vn",
"-ar",
str(AUDIO_RATE),
"-ac",
"1",
"-c:a",
"pcm_f32le",
str(destination),
]
result = subprocess.run(command, capture_output=True, check=False)
if result.returncode != 0:
raise RuntimeError(
f"ffmpeg failed to normalize {source}: "
f"{result.stderr.decode('utf-8', errors='ignore')}"
)
model_waveform = load_omni_audio_array(source, AUDIO_RATE)
rendered_waveform = load_omni_audio_array(destination, AUDIO_RATE)
if not np.array_equal(model_waveform, rendered_waveform):
max_abs = (
float(np.max(np.abs(model_waveform - rendered_waveform)))
if model_waveform.shape == rendered_waveform.shape
else float("inf")
)
raise RuntimeError(
"Normalized inference WAV differs from the waveform used by the "
"HFDv2 online Qwen input path: "
f"source_shape={model_waveform.shape}, "
f"normalized_shape={rendered_waveform.shape}, max_abs={max_abs:g}"
)
return model_waveform
def resolve_prompt(raw_prompt: str, audio_path: Path, launch_config: LaunchConfig) -> str:
del audio_path, launch_config
if str(raw_prompt).strip().lower() != "auto":
return str(raw_prompt).strip()
return DEFAULT_COMMAND_PROMPT
def validate_online_prompt(prompt: str) -> None:
if str(prompt) != AUDIO_COMMAND_STREAMING_PROMPT:
raise RuntimeError(
"HFDv2 inference prefix differs from training. Use --prompt auto; "
"the spoken waveform, not an extra text command, carries intent."
)
def validate_contract(launch_config: LaunchConfig) -> None:
model = launch_config.model
data = launch_config.data
repack = data.transform.repack
root40 = getattr(model, "action_representation", "textop39") == "root40_v1"
if root40 != bool(getattr(repack, "root40_actions", False)):
raise RuntimeError("Model and dataset action representations disagree")
semantics_version = int(
getattr(launch_config.train, "full_duplex_semantics_version", 0)
)
checks = {
"model.action_chunk_size": (int(model.action_chunk_size), ACTION_CHUNK_FRAMES),
"model.action_exec_horizon": (int(model.action_exec_horizon), ACTION_EXEC_FRAMES),
"model.max_delay": (int(model.max_delay), RTC_PREFIX_FRAMES + 1),
"data.action_exec_horizon": (int(data.action_exec_horizon), ACTION_EXEC_FRAMES),
"data.rtc_prefix_frames": (int(repack.rtc_prefix_frames), RTC_PREFIX_FRAMES),
"model.action_dim": (int(model.action_dim), 40 if root40 else 39),
"model.odim": (int(model.odim), 40 if root40 else 39),
"model.view_feature_dim": (int(model.view_feature_dim), 2048),
"data.motion_fps": (int(getattr(data, "motion_fps", MOTION_FPS)), MOTION_FPS),
"data.audio_sampling_rate": (
int(getattr(data, "audio_sampling_rate", AUDIO_RATE)),
AUDIO_RATE,
),
"data.audio_sample_step_frames": (
int(getattr(data, "audio_sample_step_frames", AUDIO_STRIDE_FRAMES)),
AUDIO_STRIDE_FRAMES,
),
"data.action_sample_step_frames": (
int(
getattr(
data,
"action_sample_step_frames",
ACTION_SAMPLE_STEP_FRAMES,
)
),
ACTION_SAMPLE_STEP_FRAMES,
),
}
failed = [f"{name}={actual}, expected={expected}" for name, (actual, expected) in checks.items() if actual != expected]
if semantics_version not in {9, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24}:
failed.append(
"train.full_duplex_semantics_version="
f"{semantics_version}, expected 9, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23 or 24"
)
window = float(getattr(data, "audio_window_seconds", AUDIO_WINDOW_SECONDS))
if not math.isclose(window, AUDIO_WINDOW_SECONDS, rel_tol=0.0, abs_tol=1e-9):
failed.append(f"data.audio_window_seconds={window}, expected={AUDIO_WINDOW_SECONDS}")
if not bool(model.rtc):
failed.append("model.rtc=False, expected=True")
if not bool(repack.textop_actions):
failed.append("data.transform.repack.textop_actions=False, expected=True")
if not bool(model.use_future_progress_embedding):
failed.append("model.use_future_progress_embedding=False, expected=True")
if not bool(getattr(model, "streaming_qwen_enabled", False)):
failed.append("model.streaming_qwen_enabled=False, expected=True")
if not bool(getattr(model, "tune_vlm", False)):
failed.append("model.tune_vlm=False, expected=True")
if bool(getattr(model, "gradient_checkpointing", True)):
failed.append("model.gradient_checkpointing=True, expected=False")
if not bool(getattr(data, "online_audio", False)):
failed.append("data.online_audio=False, expected=True")
if (
bool(getattr(data, "stationary_heading_alignment", False))
!= (semantics_version in {15, 16, 17, 18, 19, 20, 21, 22, 23, 24})
):
failed.append(
"data.stationary_heading_alignment differs from semantics v15-v24"
)
if bool(getattr(model, "predict_motion_duration", False)) != (
semantics_version in {16, 17, 18, 19, 20, 21, 22, 23, 24}
):
failed.append(
"model.predict_motion_duration differs from semantics v16-v24"
)
session_voice_variants = int(
getattr(data, "session_voice_variants", 0)
)
if semantics_version in {17, 18, 19, 20, 21, 22, 23, 24}:
if session_voice_variants < 2:
failed.append(
"semantics v17-v24 requires data.session_voice_variants>=2"
)
elif session_voice_variants != 0:
failed.append(
"historical semantics require data.session_voice_variants=0"
)
interruption_sessions = bool(
getattr(data, "interruption_sessions", False)
)
if interruption_sessions != (semantics_version in {20, 21, 22, 23, 24}):
failed.append(
"data.interruption_sessions differs from semantics v20-v24"
)
state_aware_control = bool(
getattr(model, "state_aware_serial_control", False)
)
if state_aware_control != (semantics_version == 21):
failed.append(
"model.state_aware_serial_control differs from versioned policy"
)
independent_task_pools = bool(
getattr(data, "independent_task_pools", False)
)
if independent_task_pools != (semantics_version in {21, 22, 23, 24}):
failed.append(
"data.independent_task_pools differs from semantics v21-v24"
)
idle_residual_retargeting = bool(
getattr(data, "stationary_idle_residual_retargeting", False)
)
if idle_residual_retargeting != (semantics_version == 24):
failed.append(
"data.stationary_idle_residual_retargeting differs from "
"semantics v24"
)
implicit_repair_ratio = float(
getattr(data, "self_repair_implicit_ratio", 0.0)
)
if semantics_version == 24:
if not 0.0 < implicit_repair_ratio < 1.0:
failed.append(
"semantics v24 requires 0 < "
"data.self_repair_implicit_ratio < 1"
)
elif implicit_repair_ratio != 0.0:
failed.append(
"historical semantics require data.self_repair_implicit_ratio=0"
)
if failed:
raise ValueError("Full-duplex train/infer contract mismatch: " + "; ".join(failed))
def validate_checkpoint_encoder_contract(
run_dir: Path,
ckpt_step: int | str,
current_contract: dict[str, Any],
*,
expected_global_step: int | None = None,
) -> dict[str, Any]:
checkpoint_name = str(ckpt_step)
if checkpoint_name in {"best", "best_ckpt"}:
checkpoint_dir = run_dir / "checkpoints" / "best_ckpt"
elif checkpoint_name.startswith("ckpt_"):
checkpoint_dir = run_dir / "checkpoints" / checkpoint_name
else:
checkpoint_dir = run_dir / "checkpoints" / f"ckpt_{checkpoint_name}"
marker_path = checkpoint_dir / "full_duplex_source_boundary.json"
if not marker_path.is_file():
raise RuntimeError(
"This checkpoint has no HFDv2 source-boundary/contract marker"
)
marker = json.loads(marker_path.read_text(encoding="utf-8"))
try:
global_step = int(marker["global_step"])
except (KeyError, TypeError, ValueError) as error:
raise RuntimeError(
f"Checkpoint has no valid global_step marker: {marker_path}"
) from error
if global_step <= 0:
raise RuntimeError(
f"Checkpoint has invalid global_step={global_step}: {marker_path}"
)
if expected_global_step is not None and global_step != int(
expected_global_step
):
raise RuntimeError(
"Checkpoint changed while automatic inference was starting: "
f"expected global_step={expected_global_step}, observed={global_step}. "
"Retry so one output directory cannot mix two best_ckpt versions."
)
if checkpoint_name not in {"best", "best_ckpt"}:
directory_step = int(checkpoint_name.removeprefix("ckpt_"))
if global_step != directory_step:
raise RuntimeError(
"Immutable checkpoint directory/marker mismatch: "
f"directory={checkpoint_name}, marker global_step={global_step}"
)
if (
int(marker.get("full_duplex_semantics_version", 0))
not in {9, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24}
or marker.get("architecture")
!= "hfd_qwen_native_kv_serial_control"
):
raise RuntimeError(
"Refusing to run HFDv2 inference on a legacy/cached-hidden checkpoint"
)
checkpoint_contract = marker.get("train_infer_contract")
if checkpoint_contract != current_contract:
mismatch_paths = _contract_mismatch_paths(
checkpoint_contract,
current_contract,
)
mismatch_summary = ", ".join(mismatch_paths[:8]) or "unknown"
raise RuntimeError(
"Checkpoint HFDv2 contract differs from active base Qwen, "
"prompt, raw data, tail tuning, KV horizon, effective batch, "
"or RTC geometry; differing fields: "
f"{mismatch_summary}"
)
return marker
def _contract_mismatch_paths(
checkpoint_value: Any,
active_value: Any,
prefix: str = "",
) -> list[str]:
if isinstance(checkpoint_value, dict) and isinstance(active_value, dict):
paths: list[str] = []
for key in sorted(set(checkpoint_value) | set(active_value)):
child_prefix = f"{prefix}.{key}" if prefix else str(key)
if key not in checkpoint_value or key not in active_value:
paths.append(child_prefix)
continue
paths.extend(
_contract_mismatch_paths(
checkpoint_value[key],
active_value[key],
child_prefix,
)
)
return paths
if checkpoint_value != active_value:
return [prefix or "<root>"]
return []
def _build_active_hfdv2_contract(
launch_config: LaunchConfig,
*,
semantics_version: int,
backbone: Path,
continuous_audio_root: Path,
background_assignment_path: Path,
) -> dict[str, Any]:
from psi.models.relative_instruction_memory import architecture_contract
return hfdv2_train_infer_contract(
architecture_options=architecture_contract(launch_config.model),
full_duplex_semantics_version=semantics_version,
backbone=backbone,
continuous_audio_root=continuous_audio_root,
background_assignment_path=background_assignment_path,
qwen_trainable_last_layers=(
launch_config.model.streaming_qwen_trainable_last_layers
),
qwen_train_final_norm=launch_config.model.streaming_qwen_train_final_norm,
qwen_kv_gradient_horizon_windows=(
launch_config.train.qwen_kv_gradient_horizon_windows
),
dense_window_accumulation_steps=int(
getattr(
launch_config.train,
"dense_window_accumulation_steps",
1,
)
),
train_microbatch_per_gpu=launch_config.train.train_batch_size,
source_accumulation_groups=(
launch_config.train.source_accumulation_groups
),
audio_sampling_rate=launch_config.data.audio_sampling_rate,
audio_window_seconds=launch_config.data.audio_window_seconds,
audio_stride_frames=launch_config.data.audio_sample_step_frames,
action_sample_step_frames=(
launch_config.data.action_sample_step_frames
),
motion_fps=launch_config.data.motion_fps,
action_chunk_frames=launch_config.model.action_chunk_size,
action_execute_frames=launch_config.model.action_exec_horizon,
control_condition_train_mode=(
launch_config.model.control_condition_train_mode
),
control_condition_inference_mode=(
launch_config.model.control_condition_inference_mode
),
control_condition_temperature=(
launch_config.model.control_condition_temperature
),
joint_velocity_loss_weight=(
launch_config.train.joint_velocity_loss_weight
),
joint_acceleration_loss_weight=(
launch_config.train.joint_acceleration_loss_weight
),
root_continuity_loss_weight=(
launch_config.train.root_continuity_loss_weight
),
stationary_root_height_loss_weight=(
launch_config.train.stationary_root_height_loss_weight
),
session_min_turns=getattr(launch_config.data, "session_min_turns", 1),
session_max_turns=getattr(launch_config.data, "session_max_turns", 1),
session_voice_variants=getattr(
launch_config.data,
"session_voice_variants",
0,
),
interruption_sessions=getattr(
launch_config.data,
"interruption_sessions",
False,
),
interruption_ratio=getattr(
launch_config.data,
"interruption_ratio",
0.0,
),
interruption_min_start_seconds=getattr(
launch_config.data,
"interruption_min_start_seconds",
2.0,
),
interruption_max_start_seconds=getattr(
launch_config.data,
"interruption_max_start_seconds",
6.0,
),
interruption_min_remaining_seconds=getattr(
launch_config.data,
"interruption_min_remaining_seconds",
1.5,
),
initial_idle_max_steps=getattr(
launch_config.data,
"initial_idle_max_steps",
0,
),
stationary_blend_frames=getattr(
launch_config.data,
"stationary_blend_frames",
ACTION_EXEC_FRAMES,
),
turn_transition_frames=getattr(
launch_config.data,
"turn_transition_frames",
ACTION_EXEC_FRAMES,
),
causal_listen_timing=getattr(
launch_config.data,
"causal_listen_timing",
False,
),
max_speech_onset_steps=getattr(
launch_config.data,
"max_speech_onset_steps",
0,
),
listen_reaction_steps=getattr(
launch_config.data,
"listen_reaction_steps",
0,
),
motion_duration_prediction=bool(
getattr(launch_config.model, "predict_motion_duration", False)
),
motion_duration_default_seconds=float(
getattr(
launch_config.model,
"motion_duration_default_seconds",
8.0,
)
),
motion_duration_distribution_bins=int(
getattr(
launch_config.model,
"motion_duration_distribution_bins",
1,
)
),
motion_duration_min_seconds=float(
getattr(launch_config.model, "motion_duration_min_seconds", 0.5)
),
motion_duration_max_seconds=float(
getattr(launch_config.model, "motion_duration_max_seconds", 180.0)
),
motion_duration_loss_weight=float(
getattr(
launch_config.train,
"motion_duration_loss_weight",
0.0,
)
),
latest_instruction_loss_weight=float(
getattr(
launch_config.train,
"latest_instruction_loss_weight",
0.0,
)
),
latest_instruction_metadata_path=getattr(
launch_config.data,
"metadata_jsonl",
None,
),
instruction_recency_slots=int(
getattr(
launch_config.model,
"instruction_recency_slots",
0,
)
),
instruction_memory_loss_weight=float(
getattr(
launch_config.train,
"instruction_memory_loss_weight",
0.0,
)
),
max_stream_steps=int(
getattr(launch_config.data, "max_stream_steps", 1923)
),
state_aware_serial_control=bool(
getattr(launch_config.model, "state_aware_serial_control", False)
),
independent_task_pools=bool(
getattr(launch_config.data, "independent_task_pools", False)
),
task_session_variants={
"turn-taking": int(
getattr(
launch_config.data,
"turn_taking_session_variants",
0,
)
),
"interruption": int(
getattr(
launch_config.data,
"command_interruption_session_variants",
0,
)
),
"emergency-stop": int(
getattr(
launch_config.data,
"emergency_stop_session_variants",
0,
)
),
"safe-hold": int(
getattr(
launch_config.data,
"safe_hold_session_variants",
0,
)
),
"backchannel": int(
getattr(
launch_config.data,
"backchannel_session_variants",
0,
)
),
"hesitation": int(
getattr(
launch_config.data,
"hesitation_session_variants",
0,
)
),
"pause": int(
getattr(
launch_config.data,
"pause_session_variants",
0,
)
),
},
interruption_task_session_ratio=float(
getattr(
launch_config.data,
"interruption_task_session_ratio",
0.0,
)
),
control_task_session_ratio=float(
getattr(
launch_config.data,
"control_task_session_ratio",
0.0,
)
),
backchannel_task_session_ratio=float(
getattr(
launch_config.data,
"backchannel_task_session_ratio",
0.0,
)
),
hesitation_task_window_ratio=float(
getattr(
launch_config.data,
"hesitation_task_window_ratio",
0.0,
)
),
pause_task_window_ratio=float(
getattr(launch_config.data, "pause_task_window_ratio", 0.0)
),
control_followup_ratio=float(
getattr(launch_config.data, "control_followup_ratio", 0.0)
),
control_post_idle_min_steps=int(
getattr(
launch_config.data,
"control_post_idle_min_steps",
1,
)
),
control_post_idle_max_steps=int(
getattr(
launch_config.data,
"control_post_idle_max_steps",
0,
)
),
control_post_idle_sampling=str(
getattr(
launch_config.data,
"control_post_idle_sampling",
"legacy_uniform",
)
),
control_min_start_seconds=float(
getattr(
launch_config.data,
"control_min_start_seconds",
2.0,
)
),
control_max_start_seconds=float(
getattr(
launch_config.data,
"control_max_start_seconds",
6.0,
)
),
control_min_remaining_seconds=float(
getattr(
launch_config.data,
"control_min_remaining_seconds",
0.4,
)
),
control_activity_aware_timing=bool(
getattr(
launch_config.data,
"control_activity_aware_timing",
False,
)
),
backchannel_command_manifest=getattr(
launch_config.data,
"backchannel_command_manifest",
None,
),
hesitation_pause_manifest=getattr(
launch_config.data,
"hesitation_pause_manifest",
None,
),
new_task_max_rows_per_instruction_train=int(
getattr(
launch_config.data,
"new_task_max_rows_per_instruction_train",
32,
)
),
new_task_max_rows_per_instruction_val=int(
getattr(
launch_config.data,
"new_task_max_rows_per_instruction_val",
8,
)
),
hesitation_filled_ratio=float(
getattr(launch_config.data, "hesitation_filled_ratio", 0.60)
),
self_repair_implicit_ratio=float(
getattr(launch_config.data, "self_repair_implicit_ratio", 0.0)
),
backchannel_min_start_seconds=float(
getattr(
launch_config.data,
"backchannel_min_start_seconds",
1.2,
)
),
backchannel_max_start_seconds=float(
getattr(
launch_config.data,
"backchannel_max_start_seconds",
6.0,
)
),
backchannel_min_remaining_seconds=float(
getattr(
launch_config.data,
"backchannel_min_remaining_seconds",
1.2,
)
),
filter_known_tts_translation_mismatches=bool(
getattr(
launch_config.data,
"filter_known_tts_translation_mismatches",
False,
)
),
canonical_motion_manifest=getattr(
launch_config.data,
"canonical_motion_manifest",
"",
),
control_command_manifest=getattr(
launch_config.data,
"control_command_manifest",
None,
),
safe_hold_stability_window_frames=int(
getattr(
launch_config.data,
"safe_hold_stability_window_frames",
20,
)
),
safe_stationary_anchors=bool(
getattr(launch_config.data, "safe_stationary_anchors", False)
),
stationary_idle_residual_retargeting=bool(
getattr(
launch_config.data,
"stationary_idle_residual_retargeting",
False,
)
),
dynamic_task_entry_blend=bool(
getattr(launch_config.data, "dynamic_task_entry_blend", False)
),
stationary_listen_hold=bool(getattr(launch_config.data, "stationary_listen_hold", False)),
long_session_drop_ratio=float(
getattr(launch_config.data, "long_session_drop_ratio", 0.0)
),
stop_enter_loss_weight=float(
getattr(launch_config.train, "stop_enter_loss_weight", 0.0)
),
stop_stay_loss_weight=float(
getattr(launch_config.train, "stop_stay_loss_weight", 0.0)
),
stop_release_loss_weight=float(
getattr(launch_config.train, "stop_release_loss_weight", 0.0)
),
)
def _continuous_window_count(sample_count: int) -> int:
window_samples = int(round(AUDIO_WINDOW_SECONDS * AUDIO_RATE))
stride_samples = int(
round(AUDIO_STRIDE_FRAMES / float(MOTION_FPS) * AUDIO_RATE)
)
prefix_samples = window_samples - stride_samples
return int(math.ceil((prefix_samples + int(sample_count)) / stride_samples))
def _exported_window_count(sample_count: int) -> int:
window_samples = int(round(AUDIO_WINDOW_SECONDS * AUDIO_RATE))
stride_samples = int(
round(EXPORTED_AUDIO_STRIDE_FRAMES / float(MOTION_FPS) * AUDIO_RATE)
)
prefix_samples = window_samples - stride_samples
return int(math.ceil((prefix_samples + int(sample_count)) / stride_samples))
def _file_sha256(path: Path) -> str:
digest = hashlib.sha256()
with path.open("rb") as handle:
for chunk in iter(lambda: handle.read(1024 * 1024), b""):
digest.update(chunk)
return digest.hexdigest()
def _build_continuous_mixed_waveform(
*,
speech_waveform: np.ndarray,
background_waveform: np.ndarray,
listen_start_index: int,
) -> tuple[np.ndarray, float]:
"""Reproduce the training waveform mix and its two exact RTC overlaps."""
speech = np.asarray(speech_waveform, dtype=np.float32).reshape(-1)
background = np.asarray(background_waveform, dtype=np.float32).reshape(-1)
window_samples = int(round(AUDIO_WINDOW_SECONDS * AUDIO_RATE))
stride_samples = int(
round(AUDIO_STRIDE_FRAMES / float(MOTION_FPS) * AUDIO_RATE)
)
prefix_samples = window_samples - stride_samples
listen_windows = _continuous_window_count(speech.size)
mixed_samples = (listen_windows - 1) * stride_samples + window_samples
background_start = int(listen_start_index) * stride_samples
background_end = background_start + mixed_samples
if background_start < 0 or background_end > background.size:
raise RuntimeError(
"Selected continuous background cannot cover inference speech: "
f"start={listen_start_index} samples={mixed_samples} "
f"background_samples={background.size}"
)
mixed = background[background_start:background_end].copy()
speech_peak = float(np.max(np.abs(speech), initial=0.0))
peak_limit = 10.0 ** (-3.0 / 20.0)
speech_scale = (
min(1.0, peak_limit / speech_peak)
if speech_peak > 0.0
else 1.0
)
speech_start = prefix_samples
speech_end = speech_start + speech.size
mixed[speech_start:speech_end] += speech * np.float32(speech_scale)
if float(np.max(np.abs(mixed), initial=0.0)) >= 1.0:
raise RuntimeError(
"Continuous inference mix clips; the production background "
"contract requires its original quiet level"
)
# Training writes PCM16 before Qwen export. Quantize here too so online
# inference sees the same waveform domain instead of a subtly different
# pre-quantization float sum.
mixed = (
np.rint(mixed * 32768.0)
.clip(-32768, 32767)
.astype("<i2")
.astype(np.float32)
/ np.float32(32768.0)
)
# The last prefix interval must already be pure background so the next
# motion window shares the exact same PCM overlap.
next_start = (int(listen_start_index) + listen_windows) * stride_samples
if not np.array_equal(
mixed[-prefix_samples:],
background[next_start : next_start + prefix_samples],
):
raise RuntimeError("Mixed/background RTC overlap is not continuous")
return np.ascontiguousarray(mixed, dtype=np.float32), float(speech_scale)
def _build_continuous_multiturn_waveform(
*,
speech_waveforms: Sequence[np.ndarray],
speech_scales: Sequence[float],
background_waveform: np.ndarray,
listen_start_index: int,
inter_turn_background_steps: int | Sequence[int],
) -> tuple[np.ndarray, list[int], list[int]]:
"""Place several commands on one uninterrupted inference background."""
if len(speech_waveforms) < 2:
raise ValueError("Multi-turn inference requires at least two commands")
if len(speech_waveforms) != len(speech_scales):
raise ValueError("Speech waveform/scale count mismatch")
gap_count = len(speech_waveforms) - 1
if isinstance(inter_turn_background_steps, (int, np.integer)):
gap_steps = [int(inter_turn_background_steps)] * gap_count
else:
gap_steps = [int(value) for value in inter_turn_background_steps]
if len(gap_steps) != gap_count:
raise ValueError(
"Multi-turn inference needs one background gap per adjacent "
f"turn pair: turns={len(speech_waveforms)} gaps={gap_steps}"
)
if any(value < 0 for value in gap_steps):
raise ValueError(
"Multi-turn inference background gaps cannot be negative: "
f"{gap_steps}"
)
speeches = [
np.asarray(value, dtype=np.float32).reshape(-1)
for value in speech_waveforms
]
listen_counts = [_continuous_window_count(value.size) for value in speeches]
turn_window_starts: list[int] = []
cursor = 0
for turn_index, count in enumerate(listen_counts):
turn_window_starts.append(cursor)
cursor += count
if turn_index + 1 < len(listen_counts):
cursor += gap_steps[turn_index]
total_windows = cursor
window_samples = int(round(AUDIO_WINDOW_SECONDS * AUDIO_RATE))
stride_samples = int(
round(AUDIO_STRIDE_FRAMES / float(MOTION_FPS) * AUDIO_RATE)
)
prefix_samples = window_samples - stride_samples
flat_count = (total_windows - 1) * stride_samples + window_samples
background_start = int(listen_start_index) * stride_samples
background_end = background_start + flat_count
background = np.asarray(background_waveform, dtype=np.float32).reshape(-1)
if background_start < 0 or background_end > background.size:
raise RuntimeError(
"Selected background cannot cover multi-turn inference: "
f"start={listen_start_index} windows={total_windows}"
)
mixed = background[background_start:background_end].copy()
previous_end = -1
for speech, scale, window_start in zip(
speeches,
speech_scales,
turn_window_starts,
):
start = prefix_samples + window_start * stride_samples
end = start + speech.size
if start < previous_end:
raise RuntimeError(
"Two inference commands overlap; interruption allows speech "
"over motion, not simultaneous command waveforms"
)
mixed[start:end] += speech * np.float32(scale)
previous_end = end
if float(np.max(np.abs(mixed), initial=0.0)) >= 1.0:
raise RuntimeError("Multi-turn inference speech/background mix clips")
mixed = (
np.rint(mixed * 32768.0)
.clip(-32768, 32767)
.astype("<i2")
.astype(np.float32)
/ np.float32(32768.0)
)
return (
np.ascontiguousarray(mixed, dtype=np.float32),
turn_window_starts,
listen_counts,
)
def _read_background_sessions(
continuous_audio_root: Path,
) -> dict[str, dict[str, Any]]:
manifest = (
continuous_audio_root
/ "background"
/ "sessions"
/ "session_manifest.json"
)
payload = json.loads(manifest.read_text(encoding="utf-8"))
if payload.get("format") != "hfd_continuous_background_sessions_v1":
raise RuntimeError(f"Unsupported background session manifest: {manifest}")
result = {
str(row["key"]): dict(row)
for row in payload.get("sessions", [])
}
if not result:
raise RuntimeError(f"No continuous background sessions in {manifest}")
return result
def _read_training_background_assignment(
assignment_path: Path,
audio_path: Path,
) -> dict[str, Any] | None:
# The production manifest contains tens of thousands of rows on CPFS.
# Calling Path.resolve() for every row performs a stat() on every path
# component and turns a missing lookup (for example a newly generated Stop
# command) into minutes of metadata I/O. Manifest paths are already
# absolute; compare lexical, normalized keys and resolve only the one query
# path so symlinked caller paths keep working.
def path_key(value: str | os.PathLike[str]) -> str:
relocated = relocate_hfd_storage_path(value)
return os.path.normcase(
os.path.abspath(
os.path.normpath(os.fspath(relocated))
)
)
expanded = audio_path.expanduser()
wanted = {path_key(expanded), path_key(expanded.resolve())}
with assignment_path.open("r", encoding="utf-8") as handle:
for line in handle:
if not line.strip():
continue
row = json.loads(line)
if path_key(str(row["original_audio_path"])) in wanted:
return row
return None
def _choose_continuous_background(
*,
sessions: dict[str, dict[str, Any]],
speech_samples: int,
motion_frames: int,
initial_idle_steps: int,
max_listen_tail_steps: int,
seed_key: str,
required_session_windows: int = 0,
terminal_background_steps: int = 0,
) -> tuple[str, int]:
listen_windows = (
int(required_session_windows)
if int(required_session_windows) > 0
else _continuous_window_count(speech_samples)
)
motion_windows = int(
math.ceil(max(int(motion_frames), 1) / float(ACTION_EXEC_FRAMES))
)
required_after = listen_windows + motion_windows + max(
int(max_listen_tail_steps), 1
) + max(int(terminal_background_steps), 0)
choices: list[tuple[str, int, int]] = []
total = 0
for key, row in sorted(sessions.items()):
first = max(int(initial_idle_steps), 0)
sample_count = int(row["sample_count"])
window_samples = int(round(AUDIO_WINDOW_SECONDS * AUDIO_RATE))
stride_samples = int(
round(AUDIO_STRIDE_FRAMES / float(MOTION_FPS) * AUDIO_RATE)
)
runtime_windows = (
0
if sample_count < window_samples
else (sample_count - window_samples) // stride_samples + 1
)
slots = runtime_windows - required_after - first + 1
if slots <= 0:
continue
choices.append((key, first, slots))
total += slots
if total <= 0:
raise RuntimeError(
"No one-piece background session is long enough for inference"
)
digest = hashlib.sha256(seed_key.encode("utf-8")).digest()
slot = int.from_bytes(digest[:8], "big") % total
for key, first, count in choices:
if slot < count:
return key, first + slot
slot -= count
raise AssertionError("Continuous background selection fell through")
def _control_record(condition: FullDuplexStreamCondition) -> dict[str, Any]:
probabilities = condition.control_probabilities.float()[0]
if int(probabilities.numel()) != len(CONTROL_TOKENS):
raise RuntimeError(
"Full-duplex control head must predict listen/motion/idle: "
f"logits={tuple(condition.control_logits.shape)}"
)
token_id = int(probabilities.argmax().item())
record = {
"predicted_id": token_id,
"predicted_token": CONTROL_TOKENS[token_id],
"listen_probability": float(probabilities[0].item()),
"motion_probability": float(probabilities[1].item()),
"idle_probability": float(probabilities[2].item()),
"condition_weights": [
float(value)
for value in condition.control_condition_weights.float()[0].tolist()
],
"condition_mode": str(condition.control_condition_mode),
"control_entropy": float(condition.control_entropy.float()[0].item()),
"cache_tokens": int(condition.kv_cache.sequence_length),
}
duration_prediction = getattr(
condition,
"motion_duration_log1p_seconds",
None,
)
if duration_prediction is not None:
log_duration = float(
duration_prediction.float()[0].item()
)
record["predicted_motion_duration_seconds"] = float(
math.expm1(min(max(log_duration, 0.0), math.log1p(3600.0)))
)
return record
def _motion_duration_frames_from_condition(
condition: FullDuplexStreamCondition,
*,
motion_fps: int,
max_frames: int,
) -> int | None:
"""Decode one causal duration prediction into raw primitive frames."""
prediction = getattr(condition, "motion_duration_log1p_seconds", None)
if prediction is None:
return None
fps = max(int(motion_fps), 1)
upper = max(int(max_frames), 1)
log_duration = float(prediction.float()[0].item())
seconds = math.expm1(
min(max(log_duration, 0.0), math.log1p(upper / float(fps)))
)
return min(max(int(round(seconds * fps)), 1), upper)
def _future_progress(
frame_id: int,
emitted_motion_frames: int,
transition_frames: int,
*,
inplace: bool = False,
) -> np.ndarray:
"""Match training's transition-plus-original-motion progress exactly.
Training emits ``transition_frames`` from the stationary anchor to the
original task's first pose, then appends original frames ``1..N-1``. Thus
an emitted sequence of length ``M`` represents ``M-(transition-1)``
original task frames. The complete transition is progress zero.
"""
if inplace:
from psi.runtime.swa_rollout import inplace_future_progress
return inplace_future_progress(frame_id, emitted_motion_frames, ACTION_CHUNK_FRAMES)
emitted = int(emitted_motion_frames)
transition = int(transition_frames)
if transition < 2:
raise ValueError(f"transition_frames must be at least 2, got {transition}")
original_total = emitted - (transition - 1)
if original_total <= 0:
raise ValueError(
"Motion budget must include the stationary transition and at least "
f"one original task frame: emitted={emitted}, transition={transition}"
)
denominator = float(max(original_total - 1, 1))
rows = []
# ``frame_id`` is the number of already executed action frames, hence it
# indexes the current observed state. The action token at position zero
# is the *next* state (t+1), exactly as in the v26 dataset. RTC's ten
# known positions are still part of this complete 30-frame timeline; they
# must not shift the progress labels backwards by ten frames.
for frame in range(frame_id + 1, frame_id + ACTION_CHUNK_FRAMES + 1):
original_frame = max(frame - (transition - 1), 0)
clipped = min(original_frame, original_total - 1)
progress = float(clipped) / denominator
rows.append(
[
progress,
max(0.0, 1.0 - progress),
progress,
math.sin(2.0 * math.pi * progress),
math.cos(2.0 * math.pi * progress),
]
)
return np.asarray(rows, dtype=np.float32)
def assemble_stream_audio(
*,
speech_waveform: np.ndarray,
white_noise_waveforms: Sequence[np.ndarray],
control_events: list[dict[str, Any]],
total_motion_frames: int,
) -> np.ndarray:
"""Overlap-average the exact 0.6 s windows consumed by streaming inference."""
total_samples = int(
round(int(total_motion_frames) / float(MOTION_FPS) * AUDIO_RATE)
)
if total_samples <= 0:
raise ValueError(
f"total_motion_frames must be positive, got {total_motion_frames}"
)
window_samples = int(round(AUDIO_WINDOW_SECONDS * AUDIO_RATE))
mixed = np.zeros(total_samples, dtype=np.float64)
weights = np.zeros(total_samples, dtype=np.float64)
speech = np.asarray(speech_waveform, dtype=np.float32).reshape(-1)
if not white_noise_waveforms:
raise ValueError("At least one white-noise waveform is required")
for event in control_events:
input_kind = str(event["input_kind"])
if input_kind == "wav_audio":
source_start = int(event["input_start_sample"])
audio_window = speech[source_start : source_start + window_samples]
elif input_kind in {
"initial_background",
"post_wav_background",
"terminal_background",
}:
noise_index = int(event["white_noise_index"])
audio_window = np.asarray(
white_noise_waveforms[noise_index % len(white_noise_waveforms)],
dtype=np.float32,
).reshape(-1)
else:
raise ValueError(f"Unsupported streaming input kind: {input_kind!r}")
if audio_window.shape != (window_samples,):
raise RuntimeError(
"Streaming audio event does not contain one complete window: "
f"kind={input_kind} shape={audio_window.shape}"
)
output_frame = int(event["output_frame"])
if output_frame < 0:
raise ValueError(f"Negative output_frame in audio event: {event}")
destination_start = int(round(output_frame / float(MOTION_FPS) * AUDIO_RATE))
destination_end = min(destination_start + window_samples, total_samples)
if destination_end <= destination_start:
continue
take = destination_end - destination_start
mixed[destination_start:destination_end] += audio_window[:take]
weights[destination_start:destination_end] += 1.0
valid = weights > 0
if not np.all(valid):
missing = np.flatnonzero(~valid)
raise RuntimeError(
"Streaming control events do not cover the complete rendered audio "
f"timeline: missing_samples={missing.size}, first={int(missing[0])}"
)
mixed[valid] /= weights[valid]
return np.clip(mixed, -1.0, 1.0).astype(np.float32)
def write_pcm16_wav(path: Path, waveform: np.ndarray) -> None:
values = np.asarray(waveform, dtype=np.float32).reshape(-1)
pcm = np.round(np.clip(values, -1.0, 1.0) * 32767.0).astype("<i2")
path.parent.mkdir(parents=True, exist_ok=True)
temporary = path.with_name(f".{path.name}.tmp")
with wave.open(str(temporary), "wb") as handle:
handle.setnchannels(1)
handle.setsampwidth(2)
handle.setframerate(AUDIO_RATE)
handle.writeframes(pcm.tobytes())
temporary.replace(path)
def rollout_full_duplex(
*,
model: FullDuplexPsi0Model,
background_waveforms: Sequence[np.ndarray],
waveform: np.ndarray,
prompt: str,
source_id: str,
init_abs: np.ndarray,
launch_config: LaunchConfig,
motion_frames: int,
num_inference_steps: int,
respect_control_token: bool,
max_listen_tail_steps: int,
initial_idle_steps: int,
seed: int,
white_noise_start_index: int = 0,
continuous_background_listen_start_index: int | None = None,
fixed_post_wav_background_steps: int = 0,
terminal_background_steps: int = 0,
) -> tuple[np.ndarray, dict[str, Any]]:
if motion_frames <= 0:
raise ValueError("motion_frames must be positive")
device = model.device
starts = complete_audio_clip_starts(len(waveform))
if not starts:
raise ValueError(
f"Audio must contain at least one complete 0.6 s clip; got {len(waveform)} samples"
)
initial_idle_steps = max(int(initial_idle_steps), 0)
fixed_post_wav_background_steps = max(
int(fixed_post_wav_background_steps),
0,
)
terminal_background_steps = max(int(terminal_background_steps), 0)
rollout_semantics_version = int(
getattr(
getattr(launch_config, "train", None),
"full_duplex_semantics_version",
11,
)
)
inplace_motion = bool(getattr(launch_config.data, "dynamic_task_entry_blend", False))
live_overlay = bool(getattr(launch_config.data, "multimodal_overlay_manifests", ()))
first_transition_frames = int(
getattr(
launch_config.data,
"turn_transition_frames"
if rollout_semantics_version in {14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24}
else "stationary_blend_frames",
ACTION_EXEC_FRAMES,
)
)
later_transition_frames = (
int(
getattr(
launch_config.data,
"turn_transition_frames",
first_transition_frames,
)
)
if rollout_semantics_version in {13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24}
else first_transition_frames
)
if not inplace_motion and int(motion_frames) <= max(
first_transition_frames,
later_transition_frames,
) - 1:
raise ValueError(
"motion_frames must include the complete stationary-to-task "
"transition: "
f"motion_frames={motion_frames}, first={first_transition_frames}, "
f"later={later_transition_frames}"
)
cache = model.begin_source([source_id])
control_events: list[dict[str, Any]] = []
window_samples = int(round(AUDIO_WINDOW_SECONDS * AUDIO_RATE))
maxmin = launch_config.data.transform.field
current_abs = np.asarray(init_abs, dtype=np.float32).copy()
# The first source window has no previous executed action. It predicts a
# complete 30-frame chunk; only subsequent windows use the previous
# chunk's final 10 frames as the fixed RTC prefix.
known_prefix_np: np.ndarray | None = None
last_action_norm = None
last_action_origin = None
previous_executed_control: str | None = None
timeline_generated: list[np.ndarray] = []
timeline_frame_count = 0
post_wav_motion_frames = 0
post_wav_background_steps = 0
task_motion_progress = 0
motion_turn_index = -1
active_transition_frames = first_transition_frames
active_motion_frames = int(motion_frames)
used_transition_frames: list[int] = []
predicted_motion_raw_frames_by_turn: list[int] = []
initial_input_motion_frames = 0
in_wav_motion_frames = 0
if not background_waveforms:
raise ValueError("At least one raw background window is required")
continuous_background = (
continuous_background_listen_start_index is not None
)
if continuous_background:
white_queries = (
int(continuous_background_listen_start_index) - initial_idle_steps
)
if white_queries < 0:
raise RuntimeError(
"Continuous background has no room for requested initial idle"
)
else:
white_queries = int(white_noise_start_index) % len(background_waveforms)
initial_white_noise_index = int(white_queries)
listen_tail_steps = 0
first_any_motion_frame: int | None = None
post_wav_motion_start_frame: int | None = None
stream_cache_updates = 0
max_cache_tokens = 0
generator = torch.Generator(device=device)
generator.manual_seed(int(seed))
def background_window(index: int) -> tuple[np.ndarray, int]:
resolved = int(index)
if continuous_background:
if resolved < 0 or resolved >= len(background_waveforms):
raise RuntimeError(
"Continuous background window is outside its source: "
f"index={resolved} count={len(background_waveforms)}"
)
else:
resolved %= len(background_waveforms)
return np.asarray(
background_waveforms[resolved],
dtype=np.float32,
), resolved
def encode_raw_window(audio_window: np.ndarray) -> FullDuplexStreamCondition:
state_aware_control = bool(
getattr(
launch_config.model,
"state_aware_serial_control",
False,
)
)
control_context_kwargs: dict[str, torch.Tensor] = {}
if state_aware_control:
states_np = build_model_states_from_abs(
[current_abs],
textop_actions=True,
odim=int(launch_config.model.odim),
)
if bool(maxmin.normalize_state):
states_np = maxmin.normalize_state_func(states_np)
states = torch.from_numpy(states_np).unsqueeze(0).to(
device=device
)
rtc_prefix_np = (
np.zeros(
(
RTC_PREFIX_FRAMES,
int(
getattr(
launch_config.model,
"action_dim",
model.action_dim,
)
),
),
dtype=np.float32,
)
if known_prefix_np is None
else np.asarray(known_prefix_np, dtype=np.float32)
)
rtc_mask_np = np.full(
RTC_PREFIX_FRAMES,
known_prefix_np is not None,
dtype=np.bool_,
)
previous_ids = {
None: 3,
"listen": 0,
"motion": 1,
"idle": 2,
}
if previous_executed_control == "motion":
original_total = max(
int(active_motion_frames)
- (int(active_transition_frames) - 1),
1,
)
original_frame = max(
int(task_motion_progress)
- (int(active_transition_frames) - 1),
0,
)
progress_value = float(
min(original_frame, original_total - 1)
) / float(max(original_total - 1, 1))
control_progress_np = np.asarray(
[
progress_value,
max(0.0, 1.0 - progress_value),
progress_value,
math.sin(2.0 * math.pi * progress_value),
math.cos(2.0 * math.pi * progress_value),
],
dtype=np.float32,
)
else:
control_progress_np = np.asarray(
[0.0, 1.0, 0.0, 0.0, 1.0],
dtype=np.float32,
)
control_context_kwargs = {
"states": states,
"control_progress": torch.from_numpy(control_progress_np)
.unsqueeze(0)
.to(device=device),
"previous_control_ids": torch.tensor(
[previous_ids[previous_executed_control]],
device=device,
dtype=torch.long,
),
"control_rtc_prefix": torch.from_numpy(rtc_prefix_np)
.unsqueeze(0)
.to(device=device),
"control_rtc_mask": torch.from_numpy(rtc_mask_np)
.unsqueeze(0)
.to(device=device),
}
prepared = prepare_qwen_streaming_audio_window(
model.vlm_processor,
np.asarray(audio_window, dtype=np.float32),
first_window=stream_cache_updates == 0,
sampling_rate=AUDIO_RATE,
)
if live_overlay:
# Absence of a camera/question is a trained overlay mode, not a
# switch back to the legacy audio prompt/token packing.
from psi.data.seed.live_multimodal_overlay import live_audio_packet
model.swa_packets = [live_audio_packet(
prepared["stream_input_features"],
prepared["stream_feature_attention_mask"],
stream_cache_updates,
)]
with torch.inference_mode(), _autocast(device):
return model.encode_stream_input(
source_ids=[source_id],
kv_cache=cache,
update_semantic_cache=True,
vlm_hidden_states=None,
attention_mask=prepared["stream_attention_mask"]
.unsqueeze(0)
.to(device=device),
stream_input_ids=prepared["stream_input_ids"]
.unsqueeze(0)
.to(device=device),
stream_input_features=prepared["stream_input_features"]
.unsqueeze(0)
.to(device=device),
stream_feature_attention_mask=prepared[
"stream_feature_attention_mask"
]
.unsqueeze(0)
.to(device=device),
**control_context_kwargs,
)
def accept_stream_condition(
condition: FullDuplexStreamCondition,
) -> FullDuplexStreamCondition:
nonlocal cache, stream_cache_updates
stream_cache_updates += 1
expected = _expected_stream_cache_tokens(
stream_cache_updates,
max_cache_tokens,
)
actual = int(condition.kv_cache.sequence_length)
if actual != expected:
raise RuntimeError(
"Continuous source cache length mismatch: "
f"updates={stream_cache_updates} max={max_cache_tokens} "
f"actual={actual} expected={expected}"
)
cache = condition.kv_cache
return condition
def append_timeline(chunk_abs: np.ndarray) -> None:
nonlocal timeline_frame_count
chunk = np.asarray(chunk_abs, dtype=np.float32)
if chunk.ndim != 2 or chunk.shape[1] != 36:
raise ValueError(f"Invalid generated motion chunk: {chunk.shape}")
if not len(chunk):
return
timeline_generated.append(chunk)
timeline_frame_count += int(chunk.shape[0])
def generate_action(
condition: FullDuplexStreamCondition,
*,
task_frame: int | None,
) -> np.ndarray:
nonlocal known_prefix_np, last_action_norm, last_action_origin
states_np = build_model_states_from_abs(
[current_abs],
textop_actions=True,
odim=int(launch_config.model.odim),
)
if bool(maxmin.normalize_state):
states_np = maxmin.normalize_state_func(states_np)
states = torch.from_numpy(states_np).unsqueeze(0).to(device=device)
known_prefix = (
None
if known_prefix_np is None
else torch.from_numpy(known_prefix_np).unsqueeze(0).to(device=device)
)
progress_np = (
_stationary_future_progress(int(motion_frames))
if task_frame is None
else _future_progress(
int(task_frame),
int(active_motion_frames),
active_transition_frames,
inplace=inplace_motion,
)
)
future_progress = torch.from_numpy(progress_np).unsqueeze(0).to(
device=device
)
with torch.inference_mode():
predicted_norm = _flow_action(
model,
condition,
known_prefix=known_prefix,
states=states,
future_progress=future_progress,
num_inference_steps=num_inference_steps,
generator=generator,
)[0].cpu().numpy()
predicted_norm = np.clip(
predicted_norm.astype(np.float32),
-1.0,
1.0,
)
if known_prefix_np is not None and predicted_norm.shape[-1] == 40:
predicted_norm[:RTC_PREFIX_FRAMES] = known_prefix_np
chunk_abs = decode_action_chunk(predicted_norm, current_abs, maxmin)
last_action_norm, last_action_origin = predicted_norm.copy(), current_abs.copy()
known_prefix_np = next_rtc_prefix(predicted_norm, current_abs, chunk_abs[ACTION_EXEC_FRAMES-1], maxmin)
return chunk_abs
def execute_prediction(
condition: FullDuplexStreamCondition,
*,
predicted_id: int,
motion_budget: int | None = None,
) -> tuple[str, int]:
"""Execute this window's predicted state and close RTC on that output."""
nonlocal first_any_motion_frame, task_motion_progress, current_abs
nonlocal motion_turn_index, active_transition_frames
nonlocal active_motion_frames
nonlocal known_prefix_np, previous_executed_control
executed = _control_decision(
predicted_id=int(predicted_id),
respect_control_token=respect_control_token,
)
active_train_config = getattr(launch_config, "train", None)
known_prefix_np = _rtc_prefix_after_control_decision(
known_prefix_np,
previous_control=previous_executed_control,
current_control=executed,
semantics_version=int(
getattr(
active_train_config,
"full_duplex_semantics_version",
11,
)
),
)
entering_new_motion = (
executed == "motion"
and previous_executed_control != "motion"
)
if entering_new_motion:
# Every listen -> motion decision begins a new primitive task in
# v13. Its future-progress clock must restart instead of carrying
# the previous turn's completed value into the next command.
task_motion_progress = 0
motion_turn_index += 1
active_transition_frames = (
first_transition_frames
if motion_turn_index == 0
else later_transition_frames
)
used_transition_frames.append(active_transition_frames)
predicted_raw_frames = _motion_duration_frames_from_condition(
condition,
motion_fps=MOTION_FPS,
max_frames=(
max(
int(getattr(launch_config.data, "max_stream_steps", 1923)),
1,
)
* ACTION_EXEC_FRAMES
),
)
if predicted_raw_frames is None:
active_motion_frames = int(motion_frames)
else:
# Training keeps the known stationary join at phase zero and
# measures progress over the raw primitive. Rebuild that exact
# emitted budget and freeze it for this complete motion turn.
active_motion_frames = (
int(predicted_raw_frames)
+ (0 if inplace_motion else active_transition_frames - 1)
)
predicted_motion_raw_frames_by_turn.append(
int(predicted_raw_frames)
)
previous_executed_control = executed
output_frame = timeline_frame_count
task_frame = task_motion_progress if executed == "motion" else None
chunk_abs = generate_action(condition, task_frame=task_frame)
# RTC always advances by the deployment horizon. The first model
# call returns a full 30-frame chunk so its trailing 10 frames can be
# reused as the next window's known prefix; they must not be emitted
# twice here.
take = min(int(chunk_abs.shape[0]), ACTION_EXEC_FRAMES)
if executed == "motion" and motion_budget is not None:
take = min(take, max(int(motion_budget), 0))
if take <= 0:
return executed, 0
append_timeline(chunk_abs[:take])
# The next window observes the last frame that was actually executed,
# never the unexecuted RTC tail. The latter is retained separately in
# ``known_prefix_np``. This also handles the terminal partial chunk.
current_abs = chunk_abs[take - 1].copy()
if int(model.action_dim) == 40 and take != ACTION_EXEC_FRAMES:
known_prefix_np = next_rtc_prefix(last_action_norm, last_action_origin, current_abs,
maxmin, executed_frames=take)
if executed == "motion":
if first_any_motion_frame is None:
first_any_motion_frame = output_frame
task_motion_progress += take
return executed, take
# Before speech, stream synthetic background through the same cache and let
# the model decide the output state; input kind never hard-codes execution.
for idle_index in range(initial_idle_steps):
audio_window, background_index = background_window(white_queries)
condition = encode_raw_window(audio_window)
condition = accept_stream_condition(condition)
output_frame = timeline_frame_count
control = _control_record(condition)
executed, executed_frames = execute_prediction(
condition,
predicted_id=int(control["predicted_id"]),
)
if executed == "motion":
initial_input_motion_frames += executed_frames
event = {
"input_kind": "initial_background",
"idle_index": idle_index,
"white_noise_index": background_index,
"output_frame": output_frame,
"executed": executed,
**control,
}
control_events.append(event)
white_queries += 1
# Speech windows continue updating the same source cache. The control head
# is authoritative on every window instead of being overwritten as listen.
for clip_index, start_sample in enumerate(starts):
condition = encode_raw_window(
waveform[start_sample : start_sample + window_samples]
)
condition = accept_stream_condition(condition)
output_frame = timeline_frame_count
control = _control_record(condition)
executed, executed_frames = execute_prediction(
condition,
predicted_id=int(control["predicted_id"]),
)
if executed == "motion":
in_wav_motion_frames += executed_frames
control_events.append(
{
"input_kind": "wav_audio",
"clip_index": clip_index,
"input_start_sample": start_sample,
"input_end_sample": start_sample + window_samples,
"input_start_seconds": start_sample / float(AUDIO_RATE),
"input_end_seconds": (start_sample + window_samples)
/ float(AUDIO_RATE),
"output_frame": output_frame,
"executed": executed,
**control,
}
)
speech_cache_tokens = int(cache.sequence_length)
speech_qwen_cache_tokens = int(getattr(cache, "token_count", 0))
# Do not insert an unobserved pose hold here. Training advances once per
# 0.4 s execution stride, so an extra non-modelled tail is a direct
# train/infer mismatch and a visible source of stop-start motion.
speech_tail_frames = 0
if continuous_background:
white_queries = (
int(continuous_background_listen_start_index) + len(starts)
)
def post_wav_complete() -> bool:
if fixed_post_wav_background_steps > 0:
return post_wav_background_steps >= fixed_post_wav_background_steps
return post_wav_motion_frames >= int(motion_frames)
while not post_wav_complete():
audio_window, background_index = background_window(white_queries)
# Background audio is still a real observation and is appended to the
# exact same native Qwen KV as idle and speech windows.
condition = encode_raw_window(audio_window)
condition = accept_stream_condition(condition)
event = {
"input_kind": "post_wav_background",
"white_noise_index": background_index,
"output_frame": timeline_frame_count,
**_control_record(condition),
}
white_queries += 1
post_wav_background_steps += 1
executed, executed_frames = execute_prediction(
condition,
predicted_id=int(event["predicted_id"]),
motion_budget=(
None
if fixed_post_wav_background_steps > 0
else int(motion_frames) - post_wav_motion_frames
),
)
if executed != "motion":
listen_tail_steps += 1
event["executed"] = executed
control_events.append(event)
if (
fixed_post_wav_background_steps <= 0
and listen_tail_steps >= max(int(max_listen_tail_steps), 1)
):
post_wav_events = [
item
for item in control_events
if item["input_kind"] == "post_wav_background"
]
token_counts = {
token: sum(
item["predicted_token"] == token
for item in post_wav_events
)
for token in CONTROL_TOKENS
}
last_event = post_wav_events[-1]
message = (
"The model did not produce enough </motion> windows before "
f"the post-WAV non-motion limit ({max_listen_tail_steps}). "
f"Post-WAV predictions: {token_counts}; last probabilities: "
f"listen={last_event['listen_probability']:.6f}, "
f"motion={last_event['motion_probability']:.6f}, "
f"idle={last_event['idle_probability']:.6f}. "
"Inference will not falsify its control token by forcing motion."
)
raise FullDuplexControlTimeout(
message,
diagnostics={
"error_type": "full_duplex_control_timeout",
"message": message,
"source_id": source_id,
"audio_samples": int(len(waveform)),
"audio_duration_seconds": len(waveform)
/ float(AUDIO_RATE),
"speech_clip_count": len(starts),
"initial_idle_steps": int(initial_idle_steps),
"listen_tail_steps": int(listen_tail_steps),
"post_wav_prediction_counts": token_counts,
"stream_cache_updates": int(stream_cache_updates),
"final_cache_tokens": int(cache.sequence_length),
"streaming_max_cache_tokens": int(max_cache_tokens),
"generated_frames_before_failure": int(
timeline_frame_count
),
"control_events": control_events,
},
)
continue
if post_wav_motion_start_frame is None:
post_wav_motion_start_frame = int(event["output_frame"])
event["executed"] = "motion"
control_events.append(event)
post_wav_motion_frames += executed_frames
# A multi-turn deployment must not end on the final motion frame. Continue
# streaming the same physical background, state, RTC history and native
# Qwen KV so the video exposes whether the model naturally returns to idle.
# These windows are never forced to idle: every action and token remains the
# model's own closed-loop prediction.
terminal_events: list[dict[str, Any]] = []
for terminal_index in range(terminal_background_steps):
audio_window, background_index = background_window(white_queries)
condition = encode_raw_window(audio_window)
condition = accept_stream_condition(condition)
event = {
"input_kind": "terminal_background",
"terminal_index": terminal_index,
"white_noise_index": background_index,
"output_frame": timeline_frame_count,
**_control_record(condition),
}
white_queries += 1
executed, _ = execute_prediction(
condition,
predicted_id=int(event["predicted_id"]),
)
event["executed"] = executed
control_events.append(event)
terminal_events.append(event)
complete_motion = np.concatenate(timeline_generated, axis=0)
if (
fixed_post_wav_background_steps <= 0
and post_wav_motion_start_frame is None
):
raise RuntimeError("Full-duplex rollout completed without a motion window")
post_wav_prediction_counts = {
token: sum(
event["predicted_token"] == token
for event in control_events
if event["input_kind"] == "post_wav_background"
)
for token in CONTROL_TOKENS
}
terminal_prediction_counts = {
token: sum(
event["predicted_token"] == token
for event in terminal_events
)
for token in CONTROL_TOKENS
}
terminal_idle_streak = 0
for event in reversed(terminal_events):
if event["executed"] != "idle":
break
terminal_idle_streak += 1
metadata = {
"source_id": source_id,
"audio_samples": int(len(waveform)),
"audio_duration_seconds": len(waveform) / float(AUDIO_RATE),
"white_noise_start_index": initial_white_noise_index,
"continuous_background": bool(continuous_background),
"background_listen_start_index": (
int(continuous_background_listen_start_index)
if continuous_background
else None
),
"initial_idle_steps": int(initial_idle_steps),
"initial_idle_frames": int(initial_idle_steps * ACTION_EXEC_FRAMES),
"initial_idle_seconds": initial_idle_steps
* ACTION_EXEC_FRAMES
/ float(MOTION_FPS),
"speech_start_frame_50fps": int(
initial_idle_steps * ACTION_EXEC_FRAMES
),
"speech_start_seconds": initial_idle_steps
* ACTION_EXEC_FRAMES
/ float(MOTION_FPS),
"speech_clip_count": len(starts),
"speech_clip_start_samples": starts,
"speech_cache_tokens": speech_cache_tokens,
"final_cache_tokens": int(cache.sequence_length),
"speech_qwen_cache_tokens": speech_qwen_cache_tokens,
"final_qwen_cache_tokens": int(getattr(cache, "token_count", 0)),
"stream_cache_updates": int(stream_cache_updates),
"streaming_max_cache_tokens": int(max_cache_tokens),
"speech_tail_hold_frames": int(speech_tail_frames),
"listen_tail_steps": int(listen_tail_steps),
"fixed_post_wav_background_steps": int(
fixed_post_wav_background_steps
),
"post_wav_background_steps": int(post_wav_background_steps),
"post_wav_prediction_counts": post_wav_prediction_counts,
"terminal_background_steps": int(terminal_background_steps),
"terminal_prediction_counts": terminal_prediction_counts,
"terminal_idle_streak": int(terminal_idle_streak),
"terminal_idle_ratio": (
terminal_prediction_counts[IDLE_TOKEN]
/ float(terminal_background_steps)
if terminal_background_steps > 0
else None
),
"forced_motion_transition": False,
"forced_motion_transition_count": 0,
"initial_input_motion_frames": int(initial_input_motion_frames),
"in_wav_motion_frames": int(in_wav_motion_frames),
"first_any_motion_frame_50fps": (
int(first_any_motion_frame)
if first_any_motion_frame is not None
else None
),
"post_wav_motion_start_frame_50fps": (
int(post_wav_motion_start_frame)
if post_wav_motion_start_frame is not None
else None
),
# Backward-compatible keys retain their historical post-WAV meaning.
"motion_start_frame_50fps": (
int(post_wav_motion_start_frame)
if post_wav_motion_start_frame is not None
else None
),
"motion_start_seconds": (
post_wav_motion_start_frame / float(MOTION_FPS)
if post_wav_motion_start_frame is not None
else None
),
"pretask_motion_frames": (
int(post_wav_motion_start_frame)
if post_wav_motion_start_frame is not None
else None
),
"motion_frames": int(post_wav_motion_frames),
"task_motion_progress_frames": int(task_motion_progress),
"task_transition_frames": first_transition_frames,
"task_transition_frames_by_turn": used_transition_frames,
"predicted_motion_raw_frames_by_turn": (
predicted_motion_raw_frames_by_turn
),
"predicted_motion_seconds_by_turn": [
frames / float(MOTION_FPS)
for frames in predicted_motion_raw_frames_by_turn
],
"future_progress_original_frames": int(
active_motion_frames - (0 if inplace_motion else active_transition_frames - 1)
),
"future_progress_geometry": "inplace" if inplace_motion else "prefix",
"semantic_input_format": "live_av_audio_only" if live_overlay else "legacy_audio",
"total_frames": int(complete_motion.shape[0]),
"control_events": control_events,
}
return complete_motion, metadata
def export_motion(output_dir: Path, motion: np.ndarray) -> None:
output_dir.mkdir(parents=True, exist_ok=True)
joint_pos = motion[:, :29]
body_pos = motion[:, 29:32]
body_quat = normalize_quaternion_rows(motion[:, 32:36])
dt = 1.0 / float(TARGET_FPS)
write_csv(output_dir / "joint_pos.csv", joint_pos, [f"joint_{index}" for index in range(29)])
write_csv(output_dir / "joint_vel.csv", finite_difference(joint_pos, dt), [f"joint_vel_{index}" for index in range(29)])
write_csv(output_dir / "body_pos.csv", body_pos, ["body_0_x", "body_0_y", "body_0_z"])
write_csv(output_dir / "body_quat.csv", body_quat, ["body_0_w", "body_0_x", "body_0_y", "body_0_z"])
write_csv(output_dir / "body_lin_vel.csv", finite_difference(body_pos, dt), ["body_0_vel_x", "body_0_vel_y", "body_0_vel_z"])
write_csv(output_dir / "body_ang_vel.csv", quaternion_angular_velocity(body_quat, dt), ["body_0_angvel_x", "body_0_angvel_y", "body_0_angvel_z"])
np.save(output_dir / "motion.npy", motion.astype(np.float32))
write_metadata(output_dir / "metadata.txt", output_dir.name, int(motion.shape[0]))
def parse_args() -> argparse.Namespace:
parser = argparse.ArgumentParser(
description="HFD source-scoped full-duplex streaming audio inference"
)
parser.add_argument("--run-dir", required=True)
parser.add_argument("--ckpt-step", default="latest")
parser.add_argument("--output-dir", required=True)
parser.add_argument("--audio-path", required=True)
parser.add_argument(
"--additional-audio-path",
action="append",
default=[],
help=(
"Additional command WAVs in the same v13 KV session. The cache, "
"background, robot state and RTC history are not reset."
),
)
parser.add_argument("--init-motion-dir", required=True)
parser.add_argument("--motion-frames", type=int, default=300)
parser.add_argument(
"--inter-turn-background-steps",
type=int,
default=0,
help=(
"Continuous 0.4 s background windows between command WAVs. Zero "
"covers the prior task plus its longest aligned settling join."
),
)
parser.add_argument(
"--inter-turn-background-steps-by-turn",
default="",
help=(
"Optional comma-separated background-window counts for adjacent "
"turn pairs. This makes a deterministic multi-turn evaluation "
"exercise different waiting durations without resetting KV."
),
)
parser.add_argument(
"--terminal-background-steps",
type=int,
default=0,
help=(
"After the final requested motion budget, continue this many "
"0.4 s background windows without forcing a control token."
),
)
parser.add_argument("--num-inference-steps", type=int, default=10)
parser.add_argument("--prompt", default="auto")
parser.add_argument("--backbone", default="")
parser.add_argument(
"--white-noise-audio-dir",
default="",
help=(
"Legacy argument retained for CLI compatibility; HFDv2 reads the "
"continuous raw background session bound by the run contract."
),
)
parser.add_argument(
"--white-noise-start-index",
type=int,
default=0,
help="First cyclic white-noise clip index for this source session.",
)
parser.add_argument(
"--expected-checkpoint-global-step",
type=int,
default=0,
help=(
"Optional immutable checkpoint identity captured by the automatic "
"watcher. A mismatch aborts instead of reusing a stale best output."
),
)
parser.add_argument("--device", default="cuda:0")
parser.add_argument("--seed", type=int, default=42)
parser.add_argument("--ignore-control-token", action="store_true")
parser.add_argument(
"--control-condition-mode",
choices=("hard", "soft"),
default="",
help=(
"Optional explicit serial-control ablation. Empty uses the "
"checkpoint's inference mode (production: hard)."
),
)
parser.add_argument(
"--max-listen-tail-steps",
type=int,
default=500,
help=(
"Maximum post-WAV non-motion windows before inference fails "
"without overriding the model token."
),
)
parser.add_argument(
"--initial-idle-steps",
type=int,
default=0,
help="Number of 0.4 s synthetic background windows before the WAV.",
)
return parser.parse_args()
def main() -> None:
args = parse_args()
run_dir = relocate_hfd_storage_path(args.run_dir).resolve()
output_dir = relocate_hfd_storage_path(args.output_dir).resolve()
audio_path = relocate_hfd_storage_path(args.audio_path).resolve()
audio_paths = [audio_path] + [
relocate_hfd_storage_path(value).resolve()
for value in args.additional_audio_path
]
init_motion_dir = relocate_hfd_storage_path(args.init_motion_dir).resolve()
missing_audio = [path for path in audio_paths if not path.is_file()]
if missing_audio:
raise FileNotFoundError(f"Audio does not exist: {missing_audio[0]}")
if not init_motion_dir.is_dir():
raise FileNotFoundError(f"Initial motion does not exist: {init_motion_dir}")
output_dir.mkdir(parents=True, exist_ok=True)
cfg_template: LaunchConfig = parse_args_to_tyro_config(run_dir / "argv.txt") # type: ignore
raw_run_config = json.loads(
(run_dir / "run_config.json").read_text(encoding="utf-8")
)
saved_semantics_version = (
raw_run_config.get("train", {}).get("full_duplex_semantics_version")
if isinstance(raw_run_config, dict)
else None
)
if saved_semantics_version not in {9, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24}:
raise RuntimeError(
"Refusing fused Qwen-KV/serial-control inference on a legacy "
"checkpoint"
)
semantics_version = int(saved_semantics_version)
launch_config = cfg_template.model_validate(raw_run_config)
if int(getattr(launch_config.train, "stage", 0)) == 1:
raise RuntimeError("Stage 1 trains semantic/control only; use a stage-2 checkpoint for motion inference")
validate_contract(launch_config)
seed_everything(launch_config.seed or args.seed)
device = torch.device(args.device)
ckpt_step = resolve_ckpt_step(run_dir, args.ckpt_step)
backbone = (
str(args.backbone).strip()
or str(Path(str(launch_config.model.model_name_or_path)).expanduser())
)
if not (Path(backbone) / "config.json").is_file():
raise FileNotFoundError(f"Invalid local Qwen backbone: {backbone}")
continuous_root = relocate_hfd_storage_path(
str(launch_config.data.continuous_audio_root)
)
background_assignment_path = relocate_hfd_storage_path(
str(launch_config.data.background_assignment_path)
)
if not continuous_root.is_absolute():
continuous_root = PROJECT_ROOT / continuous_root
if not background_assignment_path.is_absolute():
background_assignment_path = (
PROJECT_ROOT / background_assignment_path
)
continuous_audio_production_identity(
continuous_root,
assignment_path=background_assignment_path,
)
active_contract = _build_active_hfdv2_contract(
launch_config,
semantics_version=semantics_version,
backbone=backbone,
continuous_audio_root=continuous_root,
background_assignment_path=background_assignment_path,
)
if getattr(launch_config.model, "interaction_architecture", "legacy") == "a1_c_deferred":
from psi.models.interaction_arch.runtime import ARCH_KEY, ArchitectureSpec, architecture_contract
active_contract[ARCH_KEY] = architecture_contract(ArchitectureSpec("a1", interaction="cross"))
expected_checkpoint_step = (
int(args.expected_checkpoint_global_step)
if int(args.expected_checkpoint_global_step) > 0
else None
)
checkpoint_marker = validate_checkpoint_encoder_contract(
run_dir,
ckpt_step,
active_contract,
expected_global_step=expected_checkpoint_step,
)
os.environ["PSI_PRETRAIN_BACKBONE_PATH"] = backbone
model_class = FullDuplexPsi0Model
if bool(getattr(launch_config.model, "swa_visual_qa_enabled", False)):
from psi.models.full_duplex_swa import FullDuplexSwaModel
from types import SimpleNamespace
from psi.models.interaction_arch.loading import load_swa_checkpoint
model_class = SimpleNamespace(from_streaming_checkpoint=load_swa_checkpoint)
model = model_class.from_streaming_checkpoint(
run_dir, ckpt_step, launch_config, device
)
control_condition_mode = (
str(args.control_condition_mode).strip()
or str(launch_config.model.control_condition_inference_mode)
)
model.set_control_condition_inference_mode(control_condition_mode)
marker_after_load = validate_checkpoint_encoder_contract(
run_dir,
ckpt_step,
active_contract,
expected_global_step=int(checkpoint_marker["global_step"]),
)
if marker_after_load != checkpoint_marker:
raise RuntimeError(
"Checkpoint metadata changed while model weights were loading; retry "
"inference to avoid an ambiguous mutable best_ckpt snapshot."
)
normalized_audio_paths: list[Path] = []
clean_waveforms: list[np.ndarray] = []
assignments: list[dict[str, Any] | None] = []
for turn_index, turn_audio_path in enumerate(audio_paths):
normalized_path = (
output_dir / "input_audio_16k_mono.wav"
if turn_index == 0
else output_dir / f"input_audio_turn_{turn_index + 1:02d}_16k_mono.wav"
)
clean = _normalise_audio(turn_audio_path, normalized_path)
assignment = _read_training_background_assignment(
background_assignment_path,
turn_audio_path,
)
if assignment is not None:
# The offline assignment builder used scipy.resample_poly for the
# 24 kHz TTS corpus. Reuse it for every known training command.
assigned = _read_resampled_speech_float(
turn_audio_path,
AUDIO_RATE,
)
if assigned.size != int(assignment["speech_samples"]):
raise RuntimeError(
"Training speech length changed after assignment: "
f"path={turn_audio_path} actual={assigned.size} "
f"expected={assignment['speech_samples']}"
)
clean = assigned
normalized_audio_paths.append(normalized_path)
clean_waveforms.append(clean)
assignments.append(assignment)
normalized_audio_path = normalized_audio_paths[0]
clean_waveform = clean_waveforms[0]
waveform = clean_waveform
continuous_background_listen_start_index: int | None = None
sessions = _read_background_sessions(continuous_root)
assignment = assignments[0]
is_multiturn = len(audio_paths) > 1
if is_multiturn and semantics_version not in {13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24}:
raise RuntimeError(
"Multiple command WAVs require a multi-turn semantics-v13+ checkpoint"
)
# A single-turn benchmark historically waits until the model has emitted
# the requested number of motion frames. That stopping rule is invalid
# for a multi-turn session: earlier commands already execute inside the
# inter-turn gaps, and the final command may naturally return to idle.
# Give the final command one fixed wall-clock action horizon and render
# every predicted state instead of waiting for extra </motion> tokens.
fixed_post_wav_background_steps = (
int(
math.ceil(
int(args.motion_frames) / float(ACTION_EXEC_FRAMES)
)
)
if is_multiturn
else 0
)
default_inter_turn_background_steps = (
max(int(args.inter_turn_background_steps), 1)
if int(args.inter_turn_background_steps) > 0
else int(
math.ceil(
(
int(args.motion_frames)
+ int(
getattr(
launch_config.data,
"turn_transition_frames",
ACTION_EXEC_FRAMES,
)
)
+ ACTION_EXEC_FRAMES
- 1
)
/ float(ACTION_EXEC_FRAMES)
)
)
)
inter_turn_background_steps_by_turn: list[int] = []
if is_multiturn:
raw_gap_steps = str(args.inter_turn_background_steps_by_turn).strip()
if raw_gap_steps:
try:
inter_turn_background_steps_by_turn = [
int(value.strip())
for value in raw_gap_steps.split(",")
if value.strip()
]
except ValueError as error:
raise ValueError(
"--inter-turn-background-steps-by-turn must be a "
f"comma-separated integer list, got {raw_gap_steps!r}"
) from error
expected_gaps = len(clean_waveforms) - 1
if len(inter_turn_background_steps_by_turn) != expected_gaps:
raise ValueError(
"--inter-turn-background-steps-by-turn needs one value "
f"per adjacent pair: turns={len(clean_waveforms)} "
f"gaps={inter_turn_background_steps_by_turn}"
)
if any(value < 0 for value in inter_turn_background_steps_by_turn):
raise ValueError(
"--inter-turn-background-steps-by-turn cannot contain "
f"negative values: {inter_turn_background_steps_by_turn}"
)
else:
inter_turn_background_steps_by_turn = [
default_inter_turn_background_steps
] * (len(clean_waveforms) - 1)
expected_exported_listen_windows = _exported_window_count(
clean_waveform.size
)
if (
not is_multiturn
and assignment is not None
and int(assignment["listen_window_count"])
== expected_exported_listen_windows
):
background_key = str(assignment["background_key"])
continuous_background_listen_start_index = (
int(assignment["background_listen_start_index"])
* EXPORTED_AUDIO_STRIDE_FRAMES
// AUDIO_STRIDE_FRAMES
)
selection = "exact_training_assignment"
else:
session_window_count = (
sum(_continuous_window_count(value.size) for value in clean_waveforms)
+ sum(inter_turn_background_steps_by_turn)
)
(
background_key,
continuous_background_listen_start_index,
) = _choose_continuous_background(
sessions=sessions,
speech_samples=clean_waveform.size,
motion_frames=int(args.motion_frames),
initial_idle_steps=max(int(args.initial_idle_steps), 0),
max_listen_tail_steps=max(
int(args.max_listen_tail_steps), 1
),
seed_key=(
"::".join(str(path) for path in audio_paths)
+ f"::{int(args.seed)}"
),
required_session_windows=(
session_window_count if is_multiturn else 0
),
terminal_background_steps=max(
int(args.terminal_background_steps), 0
),
)
selection = (
"deterministic_multiturn_assignment"
if is_multiturn
else "deterministic_inference_assignment"
)
session = sessions.get(background_key)
if session is None:
raise RuntimeError(
f"Assignment names an unknown background: {background_key}"
)
background_audio_path = Path(str(session["path"]))
if _file_sha256(background_audio_path) != str(session["sha256"]):
raise RuntimeError(
f"Continuous background WAV changed after production: "
f"{background_audio_path}"
)
background_waveform = load_omni_audio_array(
background_audio_path,
AUDIO_RATE,
)
turn_window_starts = [0]
turn_listen_counts = [_continuous_window_count(clean_waveform.size)]
if is_multiturn:
peak_limit = 10.0 ** (-3.0 / 20.0)
speech_scales = []
for clean, turn_assignment in zip(clean_waveforms, assignments):
if turn_assignment is not None:
speech_scales.append(float(turn_assignment["speech_scale"]))
continue
peak = float(np.max(np.abs(clean), initial=0.0))
speech_scales.append(
min(1.0, peak_limit / peak) if peak > 0.0 else 1.0
)
(
waveform,
turn_window_starts,
turn_listen_counts,
) = _build_continuous_multiturn_waveform(
speech_waveforms=clean_waveforms,
speech_scales=speech_scales,
background_waveform=background_waveform,
listen_start_index=continuous_background_listen_start_index,
inter_turn_background_steps=inter_turn_background_steps_by_turn,
)
speech_scale: float | list[float] = speech_scales
elif selection == "exact_training_assignment":
waveform, speech_scale = _build_continuous_mixed_waveform(
speech_waveform=clean_waveform,
background_waveform=background_waveform,
listen_start_index=continuous_background_listen_start_index,
)
if not math.isclose(
speech_scale,
float(assignment["speech_scale"]),
rel_tol=0.0,
abs_tol=1e-8,
):
raise RuntimeError("Runtime speech scale differs from training assignment")
if (
_continuous_window_count(clean_waveform.size)
!= len(complete_audio_clip_starts(len(waveform)))
):
raise RuntimeError(
"Exact training mixed WAV no longer matches its assignment: "
f"{audio_path}"
)
else:
waveform, speech_scale = _build_continuous_mixed_waveform(
speech_waveform=clean_waveform,
background_waveform=background_waveform,
listen_start_index=continuous_background_listen_start_index,
)
mixed_audio_path = output_dir / "input_audio_mixed_16k_mono.wav"
write_pcm16_wav(mixed_audio_path, waveform)
window_samples = int(round(AUDIO_WINDOW_SECONDS * AUDIO_RATE))
background_starts = complete_audio_clip_starts(
len(background_waveform)
)
white_noise_audio = [
background_waveform[
start : start + window_samples
]
for start in background_starts
]
if any(
window.shape != (window_samples,)
for window in white_noise_audio
):
raise RuntimeError(
f"Background WAV/index count mismatch: {background_key}"
)
background_metadata = {
"semantics_version": semantics_version,
"mode": "continuous_source_local",
"selection": selection,
"background_key": background_key,
"background_listen_start_index": int(
continuous_background_listen_start_index
),
"background_window_count": len(white_noise_audio),
"speech_scale": speech_scale,
"turn_window_starts": turn_window_starts,
"turn_listen_window_counts": turn_listen_counts,
"inter_turn_background_steps": (
default_inter_turn_background_steps if is_multiturn else 0
),
"inter_turn_background_steps_by_turn": (
inter_turn_background_steps_by_turn if is_multiturn else []
),
"fixed_post_wav_background_steps": int(
fixed_post_wav_background_steps
),
"mixed_audio_path": str(mixed_audio_path),
}
prompt = resolve_prompt(args.prompt, audio_path, launch_config)
validate_online_prompt(prompt)
init_abs = load_pose_from_motion_dir(init_motion_dir)
audio_prefix_seconds = AUDIO_WINDOW_SECONDS - (
AUDIO_STRIDE_FRAMES / float(MOTION_FPS)
)
audible_speech_onset_seconds = [
max(int(args.initial_idle_steps), 0)
* AUDIO_STRIDE_FRAMES
/ float(MOTION_FPS)
+ int(window_start) * AUDIO_STRIDE_FRAMES / float(MOTION_FPS)
+ audio_prefix_seconds
+ detect_clean_speech_onset_sample(clean, AUDIO_RATE)
/ float(AUDIO_RATE)
for clean, window_start in zip(clean_waveforms, turn_window_starts)
]
try:
motion, metadata = rollout_full_duplex(
model=model,
background_waveforms=white_noise_audio,
waveform=waveform,
prompt=prompt,
source_id="::".join(str(path) for path in audio_paths),
init_abs=init_abs,
launch_config=launch_config,
motion_frames=int(args.motion_frames),
num_inference_steps=int(args.num_inference_steps),
respect_control_token=not bool(args.ignore_control_token),
max_listen_tail_steps=max(int(args.max_listen_tail_steps), 1),
initial_idle_steps=max(int(args.initial_idle_steps), 0),
seed=int(args.seed),
white_noise_start_index=int(args.white_noise_start_index),
continuous_background_listen_start_index=(
continuous_background_listen_start_index
),
fixed_post_wav_background_steps=int(
fixed_post_wav_background_steps
),
terminal_background_steps=max(
int(args.terminal_background_steps), 0
),
)
except FullDuplexControlTimeout as error:
failure = dict(error.diagnostics)
failure.update(
{
"run_dir": str(run_dir),
"ckpt_step": ckpt_step,
"audio_path": str(audio_path),
"audio_paths": [str(path) for path in audio_paths],
"normalized_audio_path": str(normalized_audio_path),
"normalized_audio_paths": [
str(path) for path in normalized_audio_paths
],
"init_motion_dir": str(init_motion_dir),
"prompt": prompt,
"respect_control_token": not bool(
args.ignore_control_token
),
"initial_idle_steps_requested": max(
int(args.initial_idle_steps),
0,
),
"num_inference_steps": int(args.num_inference_steps),
"white_noise_start_index": int(args.white_noise_start_index),
"terminal_background_steps_requested": max(
int(args.terminal_background_steps), 0
),
"fixed_post_wav_background_steps": int(
fixed_post_wav_background_steps
),
}
)
(output_dir / "infer_failure.json").write_text(
json.dumps(failure, ensure_ascii=False, indent=2) + "\n",
encoding="utf-8",
)
shutil.copy2(
run_dir / "run_config.json",
output_dir / "run_config.snapshot.json",
)
raise
export_motion(output_dir, motion)
assembled_audio_path = output_dir / "assembled_input_audio_16k.wav"
assembled_audio = assemble_stream_audio(
speech_waveform=waveform,
white_noise_waveforms=white_noise_audio,
control_events=metadata["control_events"],
total_motion_frames=int(metadata["total_frames"]),
)
write_pcm16_wav(assembled_audio_path, assembled_audio)
metadata.update(
{
"run_dir": str(run_dir),
"ckpt_step": ckpt_step,
"audio_path": str(audio_path),
"audio_paths": [str(path) for path in audio_paths],
"normalized_audio_path": str(normalized_audio_path),
"normalized_audio_paths": [
str(path) for path in normalized_audio_paths
],
"assembled_input_audio_path": str(assembled_audio_path),
"background_audio": background_metadata,
"init_motion_dir": str(init_motion_dir),
"prompt": prompt,
"respect_control_token": not bool(args.ignore_control_token),
"initial_idle_steps_requested": max(
int(args.initial_idle_steps),
0,
),
"terminal_background_steps_requested": max(
int(args.terminal_background_steps), 0
),
"fixed_post_wav_background_steps_requested": int(
fixed_post_wav_background_steps
),
"num_inference_steps": int(args.num_inference_steps),
"checkpoint_global_step": int(checkpoint_marker["global_step"]),
"hfdv2_train_infer_contract": active_contract,
"control_condition_mode": control_condition_mode,
"white_noise_start_index_requested": int(
args.white_noise_start_index
),
"audio_sampling_rate": AUDIO_RATE,
"motion_fps": MOTION_FPS,
"audio_window_seconds": AUDIO_WINDOW_SECONDS,
"audio_stride_frames": AUDIO_STRIDE_FRAMES,
"action_chunk_frames": ACTION_CHUNK_FRAMES,
"action_exec_frames": ACTION_EXEC_FRAMES,
"rtc_prefix_frames": RTC_PREFIX_FRAMES,
"speech_start_seconds": float(audible_speech_onset_seconds[0]),
"speech_start_frame_50fps": int(
round(audible_speech_onset_seconds[0] * MOTION_FPS)
),
"turn_audible_speech_onset_seconds": [
float(value) for value in audible_speech_onset_seconds
],
"audio_timeline": {
"format": "overlap_average_stream_windows_v1",
"sample_rate": AUDIO_RATE,
"window_samples": int(
round(AUDIO_WINDOW_SECONDS * AUDIO_RATE)
),
"stride_samples": int(
round(AUDIO_STRIDE_FRAMES / MOTION_FPS * AUDIO_RATE)
),
"samples": int(assembled_audio.shape[0]),
"includes_initial_background": True,
"includes_speech": True,
"includes_post_wav_background": True,
"includes_terminal_background": bool(
int(args.terminal_background_steps) > 0
),
},
}
)
(output_dir / "infer_meta.json").write_text(
json.dumps(metadata, ensure_ascii=False, indent=2) + "\n",
encoding="utf-8",
)
shutil.copy2(run_dir / "run_config.json", output_dir / "run_config.snapshot.json")
motion_start_seconds = metadata["motion_start_seconds"]
motion_start_text = (
f"{float(motion_start_seconds):.3f}s"
if motion_start_seconds is not None
else "none"
)
print(
f"[done] clips={metadata['speech_clip_count']} "
f"cache={metadata['speech_cache_tokens']}->{metadata['final_cache_tokens']} "
f"updates={metadata['stream_cache_updates']} "
f"motion_start={motion_start_text} frames={metadata['total_frames']} "
f"output={output_dir}",
flush=True,
)
if __name__ == "__main__":
main()
#!/usr/bin/env python3
from __future__ import annotations
import argparse
import contextlib
import hashlib
import json
import math
import os
import shutil
import subprocess
import sys
import wave
from pathlib import Path
from typing import Any, Sequence
import numpy as np
import torch
PROJECT_ROOT = Path(__file__).resolve().parents[2]
DATA_SCRIPT_DIR = PROJECT_ROOT / "scripts" / "data"
INFER_SCRIPT_DIR = PROJECT_ROOT / "scripts" / "infer"
if str(DATA_SCRIPT_DIR) not in sys.path:
sys.path.insert(0, str(DATA_SCRIPT_DIR))
if str(INFER_SCRIPT_DIR) not in sys.path:
sys.path.insert(0, str(INFER_SCRIPT_DIR))
from cache_omni_hidden_states import ( # noqa: E402
load_audio_array as load_omni_audio_array,
)
from psi.config.config import LaunchConfig # noqa: E402
from psi.data.seed.full_duplex_audio_contract import ( # noqa: E402
AUDIO_COMMAND_STREAMING_PROMPT,
)
from psi.data.seed.full_duplex_audio_command_dataset import ( # noqa: E402
_read_resampled_speech_float,
detect_clean_speech_onset_sample,
)
from psi.data.seed.qwen_streaming_audio import ( # noqa: E402
continuous_audio_production_identity,
hfdv2_train_infer_contract,
prepare_qwen_streaming_audio_window,
)
from psi.data.seed.storage_paths import ( # noqa: E402
relocate_hfd_storage_path,
)
from psi.data.seed.motion_repr import ( # noqa: E402
build_seed_textop_action,
recover_seed_abs_from_textop_action,
)
from psi.data.seed.full_duplex_timing import ( # noqa: E402
ACTION_CHUNK_FRAMES,
ACTION_EXEC_FRAMES,
ACTION_SAMPLE_STEP_FRAMES,
AUDIO_RATE,
AUDIO_STRIDE_FRAMES,
AUDIO_WINDOW_SECONDS,
MOTION_FPS,
RTC_PREFIX_FRAMES,
complete_audio_clip_starts,
)
from psi.models.full_duplex_psi0 import ( # noqa: E402
FullDuplexPsi0Model,
FullDuplexStreamCondition,
)
from psi.utils import parse_args_to_tyro_config, seed_everything # noqa: E402
from psi.runtime.motion_rollout import _flow_action, _stationary_future_progress # noqa: E402
from psi.runtime.motion_rollout import decode_action_chunk, next_rtc_prefix
from seed_audio_to_motion import normalize_action_for_rtc # noqa: E402
from seed_text_to_motion import ( # noqa: E402
TARGET_FPS,
build_model_states_from_abs,
finite_difference,
load_pose_from_motion_dir,
normalize_quaternion_rows,
quaternion_angular_velocity,
resolve_ckpt_step,
write_csv,
write_metadata,
)
LISTEN_TOKEN = "</listen>"
MOTION_TOKEN = "</motion>"
IDLE_TOKEN = "</idle>"
CONTROL_TOKENS = (LISTEN_TOKEN, MOTION_TOKEN, IDLE_TOKEN)
DEFAULT_COMMAND_PROMPT = AUDIO_COMMAND_STREAMING_PROMPT
EXPORTED_AUDIO_STRIDE_FRAMES = 20
class FullDuplexControlTimeout(RuntimeError):
"""A strict rollout ended because the model never selected motion."""
def __init__(self, message: str, *, diagnostics: dict[str, Any]) -> None:
super().__init__(message)
self.diagnostics = diagnostics