Download src/psi/data/seed/full_duplex_audio_command_dataset.py from Tinghong-Ye22/code-HFD: direct link, hf CLI and curl.
- Browser
- Download file 367 kB
-
https://huggingface.co/Tinghong-Ye22/code-HFD/resolve/main/src/psi/data/seed/full_duplex_audio_command_dataset.py
- Command line
-
hf download hf://Tinghong-Ye22/code-HFD/src/psi/data/seed/full_duplex_audio_command_dataset.py
-
curl -L -o full_duplex_audio_command_dataset.py https://huggingface.co/Tinghong-Ye22/code-HFD/resolve/main/src/psi/data/seed/full_duplex_audio_command_dataset.py
367 kB
from __future__ import annotations
import hashlib
import json
import math
import random
import re
import sqlite3
import wave
from collections import OrderedDict
from dataclasses import dataclass
from pathlib import Path
from typing import Any, Iterable
import numpy as np
import pandas as pd
import torch
from safetensors import safe_open
from safetensors.torch import load_file
from scipy.signal import resample_poly
from psi.data.seed.omni_hidden_cache import (
load_hidden_states_slice,
load_hidden_states_source,
)
from psi.data.seed.motion_repr import (
estimate_two_foot_contacts,
quat_mul_wxyz,
quat_to_yaw_wxyz,
rpy_to_quat_wxyz,
wrap_angle,
)
from psi.data.seed.source_split import motion_source_id, select_source_row_ids
from psi.data.seed.storage_paths import relocate_hfd_storage_path
LISTEN_TOKEN_ID = 0
MOTION_TOKEN_ID = 1
IDLE_TOKEN_ID = 2
CONTROL_TOKEN_NAMES = ("</listen>", "</motion>", "</idle>")
CANONICAL_MOTION_MANIFEST_FORMAT = "hfd_canonical_motion_per_instruction_v1"
HESITATION_PAUSE_MANIFEST_FORMAT = "hfd_hesitation_pause_tts_v1"
DISFLUENCY_UNCERTAINTY_ROLES = frozenset(
{
"background-only-pause",
"hesitation-background-before",
"hesitation-filler",
"hesitation-background-after",
"repair-background-before",
"repair-cue",
"repair-background-after",
"repair-implicit-background",
}
)
# The continuous-audio corpus is exported at the deployment cadence: one
# 0.6-second window every 20 motion frames (0.4 seconds).
EXPORTED_AUDIO_STRIDE_FRAMES = 20
# These are deliberately exact metadata labels, not fuzzy keyword matches.
# SEED contains unsafe names such as idle_handstand, injured_*_idle and
# crouch_*_idle. The selected labels were manually audited against every
# natural-language description and kinematically checked for root drift.
LISTEN_MOTION_CONTENT_NAMES = frozenset({"listening_R"})
IDLE_MOTION_CONTENT_NAMES = frozenset({"neutral_idle"})
@dataclass(frozen=True)
class HiddenSlice:
frame_id: int
shard_path: Path
offset: int
length: int
@dataclass(frozen=True)
class AudioSource:
source_key: str
clips: tuple[HiddenSlice, ...]
packed_path: Path | None = None
packed_count: int = 0
# HFDv2 reads the exact continuous mixed waveform online. This field is
# mutually exclusive with the two cached-hidden layouts above.
waveform_path: Path | None = None
waveform_count: int = 0
@property
def window_count(self) -> int:
if self.waveform_path is not None:
return int(self.waveform_count)
return (
int(self.packed_count)
if self.packed_path is not None
else len(self.clips)
)
def instruction_text_for_voice(
record: dict[str, Any],
voice: AudioSource,
) -> str:
"""Return the exact text used to synthesize this voice waveform.
Qwen3-TTS uses ``content_short_description`` for ``*_en.wav`` and
``content_short_description_zh`` for ``*_zh.wav``. The action metadata is
English-only, so selecting ``record['instruction']`` unconditionally would
give every Chinese waveform a false auxiliary transcript target.
"""
voice_name = Path(voice.source_key).name
if voice_name.endswith("_zh"):
key = "instruction_zh"
elif voice_name.endswith("_en"):
key = "instruction"
else:
# Retain lightweight synthetic fixtures and legacy private caches that
# predate the production ``*_en``/``*_zh`` filename contract.
key = "instruction"
text = str(record.get(key, "")).strip()
if not text:
raise RuntimeError(
"The selected TTS voice has no exact transcript label: "
f"row={record.get('row_id')} voice={voice.source_key} key={key}"
)
return text
def has_known_tts_translation_mismatch(
instruction: str,
instruction_zh: str,
) -> bool:
"""Identify the audited ``retro`` -> ``后退`` TTS mistranslation."""
return bool(
re.search(r"\bretro\b", str(instruction), flags=re.IGNORECASE)
and "后退" in str(instruction_zh)
)
def _load_canonical_motion_manifest(
path: str | Path,
) -> tuple[dict[tuple[str, str], str], frozenset[str]]:
"""Load one globally selected motion for every bilingual instruction.
Selection happens before the source-level train/validation split. This
prevents the same instruction from mapping to one trajectory in train and
a different trajectory in validation, which would silently reintroduce
the ambiguity that this opt-in ablation is intended to remove.
"""
manifest_path = Path(path)
if not manifest_path.is_file():
raise FileNotFoundError(
f"Missing canonical-motion manifest: {manifest_path}"
)
with manifest_path.open("r", encoding="utf-8") as handle:
payload = json.load(handle)
if payload.get("format") != CANONICAL_MOTION_MANIFEST_FORMAT:
raise RuntimeError(
"Unsupported canonical-motion manifest format: "
f"{payload.get('format')!r}"
)
entries = payload.get("selections")
if not isinstance(entries, list) or not entries:
raise RuntimeError(
f"Canonical-motion manifest has no selections: {manifest_path}"
)
result: dict[tuple[str, str], str] = {}
selected_filenames: set[str] = set()
for entry in entries:
if not isinstance(entry, dict):
raise RuntimeError(
f"Invalid canonical-motion selection row: {entry!r}"
)
key = (
str(entry.get("instruction", "")).strip(),
str(entry.get("instruction_zh", "")).strip(),
)
filename = str(entry.get("filename", "")).strip()
if not all(key) or not filename:
raise RuntimeError(
f"Incomplete canonical-motion selection row: {entry!r}"
)
if key in result:
raise RuntimeError(
f"Duplicate canonical instruction pair: {key!r}"
)
if filename in selected_filenames:
raise RuntimeError(
"One canonical motion cannot represent two different exact "
f"instructions: {filename}"
)
result[key] = filename
selected_filenames.add(filename)
chinese_to_keys: dict[str, list[tuple[str, str]]] = {}
for key in result:
chinese_to_keys.setdefault(key[1], []).append(key)
ambiguous_chinese = frozenset(
text for text, keys in chinese_to_keys.items() if len(keys) > 1
)
declared_exclusions = payload.get(
"excluded_ambiguous_chinese_instructions",
[],
)
if not isinstance(declared_exclusions, list) or any(
not isinstance(text, str) or not text.strip()
for text in declared_exclusions
):
raise RuntimeError(
"Canonical-motion Chinese exclusions must be non-empty strings"
)
declared_exclusions = frozenset(
text.strip() for text in declared_exclusions
)
if declared_exclusions != ambiguous_chinese:
raise RuntimeError(
"Canonical-motion manifest must exclude every Chinese prompt that "
"still maps to multiple English instructions: "
f"expected={sorted(ambiguous_chinese)[:3]} "
f"declared={sorted(declared_exclusions)[:3]}"
)
return result, ambiguous_chinese
def load_canonical_motion_selection(
path: str | Path,
) -> dict[tuple[str, str], str]:
"""Return the audited bilingual-instruction to motion mapping."""
selection, _ = _load_canonical_motion_manifest(path)
return selection
@dataclass(frozen=True)
class StationaryMotionSource:
filename: str
motion_dir: Path
declared_frames: int
content_name: str
@dataclass(frozen=True)
class BackgroundHiddenSource:
source_key: str
packed_path: Path
window_count: int
offsets: tuple[int, ...]
@dataclass(frozen=True)
class BackgroundWaveformSource:
source_key: str
waveform_path: Path
window_count: int
@dataclass(frozen=True)
class SourceBackgroundAssignment:
source_key: str
background_key: str
listen_start_index: int
listen_window_count: int
background_window_count: int
initial_idle_max_steps: int
max_motion_steps: int
safety_steps: int
mixed_audio_path: Path | None = None
original_audio_path: Path | None = None
speech_samples: int = 0
speech_scale: float = 1.0
@dataclass(frozen=True)
class ControlCommandSource:
"""One audited emergency/hold command rendition used inside a session."""
intent: str
phrase_id: str
voice_name: str
language: str
text: str
waveform_path: Path
speech_samples: int
speech_scale: float = 1.0
@dataclass(frozen=True)
class BackchannelSource:
"""One short same-speaker acknowledgement overlaid during motion."""
category: str
phrase_id: str
split: str
voice_name: str
language: str
text: str
waveform_path: Path
speech_samples: int
speech_scale: float = 1.0
@dataclass(frozen=True)
class DisfluencyAudioSource:
"""One regenerated instruction segment or split-specific discourse cue."""
category: str
segment_id: str
split: str
voice_name: str
language: str
text: str
waveform_path: Path
speech_samples: int
speech_scale: float = 1.0
audio_dir: str = ""
instruction: str = ""
instruction_zh: str = ""
segment_role: str = ""
def _normalise_audio_dir(value: str) -> str:
"""Map an old absolute Qwen3-TTS path to the portable hidden-cache key."""
parts = Path(str(value).replace("\\", "/")).parts
try:
start = parts.index("seed_audio")
except ValueError:
return Path(str(value)).as_posix().strip("/")
return Path(*parts[start:]).as_posix()
def _pcm16_wave_num_samples(path: Path, expected_rate: int) -> int:
if not path.is_file():
raise FileNotFoundError(f"Missing online HFDv2 waveform: {path}")
with wave.open(str(path), "rb") as handle:
if (
handle.getnchannels() != 1
or handle.getsampwidth() != 2
or handle.getframerate() != int(expected_rate)
):
raise ValueError(
"HFDv2 online waveforms must be mono PCM16 at "
f"{expected_rate} Hz: {path}"
)
return int(handle.getnframes())
def _read_pcm16_wave_segment(
path: Path,
*,
start_sample: int,
sample_count: int,
expected_rate: int,
) -> np.ndarray:
start = int(start_sample)
count = int(sample_count)
if start < 0 or count < 0:
raise ValueError(
f"Invalid waveform slice at {path}: start={start}, count={count}"
)
with wave.open(str(path), "rb") as handle:
if (
handle.getnchannels() != 1
or handle.getsampwidth() != 2
or handle.getframerate() != int(expected_rate)
):
raise ValueError(
"HFDv2 online waveforms must be mono PCM16 at "
f"{expected_rate} Hz: {path}"
)
if start + count > int(handle.getnframes()):
raise IndexError(
f"Waveform slice exceeds {path}: start={start}, count={count}, "
f"samples={handle.getnframes()}"
)
handle.setpos(start)
raw = handle.readframes(count)
pcm = np.frombuffer(raw, dtype="<i2")
if pcm.shape != (count,):
raise RuntimeError(
f"Short PCM read at {path}: got={pcm.shape} expected={(count,)}"
)
# Match the existing online Qwen inference loader's PCM16 normalization.
return (pcm.astype(np.float32) / 32768.0).copy()
def _read_resampled_speech_float(path: Path, expected_rate: int) -> np.ndarray:
"""Decode the TTS PCM16 WAV exactly like the offline corpus builder."""
if not path.is_file():
raise FileNotFoundError(f"Missing original speech waveform: {path}")
with wave.open(str(path), "rb") as handle:
channels = int(handle.getnchannels())
sample_width = int(handle.getsampwidth())
source_rate = int(handle.getframerate())
compression = str(handle.getcomptype())
frames = int(handle.getnframes())
raw = handle.readframes(frames)
if (
channels != 1
or sample_width != 2
or source_rate <= 0
or compression != "NONE"
):
raise ValueError(
"Expected uncompressed mono PCM16 TTS WAV: "
f"path={path} channels={channels} width={sample_width} "
f"rate={source_rate} compression={compression}"
)
speech = (
np.frombuffer(raw, dtype="<i2").astype(np.float32)
/ np.float32(32768.0)
)
if source_rate != int(expected_rate):
divisor = math.gcd(source_rate, int(expected_rate))
speech = resample_poly(
speech,
int(expected_rate) // divisor,
source_rate // divisor,
).astype(np.float32, copy=False)
expected = int(round(frames * int(expected_rate) / float(source_rate)))
if speech.size > expected:
speech = speech[:expected]
elif speech.size < expected:
speech = np.pad(speech, (0, expected - speech.size))
if speech.size <= 0 or not np.isfinite(speech).all():
raise RuntimeError(f"Invalid decoded speech source: {path}")
return np.ascontiguousarray(speech, dtype=np.float32)
def detect_clean_speech_onset_sample(
waveform: np.ndarray,
sampling_rate: int,
) -> int:
"""Locate the first sustained speech energy in one clean TTS waveform.
The detector is deliberately applied before office background is mixed in.
A relative RMS threshold handles different speaker gains, while three
adjacent 10 ms hops reject isolated PCM clicks. The returned sample is
only used to move a control boundary to a later 0.4 s Qwen update; it does
not trim, stretch or otherwise modify the speech waveform.
"""
values = np.asarray(waveform, dtype=np.float32).reshape(-1)
rate = int(sampling_rate)
if rate <= 0 or values.size <= 0 or not np.isfinite(values).all():
raise ValueError(
"Speech onset detection needs a finite non-empty waveform and "
f"positive sample rate, got samples={values.size} rate={rate}"
)
frame = max(int(round(0.020 * rate)), 1)
hop = max(int(round(0.010 * rate)), 1)
if values.size < frame:
rms = np.asarray(
[float(np.sqrt(np.mean(np.square(values, dtype=np.float64))))],
dtype=np.float64,
)
else:
starts = np.arange(0, values.size - frame + 1, hop, dtype=np.int64)
rms = np.asarray(
[
np.sqrt(
np.mean(
np.square(values[start : start + frame], dtype=np.float64)
)
)
for start in starts
],
dtype=np.float64,
)
floor = float(np.quantile(rms, 0.10))
peak = float(np.quantile(rms, 0.95))
threshold = max(2.5e-3, floor + 0.08 * max(peak - floor, 0.0))
active = rms >= threshold
if active.size >= 3:
sustained = np.convolve(
active.astype(np.int8),
np.ones(3, dtype=np.int8),
mode="valid",
)
matches = np.flatnonzero(sustained == 3)
if matches.size:
return min(int(matches[0]) * hop, values.size - 1)
matches = np.flatnonzero(active)
return min(int(matches[0]) * hop, values.size - 1) if matches.size else 0
def _map_seed_motion_dir(root_dir: Path, move_g1_path: str) -> Path:
rel = str(move_g1_path).strip().replace("\\", "/")
rel = rel.replace("g1/csv/", "g1/csv_sonic/")
rel = rel.replace("/csv/", "/csv_sonic/")
if rel.endswith(".csv"):
rel = rel[:-4]
if rel.startswith("dataset/seed/"):
rel = rel[len("dataset/seed/") :]
if rel.startswith("seed/"):
rel = rel[len("seed/") :]
path = Path(rel)
return path if path.is_absolute() else root_dir / path
def _progress_rows(frame_indices: Iterable[int], total_frames: int) -> np.ndarray:
denominator = float(max(int(total_frames) - 1, 1))
rows = []
for frame_index in frame_indices:
progress = float(min(max(int(frame_index), 0), total_frames - 1)) / denominator
rows.append(
[
progress,
max(0.0, 1.0 - progress),
progress,
math.sin(2.0 * math.pi * progress),
math.cos(2.0 * math.pi * progress),
]
)
return np.asarray(rows, dtype=np.float32)
def _normalise_motion_quaternions(motion: np.ndarray) -> np.ndarray:
result = np.asarray(motion, dtype=np.float32).copy()
quat = result[:, 32:36]
norms = np.linalg.norm(quat, axis=-1, keepdims=True)
fallback = np.zeros_like(quat)
fallback[:, 0] = 1.0
result[:, 32:36] = np.where(
norms > 1e-8,
quat / np.maximum(norms, 1e-8),
fallback,
)
return result
def _blend_pose_rows(
anchor: np.ndarray,
motion: np.ndarray,
weight: np.ndarray,
) -> np.ndarray:
"""Blend absolute G1 poses while keeping quaternion signs and norms valid."""
anchor_rows = np.broadcast_to(
np.asarray(anchor, dtype=np.float32).reshape(1, 36),
motion.shape,
).copy()
moving = np.asarray(motion, dtype=np.float32).copy()
# q and -q are the same rotation. Flip before linear blending so an
# equivalent sign choice cannot pass through a zero quaternion.
quat_dot = (anchor_rows[:, 32:36] * moving[:, 32:36]).sum(axis=-1)
moving[quat_dot < 0.0, 32:36] *= -1.0
alpha = np.asarray(weight, dtype=np.float32).reshape(-1, 1)
blended = anchor_rows * (1.0 - alpha) + moving * alpha
return _normalise_motion_quaternions(blended)
def _align_task_motion_to_anchor_heading(
motion: np.ndarray,
stationary_anchor: np.ndarray,
) -> np.ndarray:
"""Rigidly express a task trajectory in the anchor's world heading.
TextOp observes local root deltas and yaw deltas, not absolute world yaw.
Requiring a task recorded at one arbitrary world heading to start from an
independently sampled idle heading would therefore create an unobservable
turn target. A single world-frame yaw transform preserves the task's local
motion while making its first heading and horizontal origin match the
current stationary robot state. Root height keeps its original transition
semantics.
"""
task = np.asarray(motion, dtype=np.float32)
anchor = np.asarray(stationary_anchor, dtype=np.float32).reshape(-1)
if task.ndim != 2 or task.shape[1] != 36 or task.shape[0] <= 0:
raise ValueError(f"Invalid task motion shape: {task.shape}")
if anchor.shape != (36,):
raise ValueError(f"Invalid stationary anchor shape: {anchor.shape}")
if not np.isfinite(task).all() or not np.isfinite(anchor).all():
raise ValueError("Task motion and stationary anchor must be finite")
aligned = _normalise_motion_quaternions(task)
task_yaw = float(quat_to_yaw_wxyz(aligned[0, 32:36]))
anchor_yaw = float(quat_to_yaw_wxyz(anchor[32:36]))
yaw_delta = float(
wrap_angle(np.asarray(anchor_yaw - task_yaw, dtype=np.float32))
)
relative_root = aligned[:, 29:31] - aligned[0:1, 29:31]
cos_yaw = math.cos(yaw_delta)
sin_yaw = math.sin(yaw_delta)
rotated_x = cos_yaw * relative_root[:, 0] - sin_yaw * relative_root[:, 1]
rotated_y = sin_yaw * relative_root[:, 0] + cos_yaw * relative_root[:, 1]
aligned[:, 29] = anchor[29] + rotated_x
aligned[:, 30] = anchor[30] + rotated_y
yaw_quat = rpy_to_quat_wxyz(0.0, 0.0, yaw_delta)
aligned[:, 32:36] = quat_mul_wxyz(yaw_quat, aligned[:, 32:36])
return _normalise_motion_quaternions(aligned)
class FullDuplexAudioCommandDataset(torch.utils.data.Dataset):
"""Return one complete spoken-command source and its streaming motion.
A dataset item remains one ordered source so the DataLoader cannot shuffle
its windows. Action targets advance by one 50 Hz frame, while a new 0.6 s
audio window is encoded only every 20 frames. The 19 intervening action
samples reuse that history-conditioned Qwen output without extending KV.
"""
def __init__(
self,
*,
hidden_root: str,
metadata_jsonl: str,
motion_metadata_csv: str,
motion_root: str,
white_noise_hidden_dir: str = "",
background_hidden_root: str = "",
background_assignment_path: str = "",
continuous_audio_root: str = "",
online_audio: bool = False,
background_source_name: str = "hfd_continuous_office_background",
background_task_name: str = "audio_command_text",
split: str = "train",
val_ratio: float = 0.1,
split_seed: int = 42,
split_source_namespace: str = "seed",
source_name: str = "qwen3_tts_seed_audio_command",
task_name: str = "audio_command_text",
action_chunk_size: int = 30,
action_exec_horizon: int = 20,
motion_fps: int = 50,
motion_metadata_fps: int = 120,
audio_sampling_rate: int = 16000,
audio_window_seconds: float = 0.6,
audio_sample_step_frames: int = 20,
action_sample_step_frames: int = 1,
motion_cache_size: int = 256,
fixed_voice_per_record: bool = True,
max_voice_windows: int = 30,
max_voice_median_ratio: float = 3.0,
deterministic_eval: bool = True,
limit_sources: int = 0,
max_motion_steps: int = 0,
initial_idle_max_steps: int = 100,
stationary_blend_frames: int = 20,
stationary_heading_alignment: bool = False,
safe_stationary_anchors: bool = False,
stationary_idle_residual_retargeting: bool = False,
stationary_listen_hold: bool = False,
dynamic_task_entry_blend: bool = False,
multi_turn_sessions: bool = False,
session_min_turns: int = 2,
session_max_turns: int = 4,
session_voice_variants: int = 0,
interruption_sessions: bool = False,
interruption_ratio: float = 0.5,
interruption_min_start_seconds: float = 2.0,
interruption_max_start_seconds: float = 6.0,
interruption_min_remaining_seconds: float = 1.5,
independent_task_pools: bool = False,
turn_taking_session_variants: int = 2,
command_interruption_session_variants: int = 2,
emergency_stop_session_variants: int = 1,
safe_hold_session_variants: int = 1,
backchannel_session_variants: int = 0,
hesitation_session_variants: int = 0,
pause_session_variants: int = 0,
interruption_task_session_ratio: float = 0.0,
control_task_session_ratio: float = 0.0,
backchannel_task_session_ratio: float = 0.0,
hesitation_task_window_ratio: float = 0.0,
pause_task_window_ratio: float = 0.0,
control_followup_ratio: float = 0.0,
control_post_idle_min_steps: int = 1,
control_post_idle_max_steps: int = 0,
control_post_idle_sampling: str = "legacy_uniform",
control_min_start_seconds: float = 2.0,
control_max_start_seconds: float = 6.0,
control_min_remaining_seconds: float = 0.4,
control_activity_aware_timing: bool = False,
filter_known_tts_translation_mismatches: bool = False,
canonical_motion_manifest: str = "",
control_command_manifest: str = "",
backchannel_command_manifest: str = "",
hesitation_pause_manifest: str = "",
new_task_max_rows_per_instruction_train: int = 32,
new_task_max_rows_per_instruction_val: int = 8,
hesitation_filled_ratio: float = 0.60,
self_repair_implicit_ratio: float = 0.0,
backchannel_min_start_seconds: float = 1.2,
backchannel_max_start_seconds: float = 6.0,
backchannel_min_remaining_seconds: float = 1.2,
safe_hold_stability_window_frames: int = 20,
turn_transition_frames: int = 50,
causal_listen_timing: bool = False,
max_speech_onset_steps: int = 3,
listen_reaction_steps: int = 1,
max_stream_steps: int = 1923,
long_session_drop_ratio: float = 0.0,
) -> None:
self.hidden_root = relocate_hfd_storage_path(hidden_root)
self.metadata_jsonl = relocate_hfd_storage_path(metadata_jsonl)
self.motion_metadata_csv = relocate_hfd_storage_path(
motion_metadata_csv
)
self.motion_root = relocate_hfd_storage_path(motion_root)
self.white_noise_hidden_dir = (
relocate_hfd_storage_path(white_noise_hidden_dir)
if str(white_noise_hidden_dir)
else None
)
self.background_hidden_root = (
relocate_hfd_storage_path(background_hidden_root)
if str(background_hidden_root)
else None
)
self.background_assignment_path = (
relocate_hfd_storage_path(background_assignment_path)
if str(background_assignment_path)
else None
)
self.continuous_audio_root = (
relocate_hfd_storage_path(continuous_audio_root)
if str(continuous_audio_root)
else (
self.background_assignment_path.parent.parent
if self.background_assignment_path is not None
else None
)
)
self.online_audio = bool(online_audio)
self.background_source_name = str(background_source_name)
self.background_task_name = str(background_task_name)
self.split = str(split)
self.val_ratio = float(val_ratio)
self.split_seed = int(split_seed)
self.split_source_namespace = str(split_source_namespace)
self.source_name = str(source_name)
self.task_name = str(task_name)
self.action_chunk_size = int(action_chunk_size)
self.action_exec_horizon = int(action_exec_horizon)
self.prefix_frames = self.action_chunk_size - self.action_exec_horizon
self.motion_fps = int(motion_fps)
self.motion_metadata_fps = int(motion_metadata_fps)
self.audio_sampling_rate = int(audio_sampling_rate)
self.audio_window_seconds = float(audio_window_seconds)
self.audio_sample_step_frames = int(audio_sample_step_frames)
self.action_sample_step_frames = int(action_sample_step_frames)
self.motion_cache_size = max(int(motion_cache_size), 0)
self.fixed_voice_per_record = bool(fixed_voice_per_record)
self.max_voice_windows = int(max_voice_windows)
self.max_voice_median_ratio = float(max_voice_median_ratio)
self.deterministic_eval = bool(deterministic_eval)
self.max_motion_steps = max(int(max_motion_steps), 0)
self.initial_idle_max_steps = max(int(initial_idle_max_steps), 0)
self.stationary_blend_frames = max(int(stationary_blend_frames), 1)
self.stationary_heading_alignment = bool(
stationary_heading_alignment
)
self.safe_stationary_anchors = bool(safe_stationary_anchors)
self.stationary_idle_residual_retargeting = bool(
stationary_idle_residual_retargeting
)
self.stationary_listen_hold = bool(stationary_listen_hold)
self.dynamic_task_entry_blend = bool(dynamic_task_entry_blend)
self.multi_turn_sessions = bool(multi_turn_sessions)
self.session_min_turns = int(session_min_turns)
self.session_max_turns = int(session_max_turns)
self.session_voice_variants = int(session_voice_variants)
self.interruption_sessions = bool(interruption_sessions)
self.interruption_ratio = float(interruption_ratio)
self.interruption_min_start_seconds = float(
interruption_min_start_seconds
)
self.interruption_max_start_seconds = float(
interruption_max_start_seconds
)
self.interruption_min_remaining_seconds = float(
interruption_min_remaining_seconds
)
self.independent_task_pools = bool(independent_task_pools)
self.task_session_variants = {
"turn-taking": int(turn_taking_session_variants),
"interruption": int(command_interruption_session_variants),
"emergency-stop": int(emergency_stop_session_variants),
"safe-hold": int(safe_hold_session_variants),
"backchannel": int(backchannel_session_variants),
"hesitation": int(hesitation_session_variants),
"pause": int(pause_session_variants),
}
self.interruption_task_session_ratio = float(
interruption_task_session_ratio
)
self.control_task_session_ratio = float(
control_task_session_ratio
)
self.backchannel_task_session_ratio = float(
backchannel_task_session_ratio
)
self.hesitation_task_window_ratio = float(
hesitation_task_window_ratio
)
self.pause_task_window_ratio = float(pause_task_window_ratio)
self.control_followup_ratio = float(control_followup_ratio)
self.control_post_idle_min_steps = max(
int(control_post_idle_min_steps),
1,
)
configured_post_idle_max = int(control_post_idle_max_steps)
self.control_post_idle_max_steps_explicit = (
configured_post_idle_max > 0
)
self.control_post_idle_max_steps = (
configured_post_idle_max
if self.control_post_idle_max_steps_explicit
else max(self.initial_idle_max_steps, 1)
)
self.control_post_idle_sampling = str(
control_post_idle_sampling
).strip()
if self.control_post_idle_sampling not in {
"legacy_uniform",
"short_heavy_long_tail",
}:
raise ValueError(
"Unsupported control_post_idle_sampling: "
f"{self.control_post_idle_sampling!r}"
)
self.control_min_start_seconds = float(control_min_start_seconds)
self.control_max_start_seconds = float(control_max_start_seconds)
self.control_min_remaining_seconds = float(
control_min_remaining_seconds
)
self.control_activity_aware_timing = bool(
control_activity_aware_timing
)
self.filter_known_tts_translation_mismatches = bool(
filter_known_tts_translation_mismatches
)
self.canonical_motion_manifest = (
relocate_hfd_storage_path(canonical_motion_manifest)
if str(canonical_motion_manifest).strip()
else None
)
self.control_command_manifest = (
relocate_hfd_storage_path(control_command_manifest)
if str(control_command_manifest).strip()
else None
)
self.backchannel_command_manifest = (
relocate_hfd_storage_path(backchannel_command_manifest)
if str(backchannel_command_manifest).strip()
else None
)
self.hesitation_pause_manifest = (
relocate_hfd_storage_path(hesitation_pause_manifest)
if str(hesitation_pause_manifest).strip()
else None
)
self.new_task_max_rows_per_instruction_train = max(
int(new_task_max_rows_per_instruction_train),
1,
)
self.new_task_max_rows_per_instruction_val = max(
int(new_task_max_rows_per_instruction_val),
1,
)
self.hesitation_filled_ratio = float(hesitation_filled_ratio)
self.self_repair_implicit_ratio = float(
self_repair_implicit_ratio
)
self.backchannel_min_start_seconds = float(
backchannel_min_start_seconds
)
self.backchannel_max_start_seconds = float(
backchannel_max_start_seconds
)
self.backchannel_min_remaining_seconds = float(
backchannel_min_remaining_seconds
)
self.safe_hold_stability_window_frames = max(
int(safe_hold_stability_window_frames),
2,
)
self.turn_transition_frames = max(int(turn_transition_frames), 2)
self.causal_listen_timing = bool(causal_listen_timing)
self.max_speech_onset_steps = max(int(max_speech_onset_steps), 1)
self.listen_reaction_steps = max(int(listen_reaction_steps), 0)
self.max_stream_steps = max(int(max_stream_steps), 0)
self.long_session_drop_ratio = float(long_session_drop_ratio)
if not (
math.isfinite(self.long_session_drop_ratio)
and 0.0 <= self.long_session_drop_ratio < 1.0
):
raise ValueError(
"long_session_drop_ratio must be finite and within [0, 1), "
f"got {self.long_session_drop_ratio}"
)
if self.long_session_drop_ratio > 0.0 and not self.multi_turn_sessions:
raise ValueError(
"Long-session tail filtering requires multi_turn_sessions=True"
)
if not (
1 <= self.session_min_turns <= self.session_max_turns <= 4
):
raise ValueError(
"Sessions require 1 <= min_turns <= max_turns <= 4, "
f"got {self.session_min_turns}..{self.session_max_turns}"
)
if self.session_voice_variants < 0:
raise ValueError(
"session_voice_variants must be non-negative, got "
f"{self.session_voice_variants}"
)
if self.session_voice_variants > 0 and not self.multi_turn_sessions:
raise ValueError(
"Session voice variants require multi_turn_sessions=True"
)
if self.interruption_sessions:
if self.session_max_turns < 2:
raise ValueError("Command interruption requires at least two source turns")
if not self.multi_turn_sessions:
raise ValueError(
"Interruption sessions require multi_turn_sessions=True"
)
if not self.causal_listen_timing:
raise ValueError(
"Interruption sessions require causal_listen_timing=True"
)
if not self.independent_task_pools and self.session_voice_variants != 2:
raise ValueError(
"The 1:1 turn-taking/interruption design requires exactly "
"two train session variants"
)
if self.independent_task_pools:
invalid_weights = {
name: count
for name, count in self.task_session_variants.items()
if count < 0
}
if invalid_weights or self.task_session_variants["turn-taking"] < 1:
raise ValueError(
"Independent task variant counts must be non-negative "
"and retain turn-taking: "
f"{self.task_session_variants}"
)
if (
self.task_session_variants["emergency-stop"] > 0
or self.task_session_variants["safe-hold"] > 0
) and (
self.control_command_manifest is None
or not self.control_command_manifest.is_file()
):
raise FileNotFoundError(
"Emergency-stop/safe-hold data requires the generated "
f"control-command manifest: {self.control_command_manifest}"
)
if self.task_session_variants["backchannel"] > 0 and (
self.backchannel_command_manifest is None
or not self.backchannel_command_manifest.is_file()
):
raise FileNotFoundError(
"Backchannel data requires the generated same-speaker "
"manifest: "
f"{self.backchannel_command_manifest}"
)
disfluency_modes_enabled = any(
self.task_session_variants[mode] > 0
for mode in ("hesitation", "pause")
)
if disfluency_modes_enabled and (
self.hesitation_pause_manifest is None
or not self.hesitation_pause_manifest.is_file()
):
raise FileNotFoundError(
"Hesitation/pause data requires the regenerated segment "
f"manifest: {self.hesitation_pause_manifest}"
)
enabled_control_modes = sum(
int(self.task_session_variants[mode] > 0)
for mode in ("emergency-stop", "safe-hold")
)
if not (
math.isfinite(self.interruption_task_session_ratio)
and 0.0 <= self.interruption_task_session_ratio < 1.0
and (
self.interruption_task_session_ratio == 0.0
or self.task_session_variants["interruption"] > 0
)
):
raise ValueError(
"Interruption task session ratio must be zero or target "
"an enabled interruption pool: "
f"ratio={self.interruption_task_session_ratio}"
)
if not (
math.isfinite(self.backchannel_task_session_ratio)
and 0.0 <= self.backchannel_task_session_ratio < 1.0
and (
self.backchannel_task_session_ratio == 0.0
or self.task_session_variants["backchannel"] > 0
)
):
raise ValueError(
"Backchannel task session ratio must be zero or target "
"an enabled backchannel pool: "
f"ratio={self.backchannel_task_session_ratio}"
)
new_window_ratios = {
"hesitation": self.hesitation_task_window_ratio,
"pause": self.pause_task_window_ratio,
}
invalid_new_ratios = {
mode: ratio
for mode, ratio in new_window_ratios.items()
if (
not math.isfinite(ratio)
or ratio < 0.0
or ratio >= 1.0
or (
ratio > 0.0
and self.task_session_variants[mode] <= 0
)
)
}
if invalid_new_ratios or sum(new_window_ratios.values()) >= 1.0:
raise ValueError(
"Invalid hesitation/pause final window ratios: "
f"{new_window_ratios} invalid={invalid_new_ratios}"
)
if not (
math.isfinite(self.hesitation_filled_ratio)
and 0.0 < self.hesitation_filled_ratio < 1.0
):
raise ValueError(
"hesitation_filled_ratio must be strictly within (0, 1), "
f"got {self.hesitation_filled_ratio}"
)
if not (
math.isfinite(self.self_repair_implicit_ratio)
and 0.0 <= self.self_repair_implicit_ratio <= 1.0
):
raise ValueError(
"self_repair_implicit_ratio must be within [0, 1], "
f"got {self.self_repair_implicit_ratio}"
)
if not (
math.isfinite(self.control_task_session_ratio)
and 0.0 <= self.control_task_session_ratio
and self.control_task_session_ratio * enabled_control_modes
+ self.interruption_task_session_ratio
+ self.backchannel_task_session_ratio
< 1.0
):
raise ValueError(
"Control task session ratio must leave non-control "
"sessions in the mixture: "
f"ratio={self.control_task_session_ratio} "
f"modes={enabled_control_modes}"
)
backchannel_timing = (
self.backchannel_min_start_seconds,
self.backchannel_max_start_seconds,
self.backchannel_min_remaining_seconds,
)
if (
not all(
math.isfinite(value) and value > 0.0
for value in backchannel_timing
)
or self.backchannel_min_start_seconds
> self.backchannel_max_start_seconds
):
raise ValueError(
"Invalid backchannel timing: "
f"start={self.backchannel_min_start_seconds}.."
f"{self.backchannel_max_start_seconds}s remaining="
f"{self.backchannel_min_remaining_seconds}s"
)
if not (
math.isfinite(self.control_followup_ratio)
and 0.0 <= self.control_followup_ratio <= 1.0
):
raise ValueError(
"control_followup_ratio must be within [0, 1], got "
f"{self.control_followup_ratio}"
)
if (
self.control_post_idle_min_steps
> self.control_post_idle_max_steps
):
raise ValueError(
"Control post-idle minimum exceeds the configured "
"idle range: "
f"minimum={self.control_post_idle_min_steps} "
f"maximum={self.control_post_idle_max_steps}"
)
control_timing = (
self.control_min_start_seconds,
self.control_max_start_seconds,
self.control_min_remaining_seconds,
)
if (
not all(
math.isfinite(value) and value > 0.0
for value in control_timing
)
or self.control_min_start_seconds
> self.control_max_start_seconds
):
raise ValueError(
"Invalid control-command timing: "
f"start={self.control_min_start_seconds}.."
f"{self.control_max_start_seconds}s remaining="
f"{self.control_min_remaining_seconds}s"
)
if not self.independent_task_pools and not math.isclose(
self.interruption_ratio,
0.5,
rel_tol=0.0,
abs_tol=1e-12,
):
raise ValueError(
"Interruption v1 requires an exact 1:1 behavior ratio"
)
timing = (
self.interruption_min_start_seconds,
self.interruption_max_start_seconds,
self.interruption_min_remaining_seconds,
)
if (
not all(math.isfinite(value) and value > 0.0 for value in timing)
or self.interruption_min_start_seconds
> self.interruption_max_start_seconds
):
raise ValueError(
"Invalid interruption timing: "
f"start={self.interruption_min_start_seconds}.."
f"{self.interruption_max_start_seconds}s remaining="
f"{self.interruption_min_remaining_seconds}s"
)
if (
self.multi_turn_sessions
and self.initial_idle_max_steps * self.audio_sample_step_frames
> int(round(3.0 * self.motion_fps))
):
raise ValueError(
"Initial idle must stay within three seconds at the Qwen "
f"cadence, got steps={self.initial_idle_max_steps}"
)
if self.causal_listen_timing and self.listen_reaction_steps < 1:
raise ValueError(
"Causal listen timing requires at least one post-onset Qwen "
"reaction step"
)
if self.motion_metadata_fps <= 0:
raise ValueError(
"motion_metadata_fps must be positive, got "
f"{self.motion_metadata_fps}"
)
if self.max_voice_windows < 0:
raise ValueError(
"max_voice_windows must be non-negative, got "
f"{self.max_voice_windows}"
)
if self.max_voice_windows > 0 and self.max_voice_median_ratio < 1.0:
raise ValueError(
"max_voice_median_ratio must be at least 1 when voice "
f"filtering is enabled, got {self.max_voice_median_ratio}"
)
if EXPORTED_AUDIO_STRIDE_FRAMES % self.audio_sample_step_frames != 0:
raise ValueError(
"Runtime audio stride must divide the exported 20-frame "
f"stride, got {self.audio_sample_step_frames}"
)
self.exported_stride_scale = (
EXPORTED_AUDIO_STRIDE_FRAMES // self.audio_sample_step_frames
)
self._motion_cache: OrderedDict[str, np.ndarray] = OrderedDict()
if self.online_audio:
self.continuous_background = (
self.continuous_audio_root is not None
and self.background_assignment_path is not None
)
if not self.continuous_background:
raise ValueError(
"HFDv2 online audio requires continuous_audio_root and "
"background_assignment_path"
)
else:
self.continuous_background = (
self.background_hidden_root is not None
and self.background_assignment_path is not None
)
if (self.background_hidden_root is None) != (
self.background_assignment_path is None
):
raise ValueError(
"Continuous background requires both background_hidden_root "
"and background_assignment_path"
)
if self.action_chunk_size != 30 or self.action_exec_horizon != 20:
raise ValueError(
"Window-step HFD timing requires a 30-frame window and "
"a 20-frame execute horizon; "
f"got chunk={self.action_chunk_size}, execute={self.action_exec_horizon}"
)
if (
self.motion_fps != 50
or self.audio_sampling_rate != 16000
or not math.isclose(self.audio_window_seconds, 0.6)
or self.audio_sample_step_frames != 20
or self.action_sample_step_frames != 1
):
raise ValueError(
"Dense HFD timing requires 50 fps actions at one-frame stride "
"and 16 kHz Qwen audio at 0.6 s window / 20-frame stride; "
f"got fps={self.motion_fps}, rate={self.audio_sampling_rate}, "
f"window={self.audio_window_seconds}, audio_stride="
f"{self.audio_sample_step_frames}, action_stride="
f"{self.action_sample_step_frames}"
)
if not self.online_audio:
raise ValueError(
"Dense action sliding cannot reuse the offline hidden cache; "
"online_audio=True is required"
)
for path, label in (
(self.metadata_jsonl, "audio-command manifest"),
(self.motion_metadata_csv, "motion metadata"),
):
if not path.is_file():
raise FileNotFoundError(f"Missing {label}: {path}")
raw_audio_catalog = (
self._load_waveform_audio_catalog()
if self.online_audio
else self._load_audio_catalog()
)
(
self.audio_catalog,
self.voice_outliers_filtered,
) = self._filter_voice_duration_outliers(raw_audio_catalog)
# Session construction and its conservative length audit repeatedly
# resolve one fixed speaker identity for every turn. Index the small
# audio catalog once instead of rebuilding ``Path(...).name`` and
# linearly scanning all voice variants millions of times on every
# distributed rank. This is only a lookup acceleration: the catalog,
# selected AudioSource objects and deterministic session policy stay
# byte-for-byte identical.
self._audio_catalog_by_voice_name = {
audio_dir: {
self._voice_name(voice): voice
for voice in voices
}
for audio_dir, voices in self.audio_catalog.items()
}
if any(
len(self._audio_catalog_by_voice_name[audio_dir]) != len(voices)
for audio_dir, voices in self.audio_catalog.items()
):
raise RuntimeError(
"Duplicate speaker/language identity in HFD audio catalog"
)
if self.continuous_background:
self.background_catalog = (
self._load_background_waveform_catalog()
if self.online_audio
else self._load_background_catalog()
)
self.background_assignments = self._load_background_assignments()
self.white_noise_hidden = ()
else:
self.background_catalog = {}
self.background_assignments = {}
self.white_noise_hidden = self._load_white_noise_hidden()
self.control_command_catalog = (
self._load_control_command_catalog()
if self.independent_task_pools
and (
self.task_session_variants["emergency-stop"] > 0
or self.task_session_variants["safe-hold"] > 0
)
else {}
)
self.backchannel_catalog = (
self._load_backchannel_catalog()
if self.independent_task_pools
and self.task_session_variants["backchannel"] > 0
else {}
)
(
self.disfluency_instruction_segments,
self.disfluency_marker_catalog,
) = (
self._load_hesitation_pause_catalog()
if self.independent_task_pools
and any(
self.task_session_variants[mode] > 0
for mode in ("hesitation", "pause")
)
else ({}, {})
)
self.stationary_motion_pools = self._load_stationary_motion_pools()
self._stationary_motion_paths = frozenset(
str(source.motion_dir)
for pool in self.stationary_motion_pools.values()
for source in pool
)
self._safe_idle_motion_paths = frozenset(
str(source.motion_dir)
for source in self.stationary_motion_pools["idle"]
)
self._stationary_anchor_poses = np.empty((0, 36), dtype=np.float32)
self._stationary_joint_center = np.zeros(29, dtype=np.float32)
self._stationary_joint_scale = np.ones(29, dtype=np.float32)
self.stationary_audit_summary: dict[str, Any] = {
"enabled": False,
"accepted_clips": len(self.stationary_motion_pools["idle"]),
"rejected_clips": 0,
"anchor_frames": 0,
}
if self.safe_stationary_anchors:
self._initialise_safe_stationary_anchors()
self.records = self._load_records()
records_before_context_filter = len(self.records)
if self.max_stream_steps > 0:
self.records = [
record
for record in self.records
if self._record_max_stream_steps(record)
<= self.max_stream_steps
]
records_filtered_for_context = (
records_before_context_filter - len(self.records)
)
if limit_sources > 0:
self.records = self.records[: int(limit_sources)]
if not self.records:
raise RuntimeError(f"No full-duplex sources remain for split={self.split!r}")
sessions_filtered_for_context, sessions_filtered_for_background = (
self._prepare_session_index()
)
listen_counts = [
source.window_count
for sources in self.audio_catalog.values()
for source in sources
]
interaction_session_counts: dict[str, int] = {}
interaction_primitive_counts: dict[str, int] = {}
for session in self.sessions:
mode = str(
session[0].get("_session_interaction_mode", "turn-taking")
)
interaction_session_counts[mode] = (
interaction_session_counts.get(mode, 0) + 1
)
interaction_primitive_counts[mode] = (
interaction_primitive_counts.get(mode, 0) + len(session)
)
interaction_summary = ",".join(
f"{mode}:{interaction_session_counts[mode]}/"
f"{interaction_primitive_counts[mode]}/"
f"{100.0 * interaction_session_counts[mode] / len(self.sessions):.3f}%"
for mode in sorted(interaction_session_counts)
)
filtered_control_summary = ",".join(
f"{mode}:{count}"
for mode, count in sorted(
getattr(self, "control_task_rows_filtered", {}).items()
)
) or "none"
canonical_motion_summary = getattr(
self,
"canonical_motion_summary",
{"enabled": False},
)
print(
"[FULL DUPLEX DATA] "
f"split={self.split} rows={len(self.records):,} sessions={len(self.sessions):,} "
f"turns={self.session_min_turns}-{self.session_max_turns} "
f"wav_sources={sum(len(v) for v in self.audio_catalog.values()):,} "
f"voice_outliers_filtered={self.voice_outliers_filtered:,} "
f"translation_mismatch_rows="
f"{getattr(self, 'known_translation_mismatch_rows', 0):,} "
f"translation_mismatch_audio_dirs="
f"{getattr(self, 'known_translation_mismatch_audio_dirs', 0):,} "
f"translation_voice_variants_excluded="
f"{getattr(self, 'known_translation_voice_variants_excluded', 0):,} "
f"canonical_motion={canonical_motion_summary} "
f"listen_clips={sum(listen_counts):,} "
f"background_mode={'continuous' if self.continuous_background else 'legacy_cycle'} "
f"background_sessions={len(self.background_catalog):,} "
f"background_windows={sum(item.window_count for item in self.background_catalog.values()):,} "
f"input_layout={'raw_waveform_qwen' if self.online_audio else self.hidden_layout} "
f"cache_scope={'one_session' if self.multi_turn_sessions else 'one_source'} voice_policy="
f"{'fixed_per_session' if self.session_voice_variants > 0 else 'fixed_per_record' if self.fixed_voice_per_record else 'random'} "
f"session_voice_variants={self.session_voice_variants or 1} "
f"voice_reuse_after_coverage="
f"{getattr(self, 'voice_reuse_after_coverage', {})} "
f"interaction_sessions/primitives/ratio={interaction_summary} "
f"control_task_uses_filtered={filtered_control_summary} "
f"interruption_session_target="
f"{self.interruption_task_session_ratio:g} "
f"control_session_ratio={self.control_task_session_ratio:g} "
f"backchannel_session_target="
f"{self.backchannel_task_session_ratio:g} "
f"backchannel_start_seconds="
f"{self.backchannel_min_start_seconds:g}-"
f"{self.backchannel_max_start_seconds:g} "
f"backchannel_min_remaining_seconds="
f"{self.backchannel_min_remaining_seconds:g} "
f"control_followup_ratio={self.control_followup_ratio:g} "
f"control_post_idle_min_steps={self.control_post_idle_min_steps} "
f"control_post_idle_max_steps={self.control_post_idle_max_steps} "
f"control_post_idle_sampling={self.control_post_idle_sampling} "
f"control_start_seconds="
f"{self.control_min_start_seconds:g}-"
f"{self.control_max_start_seconds:g} "
f"control_min_remaining_seconds="
f"{self.control_min_remaining_seconds:g} "
f"control_activity_aware="
f"{int(self.control_activity_aware_timing)} "
f"interruption_ratio="
f"{self.interruption_ratio if self.interruption_sessions else 0.0:g} "
f"interruption_start_seconds="
f"{self.interruption_min_start_seconds:g}-"
f"{self.interruption_max_start_seconds:g} "
f"interruption_min_remaining_seconds="
f"{self.interruption_min_remaining_seconds:g} "
f"idle_pool={len(self.stationary_motion_pools['idle']):,} "
f"listen_pool={len(self.stationary_motion_pools['listen']):,} "
f"safe_idle_clips="
f"{self.stationary_audit_summary['accepted_clips']:,}/"
f"{len(self.stationary_motion_pools['idle']):,} "
f"safe_idle_anchor_frames="
f"{self.stationary_audit_summary['anchor_frames']:,} "
f"long_session_drop_ratio={self.long_session_drop_ratio:g} "
f"long_session_filter={self.long_session_filter_summary or 'disabled'} "
f"dynamic_task_entry_blend={int(self.dynamic_task_entry_blend)} "
f"initial_idle_steps=uniform[0,{self.initial_idle_max_steps}]",
f"max_stream_steps={self.max_stream_steps or 'unbounded'} "
f"context_filtered_rows={records_filtered_for_context:,} "
f"context_filtered_sessions={sessions_filtered_for_context:,} "
f"background_filtered_sessions={sessions_filtered_for_background:,} "
f"unsafe_turn_rows_filtered={getattr(self, 'unsafe_turn_rows_filtered', 0):,} "
f"turn_transition_frames={self.turn_transition_frames} "
f"causal_listen_timing={int(self.causal_listen_timing)} "
f"listen_reaction_steps={self.listen_reaction_steps}",
flush=True,
)
def _prepare_session_index(self):
from psi.data.seed.startup_index_cache import cached_index
# Hash the *loaded, validated contents*, not just manifest names/mtime.
# This includes every constructor option, records, waveform geometry,
# task inventories and safe idle anchors. Augmentation still runs in
# __getitem__; no RNG, audio, motion item or Qwen KV is persisted here.
inputs = {key: value for key, value in vars(self).items()
if key != "_motion_cache"}
before = set(vars(self))
def build():
stats = self._build_session_index()
self._estimated_stream_lengths = tuple(
self._estimate_stream_steps(index) for index in range(len(self))
)
state = {key: value for key, value in vars(self).items()
if key not in before and key != "_session_length_cache"}
return state, stats
state, stats = cached_index(
f"audio-sessions-{self.split}", inputs, build,
code_paths=(Path(__file__),),
)
vars(self).update(state)
# IDs of tuples are process-local and must never survive serialization.
self._session_length_cache = {}
if len(self._estimated_stream_lengths) != len(self):
raise RuntimeError("Startup session index/length count mismatch")
return stats
def _build_session_index(self):
self.sessions = self._compose_sessions(self.records)
self._session_length_cache = {}
sessions_before_context_filter = len(self.sessions)
if self.multi_turn_sessions and self.max_stream_steps > 0:
self.sessions = tuple(
session for session in self.sessions
if self._cached_session_max_stream_steps(session) <= self.max_stream_steps
)
context_filtered = sessions_before_context_filter - len(self.sessions)
sessions_before_background_filter = len(self.sessions)
max_background_windows = max(
(int(source.window_count) for source in self.background_catalog.values()
if isinstance(source, BackgroundWaveformSource)), default=0,
)
if self.multi_turn_sessions and max_background_windows > 0:
self.sessions = tuple(
session for session in self.sessions
if self._cached_session_max_stream_steps(session) <= max_background_windows
)
background_filtered = sessions_before_background_filter - len(self.sessions)
if (self.multi_turn_sessions and self.independent_task_pools and (
self.control_task_session_ratio > 0.0
or self.interruption_task_session_ratio > 0.0
or self.backchannel_task_session_ratio > 0.0
or self.hesitation_task_window_ratio > 0.0
or self.pause_task_window_ratio > 0.0
)):
if (self.backchannel_task_session_ratio > 0.0
or self.hesitation_task_window_ratio > 0.0
or self.pause_task_window_ratio > 0.0):
self.sessions = self._balance_backchannel_task_sessions(self.sessions)
else:
self.sessions = self._balance_control_task_sessions(self.sessions)
self.long_session_filter_summary = {}
if self.long_session_drop_ratio > 0.0:
self.sessions = self._drop_longest_sessions_by_mode(self.sessions)
if not self.sessions:
raise RuntimeError(f"No full-duplex sessions remain for split={self.split!r}")
return context_filtered, background_filtered
@property
def hidden_index_path(self) -> Path:
return (
self.hidden_root
/ "audio"
/ self.source_name
/ self.task_name
/ "shards"
/ "index.rank00.sqlite"
)
@property
def source_hidden_index_paths(self) -> tuple[Path, ...]:
source_dir = (
self.hidden_root
/ "audio"
/ self.source_name
/ self.task_name
/ "sources"
)
return tuple(sorted(source_dir.glob("index.ws*.rank*.sqlite")))
@property
def audio_window_samples(self) -> int:
return int(round(self.audio_window_seconds * self.audio_sampling_rate))
@property
def audio_stride_samples(self) -> int:
return int(
round(
self.audio_sample_step_frames
/ float(self.motion_fps)
* self.audio_sampling_rate
)
)
def _runtime_listen_window_count(self, speech_samples: int) -> int:
prefix_samples = self.audio_window_samples - self.audio_stride_samples
return int(
math.ceil(
(prefix_samples + int(speech_samples))
/ float(self.audio_stride_samples)
)
)
def _filter_voice_duration_outliers(
self,
catalog: dict[str, tuple[AudioSource, ...]],
) -> tuple[dict[str, tuple[AudioSource, ...]], int]:
"""Remove broken TTS duration outliers before sampling and grouping.
Every command normally has several renditions with similar duration.
A small number of generated WAVs contain tens of seconds of unrelated
audio. Those sources otherwise dominate hundreds of dense optimizer
steps and leave the other GPUs executing padded rows. Keep a generous
absolute cap and a per-command robust cap; retain the shortest voice as
a defensive fallback if a custom corpus contains only long renditions.
"""
max_windows = int(getattr(self, "max_voice_windows", 0))
if max_windows <= 0:
return dict(catalog), 0
median_ratio = float(self.max_voice_median_ratio)
output: dict[str, tuple[AudioSource, ...]] = {}
filtered_count = 0
for audio_dir, voices in catalog.items():
if not voices:
continue
median_windows = float(
np.median([voice.window_count for voice in voices])
)
relative_floor = min(max_windows, 15)
directory_cap = min(
max_windows,
max(
relative_floor,
int(math.ceil(median_windows * median_ratio)),
),
)
retained = tuple(
voice
for voice in voices
if voice.window_count <= directory_cap
)
if not retained:
retained = (min(voices, key=lambda voice: voice.window_count),)
output[audio_dir] = retained
filtered_count += len(voices) - len(retained)
return output, filtered_count
def _load_waveform_audio_catalog(
self,
) -> dict[str, tuple[AudioSource, ...]]:
"""Index the exact mixed WAVs without touching cached Qwen hidden data."""
if self.background_assignment_path is None:
raise ValueError("background_assignment_path is not configured")
by_directory: dict[str, list[AudioSource]] = {}
with self.background_assignment_path.open(
"r",
encoding="utf-8",
) as handle:
for line in handle:
if not line.strip():
continue
row = json.loads(line)
if row.get("format") != "hfd_continuous_background_assignment_v1":
raise RuntimeError(
"Unsupported HFDv2 waveform assignment row: "
f"{row.get('source_key')}"
)
source_key = str(row["source_key"])
waveform_path = relocate_hfd_storage_path(
str(row["mixed_audio_path"])
)
exported_count = int(row["listen_window_count"])
speech_samples = int(row.get("speech_samples", 0))
count = self._runtime_listen_window_count(speech_samples)
if count <= 0:
raise RuntimeError(
"Mixed waveform/window assignment mismatch: "
f"source={source_key} exported_count={exported_count} "
f"speech_samples={speech_samples}"
)
# Do not open all 41,694 small WAVs on every distributed rank
# during dataset construction. The production marker binds the
# complete assignment file, while the exact PCM geometry is
# validated by _read_pcm16_wave_segment when a selected source
# is first consumed. This changes only fail timing for a broken
# file and removes hundreds of thousands of CPFS metadata I/Os
# from eight-rank startup.
audio_dir = Path(source_key).parent.as_posix()
by_directory.setdefault(audio_dir, []).append(
AudioSource(
source_key=source_key,
clips=(),
waveform_path=waveform_path,
waveform_count=count,
)
)
if not by_directory:
raise RuntimeError(
f"No online mixed waveforms in {self.background_assignment_path}"
)
self.hidden_layout = "disabled_online_qwen"
return {
key: tuple(sorted(values, key=lambda item: item.source_key))
for key, values in by_directory.items()
}
def _load_background_waveform_catalog(
self,
) -> dict[str, BackgroundWaveformSource]:
if self.continuous_audio_root is None:
raise ValueError("continuous_audio_root is not configured")
manifest_path = (
self.continuous_audio_root
/ "manifests"
/ "background_sessions.jsonl"
)
audio_root = self.continuous_audio_root / "background"
if not manifest_path.is_file():
raise FileNotFoundError(
f"Missing online background manifest: {manifest_path}"
)
output: dict[str, BackgroundWaveformSource] = {}
with manifest_path.open("r", encoding="utf-8") as handle:
for line in handle:
if not line.strip():
continue
row = json.loads(line)
source_key = str(row["filename"])
path = relocate_hfd_storage_path(str(row["audio_path"]))
waveform_path = path if path.is_absolute() else audio_root / path
samples = _pcm16_wave_num_samples(
waveform_path,
self.audio_sampling_rate,
)
count = (
0
if samples < self.audio_window_samples
else (
samples - self.audio_window_samples
)
// self.audio_stride_samples
+ 1
)
if count <= 0:
raise RuntimeError(
f"Background waveform is too short: {waveform_path}"
)
output[source_key] = BackgroundWaveformSource(
source_key=source_key,
waveform_path=waveform_path,
window_count=count,
)
if not output:
raise RuntimeError(
f"No HFDv2 online background sessions in {manifest_path}"
)
return output
def _load_audio_catalog(self) -> dict[str, tuple[AudioSource, ...]]:
source_index_paths = self.source_hidden_index_paths
if source_index_paths:
self.hidden_layout = "source"
return self._load_source_audio_catalog(source_index_paths)
self.hidden_layout = "legacy_shard"
index_path = self.hidden_index_path
if not index_path.is_file():
raise FileNotFoundError(f"Missing HFD audio-command SQLite index: {index_path}")
conn = sqlite3.connect(f"file:{index_path}?mode=ro", uri=True)
try:
columns = [row[1] for row in conn.execute("PRAGMA table_info(samples)")]
if not {"key", "shard", "offset", "length"}.issubset(columns):
raise RuntimeError(
f"Expected the project-local fallback SQLite schema at {index_path}; columns={columns}"
)
rows = conn.execute(
"SELECT key, shard, offset, length FROM samples ORDER BY key"
).fetchall()
finally:
conn.close()
grouped: dict[str, list[HiddenSlice]] = {}
pattern = re.compile(r"^(.*)\.f(\d{8})\.safetensors$")
for key, shard, offset, length in rows:
match = pattern.match(str(key))
if match is None:
continue
source_key = match.group(1)
frame_id = int(match.group(2))
shard_path = (index_path.parent / str(shard)).resolve()
grouped.setdefault(source_key, []).append(
HiddenSlice(frame_id, shard_path, int(offset), int(length))
)
by_directory: dict[str, list[AudioSource]] = {}
for source_key, clips in grouped.items():
clips.sort(key=lambda item: item.frame_id)
frame_ids = [item.frame_id for item in clips]
# A source is valid only when it begins at zero and is a contiguous
# 20-motion-frame stream. This prevents a missing hidden slice
# from silently changing the spoken command history.
if not frame_ids or frame_ids[0] != 0:
continue
if frame_ids != list(range(0, len(frame_ids) * 20, 20)):
continue
audio_dir = Path(source_key).parent.as_posix()
by_directory.setdefault(audio_dir, []).append(
AudioSource(source_key=source_key, clips=tuple(clips))
)
return {
key: tuple(sorted(value, key=lambda source: source.source_key))
for key, value in by_directory.items()
}
def _load_source_audio_catalog(
self,
index_paths: tuple[Path, ...],
) -> dict[str, tuple[AudioSource, ...]]:
source_rows: dict[str, tuple[Path, int]] = {}
required_columns = {
"key",
"path",
"count",
"window_seconds",
"step_frames",
"motion_fps",
"sampling_rate",
}
for index_path in index_paths:
conn = sqlite3.connect(f"file:{index_path}?mode=ro", uri=True)
try:
columns = {
str(row[1])
for row in conn.execute("PRAGMA table_info(sources)")
}
if not required_columns.issubset(columns):
raise RuntimeError(
f"Invalid HFD source index schema at {index_path}; "
f"columns={sorted(columns)}"
)
rows = conn.execute(
"SELECT key, path, count, window_seconds, step_frames, "
"motion_fps, sampling_rate FROM sources ORDER BY key"
).fetchall()
finally:
conn.close()
for (
source_key,
relative_path,
count,
window_seconds,
step_frames,
motion_fps,
sampling_rate,
) in rows:
if (
int(count) <= 0
or not math.isclose(
float(window_seconds),
self.audio_window_seconds,
rel_tol=0.0,
abs_tol=1e-9,
)
or int(step_frames) != self.audio_sample_step_frames
or int(motion_fps) != self.motion_fps
or int(sampling_rate) != self.audio_sampling_rate
):
raise RuntimeError(
"HFD source index timing mismatch at "
f"{index_path}: key={source_key}, count={count}, "
f"window={window_seconds}, step={step_frames}, "
f"fps={motion_fps}, rate={sampling_rate}"
)
source_path = index_path.parent / str(relative_path)
row_value = (source_path, int(count))
previous = source_rows.get(str(source_key))
if previous is not None and previous != row_value:
raise RuntimeError(
f"Conflicting duplicate HFD source index row: "
f"{source_key}: {previous} != {row_value}"
)
source_rows[str(source_key)] = row_value
by_directory: dict[str, list[AudioSource]] = {}
for source_key, (source_path, count) in source_rows.items():
audio_dir = Path(source_key).parent.as_posix()
by_directory.setdefault(audio_dir, []).append(
AudioSource(
source_key=source_key,
clips=(),
packed_path=source_path,
packed_count=count,
)
)
if not by_directory:
raise RuntimeError(
f"No HFD source rows found in indexes: {index_paths}"
)
return {
key: tuple(sorted(value, key=lambda source: source.source_key))
for key, value in by_directory.items()
}
def _load_white_noise_hidden(self) -> tuple[torch.Tensor, ...]:
if self.white_noise_hidden_dir is None:
raise ValueError(
"Legacy cyclic background requires white_noise_hidden_dir"
)
paths = sorted(self.white_noise_hidden_dir.rglob("*.safetensors"))
hidden: list[torch.Tensor] = []
for path in paths:
tensors = load_file(str(path), device="cpu")
value = tensors.get("hidden_states")
if value is None:
continue
if value.ndim != 2:
raise ValueError(f"White-noise hidden must be (tokens, dim), got {value.shape} at {path}")
hidden.append(value.contiguous())
if not hidden:
raise FileNotFoundError(
"No white-noise hidden states found. Run the white-noise preparation step first: "
f"{self.white_noise_hidden_dir}"
)
return tuple(hidden)
def _load_background_catalog(
self,
) -> dict[str, BackgroundHiddenSource]:
if self.background_hidden_root is None:
raise ValueError("background_hidden_root is not configured")
source_dir = (
self.background_hidden_root
/ "audio"
/ self.background_source_name
/ self.background_task_name
/ "sources"
)
rows: dict[str, tuple[Path, int]] = {}
for index_path in sorted(source_dir.glob("index.ws*.rank*.sqlite")):
connection = sqlite3.connect(
f"file:{index_path}?mode=ro",
uri=True,
)
try:
values = connection.execute(
"SELECT key, path, count, window_seconds, step_frames, "
"motion_fps, sampling_rate FROM sources ORDER BY key"
).fetchall()
finally:
connection.close()
for (
source_key,
relative_path,
count,
window_seconds,
step_frames,
motion_fps,
sampling_rate,
) in values:
if (
int(count) <= 0
or not math.isclose(
float(window_seconds),
self.audio_window_seconds,
rel_tol=0.0,
abs_tol=1e-9,
)
or int(step_frames) != self.audio_sample_step_frames
or int(motion_fps) != self.motion_fps
or int(sampling_rate) != self.audio_sampling_rate
):
raise RuntimeError(
"Continuous background timing mismatch at "
f"{index_path}: key={source_key}"
)
row_value = (
index_path.parent / str(relative_path),
int(count),
)
previous = rows.get(str(source_key))
if previous is not None and previous != row_value:
raise RuntimeError(
"Conflicting duplicate continuous-background index row: "
f"{source_key}: {previous} != {row_value}"
)
rows[str(source_key)] = row_value
result: dict[str, BackgroundHiddenSource] = {}
for source_key, (source_path, count) in rows.items():
if not source_path.is_file():
raise FileNotFoundError(
f"Missing continuous background hidden: {source_path}"
)
with safe_open(
str(source_path),
framework="pt",
device="cpu",
) as reader:
if not {"hidden_states", "offsets", "frame_ids"}.issubset(
reader.keys()
):
raise RuntimeError(
f"Incomplete background hidden source: {source_path}"
)
offsets_tensor = reader.get_tensor("offsets").to(
dtype=torch.int64
)
frame_ids = reader.get_tensor("frame_ids").to(
dtype=torch.int64
)
hidden_shape = tuple(
int(value)
for value in reader.get_slice("hidden_states").get_shape()
)
if offsets_tensor.shape != (count + 1,):
raise RuntimeError(
f"Background offsets mismatch at {source_path}"
)
expected_frames = (
torch.arange(count, dtype=torch.int64)
* self.audio_sample_step_frames
)
if not torch.equal(frame_ids, expected_frames):
raise RuntimeError(
f"Background frame IDs are not continuous: {source_path}"
)
offsets = tuple(int(value) for value in offsets_tensor.tolist())
if offsets[0] != 0 or any(
current <= previous
for previous, current in zip(offsets, offsets[1:])
):
raise RuntimeError(
f"Invalid background offsets at {source_path}"
)
if len(hidden_shape) != 2 or offsets[-1] != hidden_shape[0]:
raise RuntimeError(
"Background hidden/offset size mismatch at "
f"{source_path}: hidden={hidden_shape} offsets_end={offsets[-1]}"
)
result[source_key] = BackgroundHiddenSource(
source_key=source_key,
packed_path=source_path,
window_count=count,
offsets=offsets,
)
if not result:
raise RuntimeError(
f"No continuous background source indexes under {source_dir}"
)
return result
def _load_background_assignments(
self,
) -> dict[str, SourceBackgroundAssignment]:
if self.background_assignment_path is None:
raise ValueError("background_assignment_path is not configured")
if not self.background_assignment_path.is_file():
raise FileNotFoundError(
f"Missing source/background assignments: "
f"{self.background_assignment_path}"
)
voices = {
voice.source_key: voice
for directory_voices in self.audio_catalog.values()
for voice in directory_voices
}
result: dict[str, SourceBackgroundAssignment] = {}
with self.background_assignment_path.open(
"r",
encoding="utf-8",
) as handle:
for line in handle:
if not line.strip():
continue
raw = json.loads(line)
source_key = str(raw.get("source_key", ""))
if raw.get("format") != "hfd_continuous_background_assignment_v1":
raise RuntimeError(
f"Unsupported background assignment for {source_key}"
)
if source_key not in voices:
continue
background_key = str(raw["background_key"])
background = self.background_catalog.get(background_key)
if background is None:
raise RuntimeError(
f"Unknown background {background_key} for {source_key}"
)
exported_listen_start = int(
raw["background_listen_start_index"]
)
exported_listen_count = int(raw["listen_window_count"])
exported_idle_max = int(raw["initial_idle_max_steps"])
exported_max_motion = int(raw["max_motion_steps"])
exported_safety = int(raw["safety_steps"])
speech_samples = int(raw.get("speech_samples", 0))
runtime_listen_count = self._runtime_listen_window_count(
speech_samples
)
listen_expansion = (
runtime_listen_count
- exported_listen_count * self.exported_stride_scale
)
assignment = SourceBackgroundAssignment(
source_key=source_key,
background_key=background_key,
listen_start_index=(
exported_listen_start * self.exported_stride_scale
),
listen_window_count=runtime_listen_count,
# Runtime background geometry is derived from the actual
# session WAV, not from a lossy integer conversion.
background_window_count=int(background.window_count),
initial_idle_max_steps=(
exported_idle_max * self.exported_stride_scale
),
max_motion_steps=(
exported_max_motion * self.exported_stride_scale
),
safety_steps=max(
exported_safety * self.exported_stride_scale
- (self.exported_stride_scale - 1)
- max(listen_expansion, 0),
0,
),
mixed_audio_path=(
relocate_hfd_storage_path(
str(raw["mixed_audio_path"])
)
if self.online_audio
else None
),
original_audio_path=(
relocate_hfd_storage_path(
str(raw["original_audio_path"])
)
if self.online_audio
else None
),
speech_samples=speech_samples,
speech_scale=float(raw.get("speech_scale", 1.0)),
)
if (
exported_listen_count <= 0
or int(raw["background_window_count"]) <= 0
or
assignment.listen_window_count
!= voices[source_key].window_count
or assignment.background_window_count
!= background.window_count
or (
not self.multi_turn_sessions
and (
assignment.initial_idle_max_steps
!= self.initial_idle_max_steps
or assignment.listen_start_index
< self.initial_idle_max_steps
)
)
):
raise RuntimeError(
f"Source/background assignment mismatch: {source_key}"
)
if (
assignment.listen_start_index
+ assignment.listen_window_count
+ assignment.max_motion_steps
+ assignment.safety_steps
> assignment.background_window_count
):
raise RuntimeError(
f"Source/background assignment exceeds one session: "
f"{source_key}"
)
result[source_key] = assignment
missing = sorted(set(voices) - set(result))
if missing:
raise RuntimeError(
"Continuous background assignments are incomplete: "
f"missing={len(missing)} first={missing[:5]}"
)
return result
def _load_control_command_catalog(
self,
) -> dict[str, dict[str, tuple[ControlCommandSource, ...]]]:
"""Load small TTS controls and verify all eight voice variants."""
path = self.control_command_manifest
if path is None or not path.is_file():
raise FileNotFoundError(f"Missing control-command manifest: {path}")
expected_voices = {
f"{gender}_{speaker}_{language}"
for gender in ("female", "male")
for speaker in ("01", "02")
for language in ("zh", "en")
}
grouped: dict[str, dict[str, list[ControlCommandSource]]] = {}
phrase_voices: dict[tuple[str, str], set[str]] = {}
with path.open("r", encoding="utf-8") as handle:
for line in handle:
if not line.strip():
continue
raw = json.loads(line)
if raw.get("format") != "hfd_control_command_tts_v1":
raise RuntimeError(
"Unsupported control-command manifest row: "
f"{raw.get('format')!r}"
)
intent = str(raw["intent"])
if intent not in {"emergency-stop", "safe-hold"}:
raise RuntimeError(f"Unknown control-command intent: {intent}")
voice_name = str(raw["voice_name"])
language = str(raw["language"])
if voice_name not in expected_voices or not voice_name.endswith(
f"_{language}"
):
raise RuntimeError(
f"Invalid command voice/language: {voice_name}/{language}"
)
waveform_path = relocate_hfd_storage_path(
str(raw["audio_path"])
)
speech_samples = int(raw["speech_samples_16k"])
if not waveform_path.is_file() or speech_samples <= 0:
raise FileNotFoundError(
f"Incomplete control-command audio: {waveform_path}"
)
source = ControlCommandSource(
intent=intent,
phrase_id=str(raw["phrase_id"]),
voice_name=voice_name,
language=language,
text=str(raw["text"]).strip(),
waveform_path=waveform_path,
speech_samples=speech_samples,
speech_scale=float(raw.get("speech_scale", 1.0)),
)
if not source.text:
raise RuntimeError(
f"Empty control-command text: {intent}/{source.phrase_id}"
)
grouped.setdefault(intent, {}).setdefault(
voice_name,
[],
).append(source)
phrase_voices.setdefault(
(intent, source.phrase_id),
set(),
).add(voice_name)
incomplete = {
key: sorted(expected_voices.difference(voices))
for key, voices in phrase_voices.items()
if voices != expected_voices
}
if incomplete:
raise RuntimeError(
"Every control-command meaning needs all eight voices: "
f"{incomplete}"
)
for intent in ("emergency-stop", "safe-hold"):
if set(grouped.get(intent, {})) != expected_voices:
raise RuntimeError(
f"Control-command intent {intent} lacks an eight-voice set"
)
return {
intent: {
voice_name: tuple(
sorted(values, key=lambda item: item.phrase_id)
)
for voice_name, values in by_voice.items()
}
for intent, by_voice in grouped.items()
}
def _load_backchannel_catalog(
self,
) -> dict[str, tuple[BackchannelSource, ...]]:
"""Load held-out short acknowledgements for the active data split.
Every phrase is rendered in all eight established session voices. The
phrase split, rather than the speaker identity, is held out so
validation tests whether the semantic behaviour generalises to unseen
acknowledgements without introducing a new-speaker confound.
"""
path = self.backchannel_command_manifest
if path is None or not path.is_file():
raise FileNotFoundError(f"Missing backchannel manifest: {path}")
expected_voices = {
f"{gender}_{speaker}_{language}"
for gender in ("female", "male")
for speaker in ("01", "02")
for language in ("zh", "en")
}
grouped: dict[str, list[BackchannelSource]] = {}
phrase_voices: dict[tuple[str, str], set[str]] = {}
phrase_splits: dict[str, str] = {}
split_categories: dict[str, set[str]] = {"train": set(), "val": set()}
with path.open("r", encoding="utf-8") as handle:
for line in handle:
if not line.strip():
continue
raw = json.loads(line)
if raw.get("format") != "hfd_backchannel_tts_v1":
raise RuntimeError(
"Unsupported backchannel manifest row: "
f"{raw.get('format')!r}"
)
category = str(raw["category"])
if category not in {"continuer", "assessment"}:
raise RuntimeError(
f"Unknown backchannel category: {category!r}"
)
phrase_id = str(raw["phrase_id"])
phrase_split = str(raw["split"])
if phrase_split not in {"train", "val"}:
raise RuntimeError(
f"Invalid backchannel phrase split: {phrase_split!r}"
)
previous_split = phrase_splits.setdefault(
phrase_id,
phrase_split,
)
if previous_split != phrase_split:
raise RuntimeError(
f"Backchannel phrase crosses splits: {phrase_id}"
)
split_categories[phrase_split].add(category)
voice_name = str(raw["voice_name"])
language = str(raw["language"])
if voice_name not in expected_voices or not voice_name.endswith(
f"_{language}"
):
raise RuntimeError(
"Invalid backchannel voice/language: "
f"{voice_name}/{language}"
)
waveform_path = relocate_hfd_storage_path(
str(raw["audio_path"])
)
speech_samples = int(raw["speech_samples_16k"])
if not waveform_path.is_file() or speech_samples <= 0:
raise FileNotFoundError(
f"Incomplete backchannel audio: {waveform_path}"
)
if speech_samples >= 2 * self.audio_sampling_rate:
raise RuntimeError(
"Backchannel utterances must stay below two seconds: "
f"{waveform_path}"
)
source = BackchannelSource(
category=category,
phrase_id=phrase_id,
split=phrase_split,
voice_name=voice_name,
language=language,
text=str(raw["text"]).strip(),
waveform_path=waveform_path,
speech_samples=speech_samples,
speech_scale=float(raw.get("speech_scale", 1.0)),
)
if not source.text:
raise RuntimeError(
f"Empty backchannel text: {source.phrase_id}"
)
phrase_voices.setdefault(
(phrase_split, phrase_id),
set(),
).add(voice_name)
if phrase_split == self.split:
grouped.setdefault(voice_name, []).append(source)
incomplete = {
key: sorted(expected_voices.difference(voices))
for key, voices in phrase_voices.items()
if voices != expected_voices
}
if incomplete:
raise RuntimeError(
"Every backchannel meaning needs all eight voices: "
f"{incomplete}"
)
if any(
categories != {"continuer", "assessment"}
for categories in split_categories.values()
):
raise RuntimeError(
"Train and validation backchannel phrases must each include "
f"continuer and assessment categories: {split_categories}"
)
if set(grouped) != expected_voices:
raise RuntimeError(
f"Backchannel split {self.split!r} lacks an eight-voice set"
)
return {
voice_name: tuple(
sorted(values, key=lambda item: item.phrase_id)
)
for voice_name, values in grouped.items()
}
def _load_hesitation_pause_catalog(
self,
) -> tuple[
dict[tuple[str, str], dict[str, DisfluencyAudioSource]],
dict[str, dict[str, tuple[DisfluencyAudioSource, ...]]],
]:
"""Load regenerated command halves and held-out hesitation markers.
Prefix/suffix rows are shared linguistic material keyed by the original
TTS directory and session voice. Filled-pause and repair-cue phrase IDs
are train/validation disjoint, preventing validation from succeeding by
memorising one fixed ``um`` or ``I mean`` waveform.
"""
path = self.hesitation_pause_manifest
if path is None or not path.is_file():
raise FileNotFoundError(f"Missing hesitation/pause manifest: {path}")
manifest_sha256 = hashlib.sha256(path.read_bytes()).hexdigest()
production_marker_path = (
path.parent / "reports" / "production_complete.json"
)
if not production_marker_path.is_file():
raise FileNotFoundError(
"Missing hesitation/pause production marker: "
f"{production_marker_path}"
)
production_marker = json.loads(
production_marker_path.read_text(encoding="utf-8")
)
if (
production_marker.get("format")
!= "hfd_hesitation_pause_production_v1"
or production_marker.get("manifest_format")
!= HESITATION_PAUSE_MANIFEST_FORMAT
or production_marker.get("manifest_sha256") != manifest_sha256
or int(production_marker.get("manifest_rows", 0)) <= 0
or int(production_marker.get("voice_count", 0)) != 8
):
raise RuntimeError(
"Invalid hesitation/pause production marker: "
f"{production_marker_path}"
)
expected_by_language = {
language: {
f"{gender}_{speaker}_{language}"
for gender in ("female", "male")
for speaker in ("01", "02")
}
for language in ("en", "zh")
}
segments: dict[
tuple[str, str], dict[str, DisfluencyAudioSource]
] = {}
markers: dict[str, dict[str, list[DisfluencyAudioSource]]] = {
voice: {"hesitation_filler": [], "repair_cue": []}
for voices in expected_by_language.values()
for voice in voices
}
segment_voice_audit: dict[tuple[str, str, str], set[str]] = {}
marker_voice_audit: dict[tuple[str, str, str], set[str]] = {}
with path.open("r", encoding="utf-8") as handle:
for line in handle:
if not line.strip():
continue
raw = json.loads(line)
if raw.get("format") != HESITATION_PAUSE_MANIFEST_FORMAT:
raise RuntimeError(
"Unsupported hesitation/pause manifest row: "
f"{raw.get('format')!r}"
)
category = str(raw.get("category", ""))
if category not in {
"instruction_segment",
"hesitation_filler",
"repair_cue",
}:
raise RuntimeError(
f"Unknown hesitation/pause category: {category!r}"
)
language = str(raw.get("language", ""))
voice_name = str(raw.get("voice_name", ""))
if (
language not in expected_by_language
or voice_name not in expected_by_language[language]
):
raise RuntimeError(
"Invalid hesitation/pause voice/language: "
f"{voice_name}/{language}"
)
waveform_path = relocate_hfd_storage_path(str(raw["audio_path"]))
speech_samples = int(raw.get("speech_samples_16k", 0))
# ``finalize`` has already opened and audited every WAV, and
# the marker above binds that complete audit to this exact
# manifest. Avoid reopening 81k small files on every DDP rank
# during startup; the selected WAV is opened and its sample
# count rechecked by ``_read_disfluency_audio`` at first use.
if speech_samples <= 0:
raise RuntimeError(
f"Invalid hesitation/pause sample count: {waveform_path}"
)
source = DisfluencyAudioSource(
category=category,
segment_id=str(raw["segment_id"]),
split=str(raw.get("split", "shared")),
voice_name=voice_name,
language=language,
text=str(raw.get("text", "")).strip(),
waveform_path=waveform_path,
speech_samples=speech_samples,
speech_scale=float(raw.get("speech_scale", 1.0)),
audio_dir=_normalise_audio_dir(str(raw.get("audio_dir", ""))),
instruction=str(raw.get("instruction", "")).strip(),
instruction_zh=str(raw.get("instruction_zh", "")).strip(),
segment_role=str(raw.get("segment_role", "")),
)
if not source.text:
raise RuntimeError(
f"Empty hesitation/pause TTS text: {source.segment_id}"
)
if category == "instruction_segment":
if (
source.split != "shared"
or source.segment_role not in {"prefix", "suffix"}
or not source.audio_dir
or not source.instruction
or not source.instruction_zh
):
raise RuntimeError(
f"Incomplete instruction segment: {source}"
)
key = (source.audio_dir, voice_name)
if source.segment_role in segments.setdefault(key, {}):
raise RuntimeError(
"Duplicate hesitation instruction segment: "
f"{key}/{source.segment_role}"
)
segments[key][source.segment_role] = source
segment_voice_audit.setdefault(
(source.audio_dir, source.segment_role, language),
set(),
).add(voice_name)
else:
if source.split not in {"train", "val"}:
raise RuntimeError(
f"Invalid marker split: {source.split!r}"
)
marker_voice_audit.setdefault(
(source.split, category, source.segment_id),
set(),
).add(voice_name)
if source.split == self.split:
markers[voice_name][category].append(source)
incomplete_segments = {
key: sorted(expected_by_language[key[2]].difference(voices))
for key, voices in segment_voice_audit.items()
if voices != expected_by_language[key[2]]
}
incomplete_markers: dict[tuple[str, str, str], list[str]] = {}
# Each marker meaning is generated only in its own language, hence four
# expected voices rather than all eight bilingual identities.
for (marker_split, category, segment_id), voices in marker_voice_audit.items():
languages = {voice.rsplit("_", 1)[-1] for voice in voices}
if len(languages) != 1:
raise RuntimeError(
f"Marker phrase crosses languages: {segment_id}/{languages}"
)
expected = expected_by_language[next(iter(languages))]
if voices != expected:
incomplete_markers[(marker_split, category, segment_id)] = sorted(
expected.difference(voices)
)
if incomplete_segments or incomplete_markers:
raise RuntimeError(
"Incomplete hesitation/pause voice coverage: "
f"segments={list(incomplete_segments.items())[:3]} "
f"markers={list(incomplete_markers.items())[:3]}"
)
malformed_pairs = {
key: sorted(by_role)
for key, by_role in segments.items()
if set(by_role) != {"prefix", "suffix"}
}
if malformed_pairs:
raise RuntimeError(
"Every splittable command/voice needs prefix and suffix: "
f"{list(malformed_pairs.items())[:3]}"
)
missing_marker_categories = {
voice: [
category
for category, values in by_category.items()
if not values
]
for voice, by_category in markers.items()
if any(not values for values in by_category.values())
}
if missing_marker_categories:
raise RuntimeError(
f"Hesitation marker split {self.split!r} is incomplete: "
f"{missing_marker_categories}"
)
return (
segments,
{
voice: {
category: tuple(
sorted(values, key=lambda item: item.segment_id)
)
for category, values in by_category.items()
}
for voice, by_category in markers.items()
},
)
def _compose_sessions(
self,
records: list[dict[str, Any]],
) -> tuple[tuple[dict[str, Any], ...], ...]:
"""Build deterministic 1--4 turn sessions for one already-split set.
Historical versions partition every primitive exactly once and retain
the per-record voice. Version 17 creates static voice variants: every
turn in one session uses the same speaker/language, and every primitive
appears once per train variant with a different voice when available.
Validation remains one deterministic voice variant. Version 20 maps
the two train variants to turn-taking/interruption. Version 21 restores
both complete turn-taking variants and builds every added task in an
independent pool, so changing one task's weight cannot remove or
reshuffle source usage from another task. Train and validation were
separated first, so neither path can cross the split.
"""
if not self.multi_turn_sessions:
return tuple((record,) for record in records)
if len(records) < self.session_min_turns:
raise RuntimeError(
"Multi-turn construction needs at least "
f"{self.session_min_turns} primitive sources, got {len(records)}"
)
split_salt = 0 if self.split == "train" else 1_000_003
legacy_rng = random.Random(self.split_seed * 97_409 + split_salt)
voice_variants = int(getattr(self, "session_voice_variants", 0))
if voice_variants <= 0:
return self._partition_session_records(records, legacy_rng)
interruption_enabled = bool(
getattr(self, "interruption_sessions", False)
)
independent_pools = bool(
getattr(self, "independent_task_pools", False)
)
if interruption_enabled and independent_pools:
# Keep the two v19 turn-taking compositions on their historical
# salts 0/1. Additional tasks live in disjoint salt ranges and use
# their own speaker bookkeeping; this is the key independence
# guarantee requested for task weighting.
task_offsets = {
"turn-taking": 0,
"interruption": 100,
"emergency-stop": 200,
"safe-hold": 300,
"backchannel": 400,
"hesitation": 500,
"pause": 600,
}
variant_plan = tuple(
(
task_offsets[mode] + variant_index,
variant_index if self.split == "train" else 0,
mode,
)
for mode in (
"turn-taking",
"interruption",
"emergency-stop",
"safe-hold",
"backchannel",
"hesitation",
"pause",
)
for variant_index in range(
int(self.task_session_variants.get(mode, 0))
if self.split == "train"
else int(self.task_session_variants.get(mode, 0) > 0)
)
)
elif interruption_enabled:
if voice_variants != 2:
raise RuntimeError(
"Interruption v1 requires exactly two session variants"
)
# Training keeps the existing two primitive uses: variant 0 is
# ordinary turn-taking and variant 1 is interruption. Validation
# evaluates both behaviors with the same deterministic voice, so
# timing is the only changed variable.
legacy_plan = (
((0, "turn-taking"), (1, "interruption"))
if self.split == "train"
else ((0, "turn-taking"), (0, "interruption"))
)
variant_plan = tuple(
(composition_variant, voice_variant, interaction_mode)
for composition_variant, (voice_variant, interaction_mode)
in enumerate(legacy_plan)
)
else:
effective_variants = voice_variants if self.split == "train" else 1
variant_plan = tuple(
(variant_index, variant_index, "turn-taking")
for variant_index in range(effective_variants)
)
if independent_pools:
used_voices_by_mode: dict[str, dict[int, set[str]]] = {
mode: {int(record["row_id"]): set() for record in records}
for mode in {item[2] for item in variant_plan}
}
else:
# Historical v17--v20 variants deliberately rotate the voice of a
# primitive across their complete shared variant plan. Preserve
# that exact behavior for old checkpoints and tests.
shared_used_voices = {
int(record["row_id"]): set() for record in records
}
used_voices_by_mode = {
mode: shared_used_voices
for mode in {item[2] for item in variant_plan}
}
sessions: list[tuple[dict[str, Any], ...]] = []
expected_uses = 0
control_filtered_uses = {
"emergency-stop": 0,
"safe-hold": 0,
}
voice_reuse_after_coverage: dict[str, int] = {
mode: 0 for mode in {item[2] for item in variant_plan}
}
task_local_session_counts: dict[str, int] = {}
for composition_variant, voice_variant, interaction_mode in variant_plan:
# One action plus Stop/backchannel/disfluency is a real event.
# Command interruption needs a SECOND action instruction, however.
minimum_turns = max(
self.session_min_turns,
2 if interaction_mode == "interruption" else 1,
)
used_voices = used_voices_by_mode[interaction_mode]
rng = random.Random(
self.split_seed * 97_409
+ split_salt
+ (composition_variant + 1) * 15_485_863
)
mode_records = (
self._select_new_task_records(records)
if interaction_mode in {"hesitation", "pause"}
else list(records)
)
shuffled = list(mode_records)
rng.shuffle(shuffled)
buckets: dict[str, list[dict[str, Any]]] = {}
assignments: dict[int, str] = {}
for record in shuffled:
row_id = int(record["row_id"])
candidates = self._session_voice_names_for_mode(
record,
interaction_mode,
)
available = (
[
name
for name in candidates
if name not in used_voices[row_id]
]
if self.split == "train"
else list(candidates)
)
if not available:
if independent_pools and candidates:
# Cover every valid rendition before reusing one when
# a task asks for more composition variants than this
# rare primitive has voices. Session speaker identity
# and independently shuffled neighbours stay intact.
available = list(candidates)
voice_reuse_after_coverage[interaction_mode] += 1
else:
raise RuntimeError(
"A primitive has fewer distinct valid voices than "
"the requested session variants: "
f"row={row_id} mode={interaction_mode} "
f"voices={candidates}"
)
chooser = random.Random(
self.split_seed * 1_000_003
+ row_id * 97_409
)
chooser.shuffle(available)
voice_name = available[0]
assignments[row_id] = voice_name
buckets.setdefault(voice_name, []).append(record)
# Tasks requiring >=2 turns cannot contain a one-row voice bucket. Move
# its row to another still-unused supported voice that already has
# at least one row. This preserves both the one-speaker session
# contract and distinct per-primitive voices across variants.
while minimum_turns > 1:
singleton_names = sorted(
name for name, bucket in buckets.items() if len(bucket) == 1
)
if not singleton_names:
break
moved = False
for voice_name in singleton_names:
bucket = buckets.get(voice_name, [])
if len(bucket) != 1:
continue
record = bucket[0]
row_id = int(record["row_id"])
alternatives = [
name
for name in self._session_voice_names_for_mode(
record,
interaction_mode,
)
if name != voice_name
and name not in used_voices[row_id]
and len(buckets.get(name, ())) >= 1
]
if not alternatives and independent_pools:
alternatives = [
name
for name in self._session_voice_names_for_mode(
record,
interaction_mode,
)
if name != voice_name
and len(buckets.get(name, ())) >= 1
]
if not alternatives:
continue
target_name = min(
alternatives,
key=lambda name: (len(buckets[name]), name),
)
buckets[voice_name] = []
buckets[target_name].append(record)
assignments[row_id] = target_name
moved = True
if not moved:
unresolved = {
name: [int(record["row_id"]) for record in bucket]
for name, bucket in buckets.items()
if len(bucket) == 1
}
raise RuntimeError(
"Cannot form a same-voice multi-turn session from the "
f"available renditions: {unresolved}"
)
for voice_name in sorted(buckets):
bucket = buckets[voice_name]
if not bucket:
continue
selected_bucket = bucket
if interaction_mode in {"emergency-stop", "safe-hold"}:
selected_bucket = [
record
for record in bucket
if self._control_motion_is_eligible(
record,
intent=interaction_mode,
voice_name=voice_name,
)
]
control_filtered_uses[interaction_mode] += (
len(bucket) - len(selected_bucket)
)
# A one-row remainder cannot form the promised 2--4-turn
# session. Drop that task-specific use only; the same
# primitive remains untouched in turn-taking and ordinary
# interruption pools.
if len(selected_bucket) < self.session_min_turns:
control_filtered_uses[interaction_mode] += len(
selected_bucket
)
continue
elif interaction_mode == "backchannel":
selected_bucket = [
record
for record in bucket
if self._backchannel_motion_is_eligible(
record,
voice_name=voice_name,
)
]
if len(selected_bucket) < self.session_min_turns:
continue
expected_uses += len(selected_bucket)
bucket_sessions = (
self._partition_interruption_session_records(
selected_bucket,
rng,
)
if interaction_mode == "interruption"
else self._partition_session_records(selected_bucket, rng)
)
for session in bucket_sessions:
task_local_index = task_local_session_counts.get(
interaction_mode,
0,
)
control_turn_index: int | None = None
control_has_followup = False
backchannel_turn_index: int | None = None
disfluency_turn_index: int | None = None
disfluency_subtype: str | None = None
disfluency_gap_steps: int | None = None
disfluency_marker_id: str | None = None
disfluency_repair_style: str | None = None
provisional_record: dict[str, Any] | None = None
if (
interaction_mode in {"emergency-stop", "safe-hold"}
and float(
getattr(self, "control_task_session_ratio", 0.0)
)
> 0.0
):
# Balanced control data has exactly one stop/hold event
# per session. A configurable subset places it before
# the final turn, leaving a real background pause and
# then an ordinary new instruction/action. The rest
# keeps the event terminal and still retains a
# background-only tail.
control_has_followup = (
len(session) > 1
and rng.random()
< float(
getattr(self, "control_followup_ratio", 0.0)
)
)
control_turn_index = (
rng.randrange(len(session) - 1)
if control_has_followup
else len(session) - 1
)
if interaction_mode == "backchannel":
backchannel_turn_index = rng.randrange(len(session))
if interaction_mode in {"hesitation", "pause"}:
disfluency_turn_index = rng.randrange(len(session))
disfluency_gap_steps = self._sample_disfluency_gap_steps(
rng
)
if interaction_mode == "pause":
disfluency_subtype = "silent-pause"
else:
disfluency_subtype = (
"filled-pause"
if rng.random() < self.hesitation_filled_ratio
else "self-repair"
)
if disfluency_subtype == "self-repair":
disfluency_repair_style = (
"implicit-restart"
if rng.random()
< self.self_repair_implicit_ratio
else "explicit-cue"
)
final_record = session[disfluency_turn_index]
final_key = (
str(final_record["instruction"]).strip(),
str(final_record["instruction_zh"]).strip(),
)
provisional_candidates = [
candidate
for candidate in selected_bucket
if (
str(candidate["instruction"]).strip(),
str(candidate["instruction_zh"]).strip(),
)
!= final_key
and (
str(candidate["audio_dir"]),
voice_name,
)
in self.disfluency_instruction_segments
]
if provisional_candidates:
provisional_record = rng.choice(
provisional_candidates
)
else:
# Extremely homogeneous synthetic fixtures
# cannot express a semantic self-repair. Keep
# the sample useful as filled hesitation
# instead of inventing a same-intent repair.
disfluency_subtype = "filled-pause"
disfluency_repair_style = None
if (
disfluency_subtype == "filled-pause"
or disfluency_repair_style == "explicit-cue"
):
marker_category = (
"hesitation_filler"
if disfluency_subtype == "filled-pause"
else "repair_cue"
)
marker_candidates = (
self.disfluency_marker_catalog.get(
voice_name,
{},
).get(marker_category, ())
)
if not marker_candidates:
raise RuntimeError(
f"No {marker_category} for split={self.split} "
f"voice={voice_name}"
)
marker_source = rng.choice(marker_candidates)
disfluency_marker_id = marker_source.segment_id
minimum_span = int(
marker_source.speech_samples
+ round(0.16 * self.audio_sampling_rate)
)
disfluency_gap_steps = max(
int(disfluency_gap_steps),
int(
math.ceil(
minimum_span
/ self.audio_stride_samples
)
),
)
if disfluency_gap_steps > 7:
raise RuntimeError(
"A hesitation marker cannot fit the audited "
f"2.8 s span: {marker_source.waveform_path}"
)
stamped: list[dict[str, Any]] = []
for record in session:
row_id = int(record["row_id"])
if assignments[row_id] != voice_name:
raise AssertionError("Session voice assignment drifted")
copied = dict(record)
copied["_session_voice_name"] = voice_name
copied["_session_voice_variant"] = voice_variant
copied["_session_interaction_mode"] = interaction_mode
if control_turn_index is not None:
copied["_session_control_turn_index"] = int(
control_turn_index
)
copied["_session_control_has_followup"] = bool(
control_has_followup
)
if backchannel_turn_index is not None:
copied["_session_backchannel_turn_index"] = int(
backchannel_turn_index
)
if disfluency_turn_index is not None:
copied["_session_disfluency_turn_index"] = int(
disfluency_turn_index
)
copied["_session_disfluency_subtype"] = str(
disfluency_subtype
)
copied["_session_disfluency_gap_steps"] = int(
disfluency_gap_steps or 1
)
copied["_session_disfluency_marker_id"] = (
disfluency_marker_id
)
copied["_session_disfluency_repair_style"] = (
disfluency_repair_style or ""
)
if provisional_record is not None:
copied["_session_provisional_audio_dir"] = str(
provisional_record["audio_dir"]
)
copied["_session_provisional_instruction"] = str(
provisional_record["instruction"]
)
copied[
"_session_provisional_instruction_zh"
] = str(provisional_record["instruction_zh"])
if independent_pools:
# Preserve v19's deterministic validation
# augmentation for the complete turn-taking pool
# even after all independent task sessions are
# mixed into one outer dataset index space.
copied["_session_pool_index"] = task_local_index
stamped.append(copied)
sessions.append(tuple(stamped))
task_local_session_counts[interaction_mode] = (
task_local_index + 1
)
if self.split == "train":
for row_id, voice_name in assignments.items():
used_voices[row_id].add(voice_name)
actual_uses = sum(len(session) for session in sessions)
if actual_uses != expected_uses:
raise AssertionError(
"Static session voice variants lost primitive rows: "
f"expected={expected_uses} actual={actual_uses}"
)
self.control_task_rows_filtered = control_filtered_uses
self.voice_reuse_after_coverage = voice_reuse_after_coverage
if interruption_enabled:
# Do not leave all turn-taking sessions before all interruption
# sessions. Evaluation intentionally consumes only a bounded
# prefix, while training groups sources by length; a deterministic
# final shuffle lets both paths observe both interaction modes
# without changing primitive usage, speakers, or session contents.
mix_rng = random.Random(
self.split_seed * 1_000_003
+ split_salt
+ 20 * 15_485_863
)
mix_rng.shuffle(sessions)
return tuple(sessions)
def _balance_control_task_sessions(
self,
sessions: tuple[tuple[dict[str, Any], ...], ...],
) -> tuple[tuple[dict[str, Any], ...], ...]:
"""Make enabled interruption/control pools exact final-session ratios.
Balancing runs only after context and background filtering, so the
printed train/validation proportions describe the data that can
actually reach a worker. Turn-taking and ordinary interruption are
never removed or reshuffled relative to one another.
"""
ratio = float(self.control_task_session_ratio)
control_modes = tuple(
mode
for mode in ("emergency-stop", "safe-hold")
if int(self.task_session_variants.get(mode, 0)) > 0
)
grouped: dict[str, list[tuple[dict[str, Any], ...]]] = {
mode: [] for mode in control_modes
}
non_control: list[tuple[dict[str, Any], ...]] = []
for session in sessions:
mode = str(
session[0].get("_session_interaction_mode", "turn-taking")
)
if mode in grouped:
grouped[mode].append(session)
else:
non_control.append(session)
interruption_ratio = float(
getattr(self, "interruption_task_session_ratio", 0.0)
)
if interruption_ratio > 0.0:
interruption_pool = [
session
for session in non_control
if str(
session[0].get(
"_session_interaction_mode",
"turn-taking",
)
)
== "interruption"
]
turn_taking_pool = [
session
for session in non_control
if str(
session[0].get(
"_session_interaction_mode",
"turn-taking",
)
)
!= "interruption"
]
control_total_ratio = ratio * len(control_modes)
turn_taking_ratio = (
1.0 - control_total_ratio - interruption_ratio
)
if turn_taking_ratio <= 0.0:
raise ValueError(
"Task ratios leave no turn-taking sessions: "
f"interruption={interruption_ratio:g} "
f"control_total={control_total_ratio:g}"
)
target_interruption = int(
round(
len(turn_taking_pool)
* interruption_ratio
/ turn_taking_ratio
)
)
if len(interruption_pool) < target_interruption:
raise RuntimeError(
"Not enough interruption sessions for requested final "
f"ratio={interruption_ratio:g}: "
f"target={target_interruption} "
f"available={len(interruption_pool)}"
)
split_salt = 0 if self.split == "train" else 1_000_003
interruption_rng = random.Random(
self.split_seed * 1_000_003
+ split_salt
+ 22 * 15_485_863
)
interruption_rng.shuffle(interruption_pool)
non_control = turn_taking_pool + interruption_pool[
:target_interruption
]
self.interruption_task_sessions_available = len(
interruption_pool
)
self.interruption_task_sessions_selected = target_interruption
denominator = 1.0 - ratio * len(control_modes)
target_per_mode = int(round(len(non_control) * ratio / denominator))
available = {mode: len(values) for mode, values in grouped.items()}
insufficient = {
mode: count
for mode, count in available.items()
if count < target_per_mode
}
if insufficient:
raise RuntimeError(
"Not enough eligible control sessions for the requested "
f"ratio={ratio:g}: target={target_per_mode} "
f"available={available}"
)
split_salt = 0 if self.split == "train" else 1_000_003
rng = random.Random(
self.split_seed * 1_000_003
+ split_salt
+ 21 * 15_485_863
)
selected = list(non_control)
for mode in control_modes:
pool = list(grouped[mode])
rng.shuffle(pool)
selected.extend(pool[:target_per_mode])
rng.shuffle(selected)
self.control_task_sessions_available = available
self.control_task_sessions_selected = {
mode: target_per_mode for mode in control_modes
}
return tuple(selected)
def _balance_backchannel_task_sessions(
self,
sessions: tuple[tuple[dict[str, Any], ...], ...],
) -> tuple[tuple[dict[str, Any], ...], ...]:
"""Retain the exact v22 selection, then append v23 listen hard cases.
The pre-backchannel v22 mixture used the complete turn-taking pool as
its fixed reference. Backchannel v1 keeps that invariant and enlarges
the other pools to the requested 60/15/10/15 final proportions. It
therefore never obtains a prettier ratio by throwing away established
turn-taking supervision.
"""
legacy_ratios = {
"interruption": float(self.interruption_task_session_ratio),
"emergency-stop": (
float(self.control_task_session_ratio)
if int(self.task_session_variants.get("emergency-stop", 0)) > 0
else 0.0
),
"safe-hold": (
float(self.control_task_session_ratio)
if int(self.task_session_variants.get("safe-hold", 0)) > 0
else 0.0
),
"backchannel": float(self.backchannel_task_session_ratio),
}
turn_ratio = 1.0 - sum(legacy_ratios.values())
if not math.isfinite(turn_ratio) or turn_ratio <= 0.0:
raise ValueError(
f"Task ratios leave no turn-taking sessions: {legacy_ratios}"
)
new_window_ratios = {
mode: float(ratio)
for mode, ratio in {
"hesitation": float(
getattr(self, "hesitation_task_window_ratio", 0.0)
),
"pause": float(
getattr(self, "pause_task_window_ratio", 0.0)
),
}.items()
if float(ratio) > 0.0
}
grouped: dict[str, list[tuple[dict[str, Any], ...]]] = {
mode: []
for mode in (
"turn-taking",
*legacy_ratios,
*new_window_ratios,
)
}
for session in sessions:
mode = str(
session[0].get("_session_interaction_mode", "turn-taking")
)
if mode not in grouped:
raise RuntimeError(
f"Unexpected task pool during backchannel balance: {mode}"
)
grouped[mode].append(session)
turn_pool = grouped["turn-taking"]
if not turn_pool:
raise RuntimeError("Backchannel mixture has no turn-taking pool")
target_total = int(round(len(turn_pool) / turn_ratio))
legacy_targets = {
mode: int(round(target_total * ratio))
for mode, ratio in legacy_ratios.items()
if ratio > 0.0
}
insufficient = {
mode: {
"target": target,
"available": len(grouped[mode]),
}
for mode, target in legacy_targets.items()
if len(grouped[mode]) < target
}
if insufficient:
raise RuntimeError(
"Not enough independent sessions for the requested "
f"backchannel mixture: {insufficient}"
)
split_salt = 0 if self.split == "train" else 1_000_003
rng = random.Random(
self.split_seed * 1_000_003
+ split_salt
+ 23 * 15_485_863
)
selected = list(turn_pool)
available: dict[str, int] = {}
selected_counts: dict[str, int] = {"turn-taking": len(turn_pool)}
for mode in legacy_ratios:
pool_values = grouped[mode]
pool = list(pool_values)
available[mode] = len(pool)
target = legacy_targets.get(mode, 0)
rng.shuffle(pool)
selected.extend(pool[:target])
selected_counts[mode] = target
legacy_selected = list(selected)
if not new_window_ratios:
# Preserve the exact v22 path. Besides avoiding unnecessary
# length materialisation, this keeps legacy-only unit fixtures
# and runs independent of the v23 timeline fields.
rng.shuffle(selected)
self.task_sessions_available = {
"turn-taking": len(turn_pool),
**available,
}
self.task_sessions_selected = selected_counts
self.task_windows_selected = {}
return tuple(selected)
legacy_window_count = sum(
self._cached_session_max_stream_steps(session)
for session in legacy_selected
)
new_ratio_sum = sum(new_window_ratios.values())
if new_ratio_sum >= 1.0:
raise ValueError(
"Hesitation/pause ratios leave no established task windows: "
f"{new_window_ratios}"
)
selected_window_counts = {
"legacy-total": int(legacy_window_count)
}
for mode, ratio in new_window_ratios.items():
pool = list(grouped[mode])
available[mode] = len(pool)
rng.shuffle(pool)
target_windows = int(
round(legacy_window_count * ratio / (1.0 - new_ratio_sum))
)
chosen: list[tuple[dict[str, Any], ...]] = []
chosen_windows = 0
for candidate in pool:
candidate_windows = self._cached_session_max_stream_steps(
candidate
)
before_error = abs(target_windows - chosen_windows)
after_error = abs(
target_windows - chosen_windows - candidate_windows
)
if chosen_windows >= target_windows:
break
if after_error > before_error and chosen:
break
chosen.append(candidate)
chosen_windows += candidate_windows
if target_windows > 0 and not chosen:
raise RuntimeError(
f"No {mode} session can satisfy target window budget "
f"{target_windows}"
)
if chosen_windows < int(round(target_windows * 0.98)):
raise RuntimeError(
f"Not enough {mode} windows for requested share: "
f"target={target_windows} selected={chosen_windows} "
f"available_sessions={len(pool)}"
)
selected.extend(chosen)
selected_counts[mode] = len(chosen)
selected_window_counts[mode] = int(chosen_windows)
rng.shuffle(selected)
self.task_sessions_available = {
"turn-taking": len(turn_pool),
**available,
}
self.task_sessions_selected = selected_counts
self.task_windows_selected = selected_window_counts
return tuple(selected)
def _drop_longest_sessions_by_mode(
self,
sessions: tuple[tuple[dict[str, Any], ...], ...],
) -> tuple[tuple[dict[str, Any], ...], ...]:
"""Remove the deterministic longest tail without changing task ratios.
Filtering is performed independently inside each interaction mode. A
global cutoff would disproportionately remove interruption or control
sessions and silently change the requested task mixture.
The stable original index breaks equal-length ties, so every rank and
every restart builds the same retained set.
"""
ratio = float(self.long_session_drop_ratio)
if ratio <= 0.0:
return sessions
grouped: dict[str, list[tuple[int, int]]] = {}
for index, session in enumerate(sessions):
mode = str(
session[0].get("_session_interaction_mode", "turn-taking")
)
grouped.setdefault(mode, []).append(
(index, self._cached_session_max_stream_steps(session))
)
dropped: set[int] = set()
summary: dict[str, dict[str, float | int]] = {}
for mode, values in sorted(grouped.items()):
drop_count = min(
int(math.ceil(len(values) * ratio)),
max(len(values) - 1, 0),
)
ordered = sorted(values, key=lambda item: (item[1], item[0]))
selected = ordered[-drop_count:] if drop_count else []
dropped.update(index for index, _ in selected)
retained_lengths = [
length for index, length in values if index not in dropped
]
summary[mode] = {
"before": len(values),
"dropped": drop_count,
"retained": len(values) - drop_count,
"shortest_dropped_steps": (
min(length for _, length in selected) if selected else 0
),
"longest_retained_steps": (
max(retained_lengths) if retained_lengths else 0
),
}
self.long_session_filter_summary = summary
return tuple(
session for index, session in enumerate(sessions) if index not in dropped
)
def _interruption_motion_is_eligible(
self,
record: dict[str, Any],
) -> bool:
"""Whether a primitive can be interrupted after 2 s with a real tail."""
required_frames = int(
math.ceil(
(
self.interruption_min_start_seconds
+ self.interruption_min_remaining_seconds
)
* self.motion_fps
)
)
return self._estimated_runtime_motion_frames(record) >= required_frames
def _partition_interruption_session_records(
self,
records: list[dict[str, Any]],
rng: random.Random,
) -> tuple[tuple[dict[str, Any], ...], ...]:
"""Put every short primitive last so all interrupted turns are valid."""
eligible = [
record for record in records if self._interruption_motion_is_eligible(record)
]
terminal_only = [
record
for record in records
if not self._interruption_motion_is_eligible(record)
]
rng.shuffle(eligible)
rng.shuffle(terminal_only)
count = len(records)
minimum_turns = max(2, self.session_min_turns)
if self.session_max_turns < minimum_turns:
raise ValueError("Command interruption requires at least two source turns")
minimum_sessions = int(math.ceil(count / self.session_max_turns))
maximum_sessions = count // minimum_turns
target_sessions = int(round(count / 2.7))
session_count = min(
max(target_sessions, minimum_sessions, len(terminal_only)),
maximum_sessions,
len(eligible),
)
if (
session_count < minimum_sessions
or session_count < len(terminal_only)
or session_count > len(eligible)
):
raise RuntimeError(
"Cannot form interruption sessions with a valid non-final "
"motion in every turn: "
f"rows={count} eligible={len(eligible)} "
f"terminal_only={len(terminal_only)}"
)
final_rows = list(terminal_only)
final_rows.extend(eligible[: session_count - len(final_rows)])
nonfinal_rows = eligible[session_count - len(terminal_only) :]
rng.shuffle(final_rows)
rng.shuffle(nonfinal_rows)
buckets: list[list[dict[str, Any]]] = [
[nonfinal_rows.pop()] for _ in range(session_count)
]
# Fill the third turn before the fourth, so 2/3-turn sessions dominate.
order = list(range(session_count))
rng.shuffle(order)
while nonfinal_rows:
progressed = False
for bucket_index in order:
if len(buckets[bucket_index]) >= self.session_max_turns - 1:
continue
buckets[bucket_index].append(nonfinal_rows.pop())
progressed = True
if not nonfinal_rows:
break
if not progressed:
raise AssertionError("Interruption partition exceeded four turns")
sessions = [
tuple([*bucket, final_row])
for bucket, final_row in zip(buckets, final_rows)
]
rng.shuffle(sessions)
if any(
not minimum_turns <= len(session) <= self.session_max_turns
or any(
not self._interruption_motion_is_eligible(record)
for record in session[:-1]
)
for session in sessions
):
raise AssertionError("Invalid interruption session partition")
return tuple(sessions)
def _partition_session_records(
self,
records: list[dict[str, Any]],
rng: random.Random,
) -> tuple[tuple[dict[str, Any], ...], ...]:
"""Partition records into valid groups while preserving legacy RNG."""
shuffled = list(records)
rng.shuffle(shuffled)
sessions: list[tuple[dict[str, Any], ...]] = []
cursor = 0
while cursor < len(shuffled):
remaining = len(shuffled) - cursor
if remaining <= self.session_max_turns:
size = remaining
else:
size = rng.randint(
self.session_min_turns,
self.session_max_turns,
)
leftover = remaining - size
if 0 < leftover < self.session_min_turns:
size = remaining - self.session_min_turns
if size < self.session_min_turns:
# Only a one-row tail can reach here. Move one row from the
# previous group while respecting the four-turn ceiling.
if not sessions or len(sessions[-1]) <= self.session_min_turns:
raise RuntimeError(
f"Cannot form valid multi-turn tail of {remaining} rows"
)
previous = list(sessions.pop())
borrowed = previous.pop()
sessions.append(tuple(previous))
sessions.append((borrowed, *shuffled[cursor:]))
cursor = len(shuffled)
break
sessions.append(tuple(shuffled[cursor : cursor + size]))
cursor += size
if any(
not self.session_min_turns <= len(session) <= self.session_max_turns
for session in sessions
):
raise AssertionError("Invalid deterministic multi-turn partition")
return tuple(sessions)
@staticmethod
def _voice_name(voice: AudioSource) -> str:
"""Return the speaker/language identity shared across command dirs."""
return Path(voice.source_key).name
def _record_voices(
self,
record: dict[str, Any],
) -> tuple[AudioSource, ...]:
"""Return only semantically valid renditions for one motion record."""
voices = self.audio_catalog[record["audio_dir"]]
if bool(record.get("_exclude_zh_voices", False)):
voices = tuple(
voice
for voice in voices
if not self._voice_name(voice).endswith("_zh")
)
if not voices:
raise RuntimeError(
"No semantically valid TTS rendition remains for "
f"row={record.get('row_id')} audio_dir={record.get('audio_dir')}"
)
return voices
def _session_voice_names(self, record: dict[str, Any]) -> tuple[str, ...]:
names = tuple(
sorted(
{
self._voice_name(voice)
for voice in self._record_voices(record)
}
)
)
if not names:
raise RuntimeError(
f"No valid voice identity for row={record.get('row_id')}"
)
return names
def _session_voice_names_for_mode(
self,
record: dict[str, Any],
interaction_mode: str,
) -> tuple[str, ...]:
names = self._session_voice_names(record)
if interaction_mode not in {"hesitation", "pause"}:
return names
supported = tuple(
name
for name in names
if (str(record["audio_dir"]), name)
in self.disfluency_instruction_segments
)
if not supported:
raise RuntimeError(
"A hesitation/pause record has no regenerated split speech: "
f"row={record.get('row_id')} audio_dir={record.get('audio_dir')}"
)
return supported
def _select_new_task_records(
self,
records: list[dict[str, Any]],
) -> list[dict[str, Any]]:
"""Cap only new-task duplicate prompts with duration-stratified rows."""
cap = (
self.new_task_max_rows_per_instruction_train
if self.split == "train"
else self.new_task_max_rows_per_instruction_val
)
grouped: dict[tuple[str, str], list[dict[str, Any]]] = {}
unsupported = 0
for record in records:
if not any(
(str(record["audio_dir"]), voice_name)
in self.disfluency_instruction_segments
for voice_name in self._session_voice_names(record)
):
unsupported += 1
continue
key = (
str(record["instruction"]).strip(),
str(record["instruction_zh"]).strip(),
)
grouped.setdefault(key, []).append(record)
selected: list[dict[str, Any]] = []
for key in sorted(grouped):
values = sorted(
grouped[key],
key=lambda item: (
int(item.get("actual_frames", 0)),
int(item["row_id"]),
),
)
if len(values) <= cap:
selected.extend(values)
continue
if cap == 1:
selected.append(values[len(values) // 2])
continue
indices = [
int(round(position * (len(values) - 1) / float(cap - 1)))
for position in range(cap)
]
if len(set(indices)) != cap:
raise AssertionError(
f"Duration-stratified record selection duplicated indices: {key}"
)
selected.extend(values[index] for index in indices)
selected.sort(key=lambda item: int(item["row_id"]))
self.new_task_record_selection_summary = {
"input_rows": len(records),
"eligible_rows": sum(len(values) for values in grouped.values()),
"selected_rows": len(selected),
"instruction_pairs": len(grouped),
"cap_per_instruction": int(cap),
"unsupported_rows": int(unsupported),
}
if len(selected) < self.session_min_turns:
raise RuntimeError(
"Too few splittable instruction rows for hesitation/pause: "
f"{self.new_task_record_selection_summary}"
)
return selected
def _voice_for_record(self, record: dict[str, Any]) -> AudioSource:
voices = self._record_voices(record)
session_voice_name = record.get("_session_voice_name")
if session_voice_name is None:
return self._select_voice(voices, row_id=int(record["row_id"]))
voice_name = str(session_voice_name)
indexed_catalog = getattr(
self,
"_audio_catalog_by_voice_name",
None,
)
selected = (
indexed_catalog.get(record["audio_dir"], {}).get(voice_name)
if indexed_catalog is not None
else next(
(
voice
for voice in voices
if self._voice_name(voice) == voice_name
),
None,
)
)
# Preserve the per-record language filter even though the production
# index contains every rendition in the directory.
if selected is None or selected not in voices:
raise RuntimeError(
"Session voice is unavailable or ambiguous for a turn: "
f"row={record.get('row_id')} voice={session_voice_name} "
f"matches={int(selected is not None and selected in voices)}"
)
return selected
def _disfluency_listen_window_count(
self,
session: tuple[dict[str, Any], ...],
turn_index: int,
voice: AudioSource,
*,
conservative: bool,
) -> int:
event_turn = session[0].get("_session_disfluency_turn_index")
if event_turn is None or int(event_turn) != int(turn_index):
return int(voice.window_count)
subtype = str(session[0].get("_session_disfluency_subtype", ""))
voice_name = self._voice_name(voice)
final_roles = self.disfluency_instruction_segments.get(
(str(session[turn_index]["audio_dir"]), voice_name)
)
if final_roles is None:
raise RuntimeError("Missing disfluency length-audit instruction segments")
stamped_gap = int(
session[0].get("_session_disfluency_gap_steps", 0)
)
if not 1 <= stamped_gap <= 7:
raise RuntimeError(
f"Missing/invalid stamped disfluency gap: {stamped_gap}"
)
# The exact gap is known at composition time. Segment and complete
# command lengths also come from the audited manifest, so conservative
# and runtime estimates are intentionally identical.
gap_steps = stamped_gap
if subtype in {"silent-pause", "filled-pause"}:
samples = (
final_roles["prefix"].speech_samples
+ gap_steps * self.audio_stride_samples
+ final_roles["suffix"].speech_samples
)
elif subtype == "self-repair":
provisional_roles = self.disfluency_instruction_segments.get(
(
str(session[0].get("_session_provisional_audio_dir", "")),
voice_name,
)
)
assignment = self.background_assignments.get(voice.source_key)
if provisional_roles is None or assignment is None:
raise RuntimeError("Incomplete self-repair length audit")
samples = (
provisional_roles["prefix"].speech_samples
+ gap_steps * self.audio_stride_samples
+ assignment.speech_samples
)
else:
raise RuntimeError(f"Unknown disfluency subtype: {subtype!r}")
return self._runtime_listen_window_count(int(samples))
def _session_max_stream_steps(
self,
session: tuple[dict[str, Any], ...],
) -> int:
"""Conservative Qwen-window count for one complete session."""
total_frames = self.initial_idle_max_steps * self.action_exec_horizon
interaction_mode = str(
session[0].get("_session_interaction_mode", "turn-taking")
)
control_turn = session[0].get("_session_control_turn_index")
control_turn = None if control_turn is None else int(control_turn)
for turn_index, record in enumerate(session):
voices = self._record_voices(record)
if record.get("_session_voice_name") is not None:
selected_voice = self._voice_for_record(record)
listen_steps = self._disfluency_listen_window_count(
session,
turn_index,
selected_voice,
conservative=True,
)
elif self.fixed_voice_per_record:
listen_steps = voices[
int(record["row_id"]) % len(voices)
].window_count
else:
listen_steps = max(voice.window_count for voice in voices)
if bool(getattr(self, "causal_listen_timing", False)):
# A clean onset cannot occur after the complete speech
# waveform. Use the selected waveform's window count as the
# conservative bound instead of treating the configured
# expected-onset budget as a fatal data constraint.
onset_bound_steps = max(
int(listen_steps),
int(self.max_speech_onset_steps),
)
total_frames += (
(
onset_bound_steps
+ self.listen_reaction_steps
)
* self.action_exec_horizon
)
entry_frames = (
self.turn_transition_frames
if bool(getattr(self, "causal_listen_timing", False))
else self.stationary_blend_frames
if turn_index == 0
else self.turn_transition_frames
)
total_frames += listen_steps * self.action_exec_horizon
total_frames += self._estimated_runtime_motion_frames(record)
if not bool(getattr(self, "dynamic_task_entry_blend", False)):
total_frames += max(entry_frames - 1, 1)
# Every completed task settles into a new upright anchor. Up to
# 19 extra frames align the next control transition to Qwen's
# 20-frame cadence.
total_frames += self.turn_transition_frames - 1
total_frames += self.action_exec_horizon - 1
if turn_index + 1 < len(session):
# Stop-persistence v22 deliberately exposes a longer silent
# background horizon than ordinary turn-taking gaps.
gap_steps = (
self.control_post_idle_max_steps
if interaction_mode == "emergency-stop"
and turn_index == control_turn
and self.control_post_idle_max_steps_explicit
else 10
)
total_frames += gap_steps * self.action_exec_horizon
else:
terminal_steps = (
self.control_post_idle_max_steps
if interaction_mode == "emergency-stop"
and turn_index == control_turn
and self.control_post_idle_max_steps_explicit
else self.initial_idle_max_steps
)
total_frames += (
terminal_steps * self.action_exec_horizon
)
valid_starts = max(total_frames - self.prefix_frames - 1, 0)
return int(math.ceil(valid_starts / self.action_exec_horizon))
def _cached_session_max_stream_steps(
self,
session: tuple[dict[str, Any], ...],
) -> int:
"""Reuse one conservative session-length calculation across filters."""
cache = getattr(self, "_session_length_cache", None)
if cache is None:
return self._session_max_stream_steps(session)
key = id(session)
value = cache.get(key)
if value is None:
value = int(self._session_max_stream_steps(session))
cache[key] = value
return value
def _record_max_stream_steps(self, record: dict[str, Any]) -> int:
"""Worst-case source length used to keep complete Qwen KV in context."""
voices = self._record_voices(record)
if int(getattr(self, "session_voice_variants", 0)) > 0:
# The record is filtered before static sessions choose a voice.
# Guard against every valid rendition so no later variant can
# exceed the complete-Qwen-context limit.
listen_steps = max(voice.window_count for voice in voices)
elif self.fixed_voice_per_record:
listen_steps = voices[int(record["row_id"]) % len(voices)].window_count
else:
listen_steps = max(voice.window_count for voice in voices)
task_frames = self._estimated_runtime_motion_frames(record)
if not bool(getattr(self, "dynamic_task_entry_blend", False)):
task_frames += max(self.stationary_blend_frames - 1, 1)
# Match v26's absolute-action index: a sample at state ``t`` targets
# frames ``t+1:t+31``. Keep enough starts for the largest RTC delay
# (10 clean frames) to retain at least one real supervised target.
dense_motion_starts = max(
task_frames - self.prefix_frames - 1,
0,
)
motion_steps = int(
math.ceil(dense_motion_starts / self.action_exec_horizon)
)
if self.max_motion_steps > 0:
motion_steps = min(motion_steps, self.max_motion_steps)
return int(self.initial_idle_max_steps + listen_steps + motion_steps)
def _estimated_runtime_motion_frames(self, record: dict[str, Any]) -> int:
"""Convert 120 Hz metadata lengths to the 50 Hz exported CSV length."""
declared_frames = max(int(record.get("declared_frames", 0)), 1)
metadata_fps = max(int(getattr(self, "motion_metadata_fps", 120)), 1)
return max(
int(round(declared_frames * self.motion_fps / metadata_fps)),
1,
)
def _load_background_window(
self,
source_key: str,
window_index: int,
) -> torch.Tensor:
source = self.background_catalog[source_key]
index = int(window_index)
if index < 0 or index >= source.window_count:
raise IndexError(
f"Background window is outside one continuous session: "
f"key={source_key} index={index} count={source.window_count}"
)
start = source.offsets[index]
end = source.offsets[index + 1]
return load_hidden_states_slice(
source.packed_path,
start,
end - start,
).contiguous()
def _load_background_window_block(
self,
source_key: str,
start_index: int,
count: int,
) -> tuple[torch.Tensor, ...]:
"""Read adjacent windows with one safetensors slice and one CPFS open."""
source = self.background_catalog[source_key]
start_index = int(start_index)
count = int(count)
if count <= 0:
return ()
end_index = start_index + count
if start_index < 0 or end_index > source.window_count:
raise IndexError(
"Background block is outside one continuous session: "
f"key={source_key} start={start_index} count={count} "
f"windows={source.window_count}"
)
flat_start = source.offsets[start_index]
flat_end = source.offsets[end_index]
flat = load_hidden_states_slice(
source.packed_path,
flat_start,
flat_end - flat_start,
)
return tuple(
flat[
source.offsets[index] - flat_start :
source.offsets[index + 1] - flat_start
]
for index in range(start_index, end_index)
)
def _load_waveform_window_block(
self,
waveform_path: Path,
*,
start_index: int,
count: int,
) -> tuple[np.ndarray, ...]:
"""Read adjacent 0.6 s windows using one seek/read operation."""
start_index = int(start_index)
count = int(count)
if start_index < 0 or count < 0:
raise ValueError(
f"Invalid waveform window block: start={start_index}, count={count}"
)
if count == 0:
return ()
flat = _read_pcm16_wave_segment(
waveform_path,
start_sample=start_index * self.audio_stride_samples,
sample_count=(
(count - 1) * self.audio_stride_samples
+ self.audio_window_samples
),
expected_rate=self.audio_sampling_rate,
)
return tuple(
flat[
offset * self.audio_stride_samples :
offset * self.audio_stride_samples
+ self.audio_window_samples
]
for offset in range(count)
)
def _load_background_waveform_window_block(
self,
source_key: str,
start_index: int,
count: int,
) -> tuple[np.ndarray, ...]:
source = self.background_catalog[source_key]
if not isinstance(source, BackgroundWaveformSource):
raise TypeError(
"Online HFDv2 expected a BackgroundWaveformSource, got "
f"{type(source).__name__}"
)
end_index = int(start_index) + int(count)
if int(start_index) < 0 or end_index > source.window_count:
raise IndexError(
"Background waveform block is outside one continuous session: "
f"key={source_key} start={start_index} count={count} "
f"windows={source.window_count}"
)
return self._load_waveform_window_block(
source.waveform_path,
start_index=int(start_index),
count=int(count),
)
def _load_mixed_waveform_window_block(
self,
voice: AudioSource,
assignment: SourceBackgroundAssignment,
) -> tuple[np.ndarray, ...]:
"""Remix the assigned speech/background with the deployment RTC overlap."""
if voice.waveform_path is None or assignment.original_audio_path is None:
raise RuntimeError(
f"Online voice has no complete waveform assignment: {voice.source_key}"
)
background = self.background_catalog[assignment.background_key]
if not isinstance(background, BackgroundWaveformSource):
raise TypeError(
"Stride-1 mixed audio requires a waveform background source"
)
count = int(assignment.listen_window_count)
flat_count = (
(count - 1) * self.audio_stride_samples
+ self.audio_window_samples
)
flat = _read_pcm16_wave_segment(
background.waveform_path,
start_sample=(
int(assignment.listen_start_index)
* self.audio_stride_samples
),
sample_count=flat_count,
expected_rate=self.audio_sampling_rate,
)
speech = _read_resampled_speech_float(
assignment.original_audio_path,
self.audio_sampling_rate,
)
speech_samples = int(speech.size)
if speech_samples != int(assignment.speech_samples):
raise RuntimeError(
"Speech waveform changed after background assignment: "
f"source={voice.source_key} actual={speech_samples} "
f"assigned={assignment.speech_samples}"
)
speech_start = self.audio_window_samples - self.audio_stride_samples
speech_end = speech_start + speech_samples
if speech_end > flat_count:
raise RuntimeError(
f"Stride-1 speech reservation is too short: {voice.source_key}"
)
flat[speech_start:speech_end] += (
speech * np.float32(assignment.speech_scale)
)
if float(np.max(np.abs(flat), initial=0.0)) >= 1.0:
raise RuntimeError(
f"Stride-1 speech/background mix clips: {voice.source_key}"
)
flat = (
np.rint(flat * 32768.0)
.clip(-32768, 32767)
.astype("<i2")
.astype(np.float32)
/ np.float32(32768.0)
)
return tuple(
flat[
offset * self.audio_stride_samples :
offset * self.audio_stride_samples
+ self.audio_window_samples
]
for offset in range(count)
)
def _load_stationary_motion_pools(
self,
) -> dict[str, tuple[StationaryMotionSource, ...]]:
"""Build audited standing-motion pools from exact SEED metadata labels.
Keyword matching is intentionally forbidden here: many unrelated
SEED descriptions merely say that an action *ends* idle, and several
filenames containing ``idle`` are handstands, crouches or injured
poses. Exact content labels plus semantic metadata keep those motions
out of the training targets.
"""
columns = {
"filename",
"move_g1_path",
"move_duration_frames",
"content_name",
"content_short_description",
"content_natural_desc_1",
"content_natural_desc_4",
"content_body_position",
"content_props",
"is_mirror",
}
table = pd.read_csv(
self.motion_metadata_csv,
usecols=lambda name: name in columns,
)
pools: dict[str, list[StationaryMotionSource]] = {
"idle": [],
"listen": [],
}
for row in table.itertuples(index=False):
mirror_value = getattr(row, "is_mirror", False)
if not pd.isna(mirror_value) and bool(mirror_value):
continue
content_name = str(getattr(row, "content_name", "")).strip()
if content_name in IDLE_MOTION_CONTENT_NAMES:
phase = "idle"
elif content_name in LISTEN_MOTION_CONTENT_NAMES:
phase = "listen"
else:
continue
body_position = str(
getattr(row, "content_body_position", "")
).strip().lower()
props = str(getattr(row, "content_props", "0")).strip().lower()
descriptions = " ".join(
str(getattr(row, key, "")).strip().lower()
for key in (
"content_short_description",
"content_natural_desc_1",
"content_natural_desc_4",
)
)
if body_position != "standing" or props not in {"", "0", "0.0", "nan"}:
continue
if phase == "idle" and not (
"neutral" in descriptions
and ("not moving" in descriptions or "standing idle" in descriptions)
):
continue
if phase == "listen" and not (
"listen" in descriptions
and ("ear" in descriptions or "right hand" in descriptions)
):
continue
motion_dir = _map_seed_motion_dir(
self.motion_root,
str(getattr(row, "move_g1_path", "")),
)
if not all(
(motion_dir / filename).is_file()
for filename in ("joint_pos.csv", "body_pos.csv", "body_quat.csv")
):
continue
frame_value = getattr(row, "move_duration_frames", 0)
try:
declared_frames = (
int(frame_value) if not pd.isna(frame_value) else 0
)
except (TypeError, ValueError):
declared_frames = 0
pools[phase].append(
StationaryMotionSource(
filename=str(getattr(row, "filename", "")).strip(),
motion_dir=motion_dir,
declared_frames=declared_frames,
content_name=content_name,
)
)
result = {
phase: tuple(sorted(values, key=lambda item: item.filename))
for phase, values in pools.items()
}
empty = [phase for phase, values in result.items() if not values]
if empty:
raise RuntimeError(
"No audited natural standing motions remain for phases "
f"{empty}; metadata={self.motion_metadata_csv}"
)
return result
def _load_records(self) -> list[dict[str, Any]]:
motion_table = pd.read_csv(
self.motion_metadata_csv,
usecols=lambda name: name in {
"filename",
"move_g1_path",
"move_duration_frames",
"content_body_position",
"content_name",
"content_short_description",
"content_natural_desc_1",
"content_natural_desc_4",
},
)
motion_index: dict[str, tuple[Path, int, bool, str]] = {}
unsafe_end_markers = (
"handstand",
"fall",
"lying",
"lie down",
"crawl",
"kneel",
"sitting",
"sit down",
"crouch",
"somersault",
"backflip",
"frontflip",
"injured",
)
for row in motion_table.itertuples(index=False):
filename = str(getattr(row, "filename")).strip()
move_path = str(getattr(row, "move_g1_path")).strip()
if not filename or not move_path:
continue
duration_value = getattr(row, "move_duration_frames", 0)
try:
frames = int(duration_value) if not pd.isna(duration_value) else 0
except (TypeError, ValueError):
frames = 0
body_position = str(
getattr(row, "content_body_position", "")
).strip().lower()
descriptions = " ".join(
str(getattr(row, key, "")).strip().lower()
for key in (
"content_name",
"content_short_description",
"content_natural_desc_1",
"content_natural_desc_4",
)
)
stable_turn_end = (
body_position == "standing"
and not any(
marker in descriptions
for marker in unsafe_end_markers
)
)
motion_index[filename] = (
_map_seed_motion_dir(self.motion_root, move_path),
frames,
stable_turn_end,
str(
getattr(row, "content_short_description", "")
).strip(),
)
manifest_rows: list[dict[str, Any]] = []
audio_directory_texts: dict[str, tuple[str, str]] = {}
unsafe_turn_rows_filtered = 0
known_translation_mismatch_rows = 0
known_translation_mismatch_audio_dirs: set[str] = set()
with self.metadata_jsonl.open("r", encoding="utf-8") as handle:
for row_id, line in enumerate(handle):
line = line.strip()
if not line:
continue
row = json.loads(line)
filename = str(row.get("filename", "")).strip()
audio_dir = _normalise_audio_dir(str(row.get("audio_path", "")))
instruction = str(
row.get("content_short_description", "")
).strip()
instruction_zh = str(
row.get("content_short_description_zh", "")
).strip()
if not instruction or not instruction_zh:
raise RuntimeError(
"Audio-command manifest needs exact English and Chinese "
f"TTS text: filename={filename} audio_dir={audio_dir}"
)
text_pair = (instruction, instruction_zh)
previous_pair = audio_directory_texts.setdefault(
audio_dir,
text_pair,
)
if previous_pair != text_pair:
raise RuntimeError(
"One TTS audio directory maps to conflicting text: "
f"audio_dir={audio_dir} first={previous_pair!r} "
f"current={text_pair!r}"
)
motion_info = motion_index.get(filename)
if motion_info is None or audio_dir not in self.audio_catalog:
continue
if instruction != motion_info[3]:
raise RuntimeError(
"English TTS source text differs from the paired motion "
"metadata after whitespace normalization: "
f"filename={filename} tts={instruction!r} "
f"motion={motion_info[3]!r}"
)
# The original motion label is "retro" (复古), but one
# generated Chinese TTS batch mistranslated it as spatial
# "后退" (backward). Keep the valid English renditions and
# exclude only the contradictory Chinese waveforms. This is
# intentionally a narrow, auditable rule rather than a fuzzy
# semantic-cleaning heuristic.
exclude_zh_voices = bool(
getattr(
self,
"filter_known_tts_translation_mismatches",
False,
)
and has_known_tts_translation_mismatch(
instruction,
instruction_zh,
)
)
if exclude_zh_voices:
known_translation_mismatch_rows += 1
known_translation_mismatch_audio_dirs.add(audio_dir)
if self.multi_turn_sessions and not motion_info[2]:
unsafe_turn_rows_filtered += 1
continue
manifest_rows.append(
{
"row_id": row_id,
"filename": filename,
"instruction": instruction,
"instruction_zh": instruction_zh,
"audio_dir": audio_dir,
"_exclude_zh_voices": exclude_zh_voices,
"motion_dir": motion_info[0],
"declared_frames": motion_info[1],
}
)
self.unsafe_turn_rows_filtered = unsafe_turn_rows_filtered
self.known_translation_mismatch_rows = (
known_translation_mismatch_rows
)
self.known_translation_mismatch_audio_dirs = len(
known_translation_mismatch_audio_dirs
)
self.known_translation_voice_variants_excluded = sum(
sum(
self._voice_name(voice).endswith("_zh")
for voice in self.audio_catalog.get(audio_dir, ())
)
for audio_dir in known_translation_mismatch_audio_dirs
)
self.canonical_motion_summary = {
"enabled": False,
"before": len(manifest_rows),
"after": len(manifest_rows),
"removed": 0,
}
if self.canonical_motion_manifest is not None:
selection, ambiguous_chinese = _load_canonical_motion_manifest(
self.canonical_motion_manifest
)
candidates: dict[tuple[str, str], set[str]] = {}
for row in manifest_rows:
key = (str(row["instruction"]), str(row["instruction_zh"]))
candidates.setdefault(key, set()).add(str(row["filename"]))
missing_keys = sorted(set(candidates).difference(selection))
invalid = {
key: selection[key]
for key in candidates.keys() & selection.keys()
if selection[key] not in candidates[key]
}
unused_keys = sorted(set(selection).difference(candidates))
if missing_keys or invalid or unused_keys:
raise RuntimeError(
"Canonical-motion manifest does not exactly match the "
"current eligible corpus: "
f"missing={missing_keys[:3]} invalid="
f"{list(invalid.items())[:3]} unused={unused_keys[:3]}"
)
before = len(manifest_rows)
manifest_rows = [
row
for row in manifest_rows
if selection[
(str(row["instruction"]), str(row["instruction_zh"]))
]
== str(row["filename"])
]
for row in manifest_rows:
if str(row["instruction_zh"]) in ambiguous_chinese:
row["_exclude_zh_voices"] = True
if len(manifest_rows) != len(selection):
raise RuntimeError(
"Canonical-motion filtering must leave exactly one row "
"per exact instruction pair: "
f"rows={len(manifest_rows)} selections={len(selection)}"
)
self.canonical_motion_summary = {
"enabled": True,
"before": before,
"after": len(manifest_rows),
"removed": before - len(manifest_rows),
"ambiguous_chinese_phrases_excluded": len(
ambiguous_chinese
),
"manifest": str(self.canonical_motion_manifest),
}
source_ids = [
motion_source_id(self.split_source_namespace, row["filename"])
for row in manifest_rows
]
selected = select_source_row_ids(
source_ids,
split=self.split,
val_ratio=self.val_ratio,
split_seed=self.split_seed,
)
return [manifest_rows[int(index)] for index in selected]
def _load_motion(self, path: Path) -> np.ndarray:
key = str(path)
# Task motions are unique in this corpus, so caching them only evicts
# the small idle/listen set that every source reuses. Reserve the LRU
# for those audited stationary motions.
cacheable = key in getattr(self, "_stationary_motion_paths", ())
if cacheable:
cached = self._motion_cache.get(key)
if cached is not None:
self._motion_cache.move_to_end(key)
return cached
joint = pd.read_csv(path / "joint_pos.csv").to_numpy(dtype=np.float32)
body_pos = pd.read_csv(path / "body_pos.csv").to_numpy(dtype=np.float32)
body_quat = pd.read_csv(path / "body_quat.csv").to_numpy(dtype=np.float32)
lengths = {joint.shape[0], body_pos.shape[0], body_quat.shape[0]}
if len(lengths) != 1 or joint.shape[1] != 29 or body_pos.shape[1] != 3 or body_quat.shape[1] != 4:
raise ValueError(f"Invalid 36D SEED motion at {path}")
motion = np.concatenate([joint, body_pos, body_quat], axis=1).astype(np.float32)
if cacheable and self.motion_cache_size > 0:
self._motion_cache[key] = motion
self._motion_cache.move_to_end(key)
while len(self._motion_cache) > self.motion_cache_size:
self._motion_cache.popitem(last=False)
return motion
@staticmethod
def _load_slice(item: HiddenSlice) -> torch.Tensor:
if not item.shard_path.is_file():
raise FileNotFoundError(f"Missing hidden-state shard: {item.shard_path}")
return load_hidden_states_slice(
item.shard_path, item.offset, item.length
).contiguous()
def _load_voice_hidden(
self,
voice: AudioSource,
) -> tuple[torch.Tensor, ...]:
if voice.waveform_path is not None:
raise RuntimeError(
"HFDv2 online waveform sources must not read cached hidden states"
)
if voice.packed_path is None:
return tuple(self._load_slice(clip) for clip in voice.clips)
frame_ids, hidden_windows = load_hidden_states_source(
voice.packed_path,
expected_window_seconds=self.audio_window_seconds,
expected_step_frames=self.audio_sample_step_frames,
expected_motion_fps=self.motion_fps,
expected_sampling_rate=self.audio_sampling_rate,
)
expected_frame_ids = tuple(
index * self.audio_sample_step_frames
for index in range(voice.window_count)
)
if frame_ids != expected_frame_ids:
raise RuntimeError(
f"HFD source/index frame mismatch for {voice.source_key}: "
f"file={frame_ids} index={expected_frame_ids}"
)
return hidden_windows
def _raw_step(
self,
*,
hidden_states: torch.Tensor | None = None,
audio_waveform: np.ndarray | None = None,
actions: np.ndarray,
action_mask: np.ndarray,
state: np.ndarray,
action_reference_state: np.ndarray,
smoothness_context_actions: np.ndarray,
smoothness_context_reference_state: np.ndarray,
action_control_labels: np.ndarray,
future_progress: np.ndarray,
control_progress: np.ndarray,
previous_control_label: int,
phase: str,
source_id: str,
step_index: int,
frame_id: int,
semantic_window_index: int,
action_substep: int,
semantic_frame_id: int,
record: dict[str, Any],
instruction_text: str,
update_semantic_cache: bool,
control_transition: bool,
instruction_history_texts: tuple[str, ...] = (),
) -> dict[str, Any]:
semantic_inputs = int(hidden_states is not None) + int(
audio_waveform is not None
)
if update_semantic_cache and semantic_inputs != 1:
raise ValueError(
"A Qwen-update step must contain exactly one semantic input"
)
if not update_semantic_cache and semantic_inputs != 0:
raise ValueError(
"A dense action-only step must reuse the previous Qwen "
"condition and contain no new semantic input"
)
progress_start = (
0.0
if phase in {"idle", "listen"}
else float(min(max(frame_id, 0), int(record["actual_frames"]) - 1))
/ float(max(int(record["actual_frames"]) - 1, 1))
)
if phase == "idle":
control_label = IDLE_TOKEN_ID
elif phase == "listen":
control_label = LISTEN_TOKEN_ID
elif phase == "motion":
control_label = MOTION_TOKEN_ID
else:
raise ValueError(f"Unsupported full-duplex phase: {phase!r}")
latest_instruction_target = bool(
update_semantic_cache
and control_transition
and control_label == MOTION_TOKEN_ID
)
exact_instruction_text = str(instruction_text).strip()
if latest_instruction_target and not exact_instruction_text:
raise RuntimeError(
"A listen-to-motion boundary has no latest-instruction text "
f"target: source={source_id} step={step_index}"
)
exact_instruction_history = tuple(
str(value).strip()
for value in instruction_history_texts
if str(value).strip()
)
memory_target = bool(
latest_instruction_target and exact_instruction_history
)
if memory_target:
if len(exact_instruction_history) == 1:
memory_slot = "previous"
memory_text = exact_instruction_history[-1]
elif len(exact_instruction_history) % 2 == 0:
# Third command: ask for the first turn. Fourth command: ask
# for the immediately previous turn. Across 2--4 turn
# sessions this balances long-range and local retrieval while
# adding only one extra sparse side readout per boundary.
memory_slot = "first"
memory_text = exact_instruction_history[0]
else:
memory_slot = "previous"
memory_text = exact_instruction_history[-1]
else:
memory_slot = ""
memory_text = ""
output = {
"actions": actions.astype(np.float32),
"action_mask": action_mask.astype(np.float32),
"states": state.reshape(1, -1).astype(np.float32),
"action_reference_states": action_reference_state.reshape(
1, -1
).astype(np.float32),
# The production temporal loss needs the two real action tokens
# immediately before this horizon. Keeping them source-local is
# what lets velocity/acceleration span a dense-window boundary
# instead of restarting at every 30-frame target.
"smoothness_context_actions": np.asarray(
smoothness_context_actions,
dtype=np.float32,
).reshape(2, -1),
"smoothness_context_reference_states": np.asarray(
smoothness_context_reference_state,
dtype=np.float32,
).reshape(1, -1),
# The 30-frame action horizon can cross idle/listen/motion
# boundaries. Keep the exact phase of every future frame instead
# of pretending that the scalar current-window token owns all 30.
"action_control_labels": np.asarray(
action_control_labels,
dtype=np.int64,
).reshape(self.action_chunk_size),
"progress": np.asarray(
[
progress_start,
progress_start,
1.0 - progress_start,
math.sin(2.0 * math.pi * progress_start),
math.cos(2.0 * math.pi * progress_start),
],
dtype=np.float32,
),
"future_progress": future_progress.astype(np.float32),
# Causal context for the serial token. This describes the state
# already executed immediately before the new 0.4 s decision; it
# never reads the target phase or future GT progress.
"control_progress": np.asarray(
control_progress,
dtype=np.float32,
).reshape(5),
"previous_control_label": int(previous_control_label),
"instruction": record["instruction"],
"dataset_name": "hfd-audio-command-full-duplex",
"source_id": source_id,
"source_audio_key": source_id.split("::", 1)[-1],
"phase": phase,
"control_label": control_label,
"control_token": CONTROL_TOKEN_NAMES[control_label],
"control_transition": bool(control_transition),
"update_semantic_cache": bool(update_semantic_cache),
"step_index": int(step_index),
"frame_id": int(frame_id),
"semantic_frame_id": int(semantic_frame_id),
"semantic_window_index": int(semantic_window_index),
"action_substep": int(action_substep),
"motion_name": record["filename"],
"motion_dir": str(record["motion_dir"]),
"total_frames": int(record["actual_frames"]),
# One target per turn, exposed only at the causal Qwen boundary
# where the current scalar control changes into motion. The raw
# primitive duration excludes the known stationary join.
"motion_duration_target_seconds": float(record["actual_frames"])
/ float(getattr(self, "motion_fps", 50)),
"motion_duration_target_mask": bool(
latest_instruction_target
),
# Version 18 applies teacher-forced text CE exactly once per turn,
# at the same causal boundary as duration supervision. No VAD or
# inference-time text decoder is introduced.
"latest_instruction_text": (
exact_instruction_text if latest_instruction_target else ""
),
"latest_instruction_target_mask": latest_instruction_target,
"instruction_memory_text": memory_text,
"instruction_memory_slot": memory_slot,
"instruction_memory_target_mask": memory_target,
}
if hidden_states is not None:
output["vlm_hidden_states"] = hidden_states
elif audio_waveform is not None:
waveform = np.asarray(audio_waveform, dtype=np.float32).reshape(-1)
if waveform.shape != (self.audio_window_samples,):
raise ValueError(
"Online HFDv2 audio window must be exactly "
f"{self.audio_window_samples} samples, got {waveform.shape}"
)
output["audio_waveform"] = waveform
output["audio_sampling_rate"] = int(self.audio_sampling_rate)
return output
def __len__(self) -> int:
if bool(getattr(self, "multi_turn_sessions", False)):
return len(self.sessions)
return len(self.records)
def estimated_stream_steps(self, index: int) -> int:
"""Estimate dense action optimizer steps for cross-rank load balance."""
lengths = getattr(self, "_estimated_stream_lengths", None)
if lengths is not None:
return lengths[int(index)]
return self._estimate_stream_steps(index)
def _estimate_stream_steps(self, index: int) -> int:
if bool(getattr(self, "multi_turn_sessions", False)):
session = self.sessions[int(index)]
interaction_mode = str(
session[0].get("_session_interaction_mode", "turn-taking")
)
control_turn = session[0].get("_session_control_turn_index")
control_turn = None if control_turn is None else int(control_turn)
total_frames = (
int(round(self.initial_idle_max_steps * 0.5))
* self.action_exec_horizon
)
for turn_index, record in enumerate(session):
voices = self._record_voices(record)
if record.get("_session_voice_name") is not None:
selected_voice = self._voice_for_record(record)
listen_steps = self._disfluency_listen_window_count(
session,
turn_index,
selected_voice,
conservative=False,
)
elif self.fixed_voice_per_record:
listen_steps = voices[
int(record["row_id"]) % len(voices)
].window_count
else:
listen_steps = int(
round(
sum(voice.window_count for voice in voices)
/ len(voices)
)
)
if bool(getattr(self, "causal_listen_timing", False)):
# Estimate from the selected waveform duration. The
# context filter above still uses the full conservative
# duration, so a long valid leading silence cannot escape
# initialization and crash a worker hours later.
onset_bound_steps = max(
int(listen_steps),
int(self.max_speech_onset_steps),
)
total_frames += max(
int(round(onset_bound_steps * 0.5)),
1,
) * self.action_exec_horizon
total_frames += (
self.listen_reaction_steps * self.action_exec_horizon
)
entry_frames = (
self.turn_transition_frames
if bool(getattr(self, "causal_listen_timing", False))
else self.stationary_blend_frames
if turn_index == 0
else self.turn_transition_frames
)
total_frames += listen_steps * self.action_exec_horizon
task_frames = self._estimated_runtime_motion_frames(record)
if not bool(
getattr(self, "dynamic_task_entry_blend", False)
):
task_frames += max(entry_frames - 1, 1)
if (
interaction_mode == "interruption"
and turn_index + 1 < len(session)
):
# Match the runtime truncation closely enough for the
# length-grouped sampler. The conservative context filter
# remains `_session_max_stream_steps`; this branch only
# prevents interrupted sources from being scheduled as if
# every discarded tail were still executed.
minimum = int(
math.ceil(
self.interruption_min_start_seconds
* self.motion_fps
/ self.action_exec_horizon
)
) * self.action_exec_horizon
requested_maximum = int(
math.floor(
self.interruption_max_start_seconds
* self.motion_fps
/ self.action_exec_horizon
)
) * self.action_exec_horizon
next_record = session[turn_index + 1]
next_listen_steps = self._voice_for_record(
next_record
).window_count
onset_estimate = max(
int(round(max(next_listen_steps, self.max_speech_onset_steps) * 0.5)),
1,
)
next_reaction_frames = (
onset_estimate + self.listen_reaction_steps
) * self.action_exec_horizon
remaining = max(
int(
math.ceil(
self.interruption_min_remaining_seconds
* self.motion_fps
)
),
next_reaction_frames,
)
latest = (
(task_frames - remaining) // self.action_exec_horizon
) * self.action_exec_horizon
maximum = min(requested_maximum, latest)
estimated_cut = (
minimum
if maximum <= minimum
else (
(minimum + maximum)
// (2 * self.action_exec_horizon)
)
* self.action_exec_horizon
)
total_frames += estimated_cut
continue
total_frames += task_frames
recovery_frames = self.turn_transition_frames
recovery_frames += (
-(total_frames + recovery_frames - 1)
) % self.action_exec_horizon
total_frames += recovery_frames - 1
if (
interaction_mode == "emergency-stop"
and turn_index == control_turn
and self.control_post_idle_max_steps_explicit
):
stop_idle_mean = int(
round(
(
self.control_post_idle_min_steps
+ self.control_post_idle_max_steps
)
* 0.5
)
)
total_frames += (
stop_idle_mean * self.action_exec_horizon
)
continue
if turn_index + 1 < len(session):
# Mean of 50% immediate, 35% 1..3 and 15% 4..10.
total_frames += 2 * self.action_exec_horizon
else:
terminal_mean = max(
int(round((self.initial_idle_max_steps + 1) * 0.5)),
1,
)
total_frames += terminal_mean * self.action_exec_horizon
semantic_estimate = int(
math.ceil(
max(total_frames - self.prefix_frames - 1, 0)
/ self.action_exec_horizon
)
)
if (
self.max_stream_steps > 0
and semantic_estimate > self.max_stream_steps
):
raise RuntimeError(
"Context-filtered session still estimates "
f"{semantic_estimate} semantic windows above "
f"limit={self.max_stream_steps}: index={index}"
)
return max(total_frames - self.prefix_frames - 1, 0)
record = self.records[int(index)]
voices = self._record_voices(record)
if self.fixed_voice_per_record:
listen_steps = voices[
int(record["row_id"]) % len(voices)
].window_count
else:
listen_steps = int(
round(
sum(voice.window_count for voice in voices)
/ len(voices)
)
)
idle_steps = int(round(self.initial_idle_max_steps * 0.5))
task_frames = self._estimated_runtime_motion_frames(record)
if not bool(getattr(self, "dynamic_task_entry_blend", False)):
task_frames += max(self.stationary_blend_frames - 1, 1)
# ``estimated_stream_steps`` returns dense action starts, not the
# number of semantic/Qwen windows. The final ``prefix+1`` states are
# excluded exactly as in v26's valid_sample_count(): after a maximal
# 10-frame RTC prefix there must still be one real target frame.
motion_action_steps = max(
task_frames - self.prefix_frames - 1,
0,
)
if self.max_motion_steps > 0:
motion_action_steps = min(
motion_action_steps,
self.max_motion_steps * self.action_exec_horizon,
)
motion_semantic_steps = int(
math.ceil(motion_action_steps / self.action_exec_horizon)
)
semantic_estimate = idle_steps + listen_steps + motion_semantic_steps
if self.max_stream_steps > 0 and semantic_estimate > self.max_stream_steps:
raise RuntimeError(
"Context-filtered record still estimates "
f"{semantic_estimate} semantic windows "
f"above limit={self.max_stream_steps}: index={index}"
)
return (
(idle_steps + listen_steps) * self.action_exec_horizon
+ motion_action_steps
)
def _select_voice(
self,
voices: tuple[AudioSource, ...],
*,
row_id: int,
) -> AudioSource:
if not voices:
raise RuntimeError("An audio-command record has no valid TTS voice")
if self.fixed_voice_per_record:
return voices[int(row_id) % len(voices)]
return random.choice(voices) if len(voices) > 1 else voices[0]
@staticmethod
def _stationary_motion_is_safe(motion: np.ndarray) -> bool:
"""Reject malformed or locomoting candidates even after metadata audit."""
if (
motion.ndim != 2
or motion.shape[1] != 36
or motion.shape[0] < 30
or not np.isfinite(motion).all()
):
return False
root = motion[:, 29:32]
horizontal_range = float(
np.linalg.norm(root[:, :2] - root[:1, :2], axis=-1).max()
)
height_range = float(np.ptp(root[:, 2]))
return horizontal_range <= 0.15 and height_range <= 0.08
@classmethod
def _idle_motion_is_quiet_and_continuous(cls, motion: np.ndarray) -> bool:
"""Reject jump-cut or gesture-like clips from the neutral idle pool.
Listen clips are intentionally expressive, so this stricter gate is
applied only to idle. At 50 Hz a neutral standing residual should not
jump by more than 0.08 rad in one frame, spend its upper tail above
0.03 rad/frame, or sweep a joint through more than 0.45 rad. These
limits retain subtle breathing/postural variation while rejecting the
audited A120 discontinuities and large unrelated arm gestures.
"""
if not cls._stationary_motion_is_safe(motion):
return False
joint_step = np.max(np.abs(np.diff(motion[:, :29], axis=0)), axis=1)
joint_range = np.ptp(motion[:, :29], axis=0)
return (
float(np.max(joint_step, initial=0.0)) <= 0.08
and float(np.percentile(joint_step, 95)) <= 0.03
and float(np.max(joint_range, initial=0.0)) <= 0.45
)
def _initialise_safe_stationary_anchors(self) -> None:
"""Audit neutral clips and cache quiet, upright two-support poses.
Metadata semantics alone cannot catch a motion that is perfectly
static but contains a corrupted whole-body pose. We therefore compare
every idle clip's median 29-D joint pose against the complete pool with
a median/MAD robust score. This is deliberately data-driven: no
filename, action class or manually maintained joint-angle blacklist is
used. Accepted clips then contribute only upright, low-velocity frames
whose two leg chains satisfy the same contact proxy used by TextOp.
"""
loaded: list[tuple[StationaryMotionSource, np.ndarray]] = []
representatives: list[np.ndarray] = []
continuity_rejected_names: list[str] = []
for source in self.stationary_motion_pools["idle"]:
motion = self._load_motion(source.motion_dir)
if not self._stationary_motion_is_safe(motion):
continue
if bool(
getattr(self, "stationary_idle_residual_retargeting", False)
) and not self._idle_motion_is_quiet_and_continuous(motion):
continuity_rejected_names.append(source.filename)
continue
loaded.append((source, motion))
representatives.append(np.median(motion[:, :29], axis=0))
if not loaded:
raise RuntimeError("No kinematically valid neutral idle clips remain")
representative_rows = np.stack(representatives).astype(np.float32)
center = np.median(representative_rows, axis=0)
mad = np.median(np.abs(representative_rows - center), axis=0)
scale = np.maximum(1.4826 * mad, 0.05).astype(np.float32)
clip_scores = np.sqrt(
np.mean(np.square((representative_rows - center) / scale), axis=1)
)
score_center = float(np.median(clip_scores))
score_mad = float(np.median(np.abs(clip_scores - score_center)))
# Six robust standard deviations is intentionally conservative. It
# rejects isolated structural corruption while retaining legitimate
# variations in neutral arm placement.
clip_score_limit = score_center + max(
6.0 * 1.4826 * score_mad,
1.0,
)
accepted_paths: set[str] = set()
rejected_names: list[str] = []
anchor_rows: list[np.ndarray] = []
sampling_stride = max(int(self.motion_fps) // 5, 1)
for (source, motion), clip_score in zip(loaded, clip_scores):
if float(clip_score) > clip_score_limit:
rejected_names.append(source.filename)
continue
joint_delta = np.diff(
motion[:, :29], axis=0, prepend=motion[:1, :29]
)
root_delta = np.diff(
motion[:, 29:32], axis=0, prepend=motion[:1, 29:32]
)
contacts = estimate_two_foot_contacts(joint_delta, root_delta)
quat = _normalise_motion_quaternions(motion)[:, 32:36]
upright = 1.0 - 2.0 * (
np.square(quat[:, 1]) + np.square(quat[:, 2])
)
frame_scores = np.sqrt(
np.mean(np.square((motion[:, :29] - center) / scale), axis=1)
)
low_velocity = (
np.linalg.norm(joint_delta, axis=-1) <= 0.05
) & (
np.linalg.norm(root_delta, axis=-1) <= 0.025
)
valid = (
np.all(contacts > 0.5, axis=-1)
& (upright >= 0.80)
& low_velocity
& (frame_scores <= clip_score_limit)
)
valid[0] = False # prepend-diff would make frame zero trivially static.
indices = np.flatnonzero(valid)
if indices.size == 0:
rejected_names.append(source.filename)
continue
accepted_paths.add(str(source.motion_dir))
selected = indices[::sampling_stride]
if selected[-1] != indices[-1]:
selected = np.concatenate([selected, indices[-1:]])
anchor_rows.extend(motion[selected].astype(np.float32, copy=True))
if not anchor_rows:
raise RuntimeError("Whole-body idle audit produced no safe anchor frames")
self._safe_idle_motion_paths = frozenset(accepted_paths)
self._stationary_anchor_poses = _normalise_motion_quaternions(
np.stack(anchor_rows)
)
self._stationary_joint_center = center.astype(np.float32)
self._stationary_joint_scale = scale
self.stationary_audit_summary = {
"enabled": True,
"accepted_clips": len(accepted_paths),
"rejected_clips": len(rejected_names),
"rejected_names": tuple(sorted(rejected_names)),
"continuity_rejected_clips": len(continuity_rejected_names),
"continuity_rejected_names": tuple(
sorted(continuity_rejected_names)
),
"anchor_frames": int(len(self._stationary_anchor_poses)),
"clip_score_limit": float(clip_score_limit),
"max_accepted_clip_score": float(
max(
score
for (_, _), score in zip(loaded, clip_scores)
if float(score) <= clip_score_limit
)
),
}
def _nearest_safe_stationary_anchor(
self,
current_pose: np.ndarray,
) -> tuple[np.ndarray, float]:
"""Return the audited standing pose closest in normalized joint space."""
pose = np.asarray(current_pose, dtype=np.float32).reshape(36)
anchors = np.asarray(self._stationary_anchor_poses, dtype=np.float32)
if anchors.ndim != 2 or anchors.shape[0] == 0:
raise RuntimeError("Safe stationary anchor catalog is empty")
scale = np.asarray(self._stationary_joint_scale, dtype=np.float32)
distance = np.mean(
np.square((anchors[:, :29] - pose[:29]) / scale[None, :]),
axis=1,
)
# Root height is physically coupled to the selected leg configuration;
# include it in nearest-neighbour selection, but never overwrite it.
height_scale = max(float(np.std(anchors[:, 31])), 0.05)
distance += 0.25 * np.square((anchors[:, 31] - pose[31]) / height_scale)
selected = int(np.argmin(distance))
anchor = anchors[selected].astype(np.float32, copy=True)
anchor[29:31] = pose[29:31]
return anchor, float(distance[selected])
@staticmethod
def _crop_or_resample_motion(
motion: np.ndarray,
frame_count: int,
) -> np.ndarray:
if frame_count <= 0:
return np.empty((0, 36), dtype=np.float32)
if motion.shape[0] >= frame_count:
max_start = int(motion.shape[0] - frame_count)
start = random.randint(0, max_start) if max_start > 0 else 0
return np.asarray(
motion[start : start + frame_count],
dtype=np.float32,
).copy()
source_x = np.linspace(0.0, 1.0, int(motion.shape[0]), dtype=np.float32)
target_x = np.linspace(0.0, 1.0, int(frame_count), dtype=np.float32)
values = np.stack(
[np.interp(target_x, source_x, motion[:, dim]) for dim in range(36)],
axis=-1,
).astype(np.float32)
return _normalise_motion_quaternions(values)
@staticmethod
def _listen_hold_target(motion: np.ndarray, anchor: np.ndarray) -> np.ndarray:
"""Pick an actual, settled expressive pose from an audited listen clip.
Use both arms, not a filename or a hard-coded joint angle. Retain a
recorded whole-body frame (including its matching height/legs), never
assemble a hand pose with unrelated root height. The upper excursion
quartile excludes the clip's neutral entry/exit; local velocity picks
a settled part rather than the fastest raising/lowering frame.
"""
arm_distance = np.linalg.norm(motion[:, 15:29] - anchor[None, 15:29], axis=1)
speed = np.linalg.norm(np.gradient(motion[:, :29], axis=0), axis=1)
radius = min(5, (len(motion) - 1) // 2)
smooth_speed = np.convolve(
np.pad(speed, (radius, radius), mode="edge"),
np.ones(2 * radius + 1) / (2 * radius + 1), mode="valid",
)
candidates = np.flatnonzero(arm_distance >= np.quantile(arm_distance, 0.75))
selected = candidates[np.argmin(smooth_speed[candidates])]
return motion[int(selected)].astype(np.float32, copy=True)
def _sample_stationary_sequence(
self,
phase: str,
*,
frame_count: int,
anchor_pose: np.ndarray,
blend_frames: int | None = None,
entry_pose: np.ndarray | None = None,
) -> tuple[np.ndarray, str | None]:
if frame_count <= 0:
return np.empty((0, 36), dtype=np.float32), None
pool = self.stationary_motion_pools[str(phase)]
candidates = list(pool)
if (
str(phase) == "idle"
and bool(getattr(self, "safe_stationary_anchors", False))
):
candidates = [
source
for source in candidates
if str(source.motion_dir) in self._safe_idle_motion_paths
]
random.shuffle(candidates)
selected: np.ndarray | None = None
selected_name = ""
hold_listen = str(phase) == "listen" and bool(
getattr(self, "stationary_listen_hold", False)
)
for candidate in candidates:
motion = self._load_motion(candidate.motion_dir)
if self._stationary_motion_is_safe(motion):
selected = self._crop_or_resample_motion(motion, frame_count)
if hold_listen:
# Keep the legacy crop RNG draw: enabling this target fix
# must not reshuffle later voices, backgrounds or turns.
pose = self._listen_hold_target(motion, np.asarray(anchor_pose))
selected = np.repeat(pose[None, :], frame_count, axis=0)
selected_name = candidate.filename
break
if selected is None:
raise RuntimeError(
f"All audited {phase} motions failed the kinematic safety gate"
)
anchor = np.asarray(anchor_pose, dtype=np.float32).reshape(36)
if bool(getattr(self, "stationary_heading_alignment", False)):
# Express every stationary candidate in the current robot heading.
# The old path aligned XY only, so a perfectly valid listen clip
# recorded facing another world direction taught an arbitrary turn
# (up to ~180 degrees in real validation sessions). This rigid yaw
# transform preserves local motion and the candidate's own height.
selected = _align_task_motion_to_anchor_heading(selected, anchor)
else:
# Historical v14 data contract: align only ground-plane position.
selected[:, 29:31] += anchor[29:31] - selected[0, 29:31]
if (
str(phase) == "idle"
and bool(
getattr(
self,
"stationary_idle_residual_retargeting",
False,
)
)
):
# An idle clip is a source of tiny *changes*, not a destination
# pose. Blending an arbitrary recorded standing pose against the
# current anchor can otherwise manufacture a large arm excursion:
# both endpoints are individually static, but their wrists may be
# far apart. Retarget joint/root residuals around the current
# anchor and hold its heading. This preserves breathing and small
# posture changes while removing the unrelated absolute pose.
selected_first = selected[0].copy()
selected[:, :29] = (
anchor[None, :29]
+ selected[:, :29]
- selected_first[None, :29]
)
selected[:, 29:32] = (
anchor[None, 29:32]
+ selected[:, 29:32]
- selected_first[None, 29:32]
)
selected[:, 32:36] = anchor[None, 32:36]
# Legacy: both ends return exactly to the target motion's first pose. The
# smooth envelope lets idle/listen move naturally in the middle while
# guaranteeing continuous idle->listen and listen->task boundaries.
if frame_count == 1:
motion_weight = np.zeros(1, dtype=np.float32)
else:
ramp = float(
max(
(
self.stationary_blend_frames
if blend_frames is None
else int(blend_frames)
)
- 1,
1,
)
)
offsets = np.arange(frame_count, dtype=np.float32)
motion_weight = np.minimum(
np.minimum(offsets / ramp, offsets[::-1] / ramp),
1.0,
)
if hold_listen:
motion_weight = np.minimum(offsets / ramp, 1.0)
motion_weight = np.clip(motion_weight, 0.0, 1.0)
if hold_listen:
motion_weight = motion_weight**3 * (
motion_weight * (motion_weight * 6.0 - 15.0) + 10.0
)
else:
motion_weight = motion_weight * motion_weight * (
3.0 - 2.0 * motion_weight
)
start_pose = entry_pose if hold_listen and entry_pose is not None else anchor
result = _blend_pose_rows(start_pose, selected, motion_weight)
if entry_pose is not None and not hold_listen:
entry = np.asarray(entry_pose, dtype=np.float32).reshape(36)
entry_count = min(
int(
self.turn_transition_frames
if blend_frames is None
else blend_frames
),
frame_count,
)
entry_weight = np.ones(frame_count, dtype=np.float32)
if entry_count > 1:
phase_weight = np.linspace(
0.0, 1.0, entry_count, dtype=np.float32
)
phase_weight = phase_weight * phase_weight * phase_weight * (
phase_weight * (phase_weight * 6.0 - 15.0) + 10.0
)
entry_weight[:entry_count] = phase_weight
else:
entry_weight[0] = 1.0
result = _blend_pose_rows(entry, result, entry_weight)
if not np.isfinite(result).all():
raise RuntimeError(
f"Non-finite blended {phase} motion from {selected_name}"
)
return result, selected_name
def _sample_stationary_anchor(self, task_start_pose: np.ndarray) -> np.ndarray:
"""Choose a verified upright neutral pose at the task's root position."""
if bool(getattr(self, "safe_stationary_anchors", False)):
anchor, _ = self._nearest_safe_stationary_anchor(task_start_pose)
return _normalise_motion_quaternions(anchor[None, :])[0]
candidates = list(self.stationary_motion_pools["idle"])
random.shuffle(candidates)
for candidate in candidates:
motion = self._load_motion(candidate.motion_dir)
if not self._stationary_motion_is_safe(motion):
continue
frame = random.randint(0, int(motion.shape[0]) - 1)
anchor = motion[frame].astype(np.float32, copy=True)
# Align the neutral pose to the command's ground-plane position,
# but keep its own physically matched root height. The existing
# smooth transition handles the later stationary-to-task Z change.
anchor[29:31] = np.asarray(task_start_pose, dtype=np.float32)[29:31]
return _normalise_motion_quaternions(anchor[None, :])[0]
raise RuntimeError("No safe neutral idle pose is available as RTC anchor")
def _build_task_sequence(
self,
motion: np.ndarray,
*,
stationary_anchor: np.ndarray,
transition_frames: int | None = None,
) -> tuple[np.ndarray, int]:
"""Align task heading, then transition smoothly out of standing."""
aligned_motion = _align_task_motion_to_anchor_heading(
motion,
stationary_anchor,
)
transition_frames = max(
int(
self.stationary_blend_frames
if transition_frames is None
else transition_frames
),
2,
)
if bool(getattr(self, "dynamic_task_entry_blend", False)):
blend_count = min(transition_frames, len(aligned_motion))
phase = np.linspace(0.0, 1.0, blend_count, dtype=np.float32)
phase = phase * phase * phase * (
phase * (phase * 6.0 - 15.0) + 10.0
)
task_sequence = aligned_motion.astype(np.float32, copy=True)
task_sequence[:blend_count] = _blend_pose_rows(
stationary_anchor,
aligned_motion[:blend_count],
phase,
)
return task_sequence, blend_count
target = np.repeat(
aligned_motion[0:1],
transition_frames,
axis=0,
)
phase = np.linspace(0.0, 1.0, transition_frames, dtype=np.float32)
if bool(getattr(self, "multi_turn_sessions", False)):
# Multi-turn joins are deliberately more relaxed: the quintic
# curve reaches both poses with zero velocity and acceleration.
# Historical single-turn versions retain their exact cubic data.
phase = phase * phase * phase * (
phase * (phase * 6.0 - 15.0) + 10.0
)
else:
phase = phase * phase * (3.0 - 2.0 * phase)
transition = _blend_pose_rows(stationary_anchor, target, phase)
# Avoid duplicating the target's first frame after the transition.
task_sequence = np.concatenate(
[transition, aligned_motion[1:]],
axis=0,
)
return task_sequence.astype(np.float32, copy=False), transition_frames
def _task_original_frame_indices(
self,
*,
sequence_frames: int,
motion_frames: int,
transition_frames: int,
) -> np.ndarray:
"""Map constructed task rows back to progress in the raw primitive."""
if bool(getattr(self, "dynamic_task_entry_blend", False)):
return np.arange(sequence_frames, dtype=np.int64).clip(
0, motion_frames - 1
)
return np.clip(
np.arange(sequence_frames, dtype=np.int64)
- (int(transition_frames) - 1),
0,
motion_frames - 1,
)
@staticmethod
def _build_pose_transition(
start_pose: np.ndarray,
end_pose: np.ndarray,
frame_count: int,
) -> np.ndarray:
"""Join two absolute poses with a zero-slope quintic transition."""
count = max(int(frame_count), 2)
phase = np.linspace(0.0, 1.0, count, dtype=np.float32)
# Smootherstep has zero first and second derivatives at both ends,
# avoiding the abrupt velocity change of the previous 0.4 s cubic join.
weight = phase * phase * phase * (
phase * (phase * 6.0 - 15.0) + 10.0
)
target = np.repeat(
np.asarray(end_pose, dtype=np.float32).reshape(1, 36),
count,
axis=0,
)
return _blend_pose_rows(
np.asarray(start_pose, dtype=np.float32).reshape(36),
target,
weight,
)
@staticmethod
def _stationary_progress_rows(frame_count: int) -> np.ndarray:
row = np.asarray([0.0, 1.0, 0.0, 0.0, 1.0], dtype=np.float32)
return np.repeat(row[None, :], max(int(frame_count), 0), axis=0)
@staticmethod
def _completed_motion_progress_rows(frame_count: int) -> np.ndarray:
row = np.asarray([1.0, 0.0, 1.0, 0.0, 1.0], dtype=np.float32)
return np.repeat(row[None, :], max(int(frame_count), 0), axis=0)
@staticmethod
def _sample_interturn_idle_steps() -> int:
"""Sample the approved immediate/short/long gap mixture on a 0.4 s grid."""
draw = random.random()
if draw < 0.50:
return 0
if draw < 0.85:
return random.randint(1, 3) # 0.4--1.2 s
return random.randint(4, 10) # 1.6--4.0 s
def _sample_control_post_idle_steps(
self,
interaction_mode: str,
*,
has_followup: bool,
) -> int:
"""Sample post-control background without changing legacy runs.
An explicit maximum enables the longer Stop-persistence curriculum.
Zero keeps the historical follow-up/terminal distributions bit for
bit, which is important for clean ablations and old run contracts.
"""
if (
str(interaction_mode) == "emergency-stop"
and bool(self.control_post_idle_max_steps_explicit)
):
if self.control_post_idle_sampling == "short_heavy_long_tail":
minimum = int(self.control_post_idle_min_steps)
maximum = int(self.control_post_idle_max_steps)
span = maximum - minimum
short_end = min(
maximum,
minimum + max(int(round(span * 4.0 / 17.0)), 0),
)
middle_start = min(short_end + 1, maximum)
middle_end = min(
maximum,
minimum + max(int(round(span * 11.0 / 17.0)), 0),
)
long_start = min(middle_end + 1, maximum)
draw = random.random()
if draw < 0.70 or middle_start > middle_end:
return random.randint(minimum, short_end)
if draw < 0.95 or long_start > maximum:
return random.randint(middle_start, middle_end)
return random.randint(long_start, maximum)
return random.randint(
int(self.control_post_idle_min_steps),
int(self.control_post_idle_max_steps),
)
if bool(has_followup):
return max(
self._sample_interturn_idle_steps(),
int(self.control_post_idle_min_steps),
)
return random.randint(
int(self.control_post_idle_min_steps),
max(self.initial_idle_max_steps, 1),
)
def _sample_interruption_prefix_frames(
self,
task_sequence: np.ndarray,
*,
next_reaction_frames: int,
) -> int:
"""Choose a Qwen-aligned speech onset inside an unfinished motion."""
sequence_frames = int(np.asarray(task_sequence).shape[0])
stride = int(self.action_exec_horizon)
minimum = int(
math.ceil(
self.interruption_min_start_seconds * self.motion_fps / stride
)
) * stride
requested_maximum = int(
math.floor(
self.interruption_max_start_seconds * self.motion_fps / stride
)
) * stride
remaining = max(
int(
math.ceil(
self.interruption_min_remaining_seconds
* self.motion_fps
)
),
int(next_reaction_frames),
)
latest = ((sequence_frames - remaining) // stride) * stride
maximum = min(requested_maximum, latest)
if maximum < minimum:
raise RuntimeError(
"Motion cannot satisfy the interruption timing contract: "
f"frames={sequence_frames} start>={minimum} "
f"remaining>={remaining}"
)
candidates = tuple(range(minimum, maximum + 1, stride))
return int(random.choice(candidates))
def _sample_control_command(
self,
intent: str,
voice_name: str,
) -> tuple[ControlCommandSource, np.ndarray, int]:
"""Pick a same-speaker command and return waveform/reaction frames."""
candidates = self.control_command_catalog.get(str(intent), {}).get(
str(voice_name),
(),
)
if not candidates:
raise RuntimeError(
f"No {intent} TTS command for session voice {voice_name}"
)
source = random.choice(candidates)
speech = _read_resampled_speech_float(
source.waveform_path,
self.audio_sampling_rate,
)
if int(speech.size) != int(source.speech_samples):
raise RuntimeError(
"Control-command waveform changed after manifest creation: "
f"{source.waveform_path}"
)
# The state changes only after the final 0.6 s window needed to hear
# the complete command. There is no synthetic </listen> interval.
reaction_frames = (
self._runtime_listen_window_count(int(speech.size))
* self.action_exec_horizon
)
return source, speech, int(reaction_frames)
def _backchannel_candidates(
self,
voice_name: str,
) -> tuple[BackchannelSource, ...]:
candidates = tuple(
self.backchannel_catalog.get(str(voice_name), ())
)
if not candidates:
raise RuntimeError(
f"No {self.split} backchannel TTS for session voice "
f"{voice_name}"
)
return candidates
def _backchannel_max_reaction_frames(self, voice_name: str) -> int:
return max(
self._runtime_listen_window_count(source.speech_samples)
* self.action_exec_horizon
for source in self._backchannel_candidates(voice_name)
)
def _backchannel_motion_is_eligible(
self,
record: dict[str, Any],
*,
voice_name: str,
) -> bool:
"""Whether every held-out acknowledgement fits inside real motion."""
stride = int(self.action_exec_horizon)
minimum_start = max(
int(
math.ceil(
self.backchannel_min_start_seconds
* self.motion_fps
/ stride
)
)
* stride,
int(math.ceil(self.turn_transition_frames / stride)) * stride,
)
minimum_remaining = max(
int(
math.ceil(
self.backchannel_min_remaining_seconds * self.motion_fps
)
),
stride,
)
required = (
minimum_start
+ self._backchannel_max_reaction_frames(voice_name)
+ minimum_remaining
)
return self._estimated_runtime_motion_frames(record) >= required
def _sample_backchannel(
self,
voice_name: str,
) -> tuple[BackchannelSource, np.ndarray, int]:
source = random.choice(self._backchannel_candidates(voice_name))
speech = _read_resampled_speech_float(
source.waveform_path,
self.audio_sampling_rate,
)
if int(speech.size) != int(source.speech_samples):
raise RuntimeError(
"Backchannel waveform changed after manifest creation: "
f"{source.waveform_path}"
)
reaction_frames = (
self._runtime_listen_window_count(int(speech.size))
* self.action_exec_horizon
)
return source, speech, int(reaction_frames)
@staticmethod
def _sample_disfluency_gap_steps(
rng: random.Random | None = None,
) -> int:
"""Sample 0.4--2.8 s with 60/30/10% short/medium/long mass."""
chooser = rng if rng is not None else random
return int(
chooser.choices(
population=(1, 2, 3, 4, 5, 6, 7),
weights=(18, 18, 9, 9, 2, 2, 2),
k=1,
)[0]
)
def _read_disfluency_audio(
self,
source: DisfluencyAudioSource,
) -> np.ndarray:
speech = _read_resampled_speech_float(
source.waveform_path,
self.audio_sampling_rate,
)
if int(speech.size) != int(source.speech_samples):
raise RuntimeError(
"Hesitation/pause waveform changed after manifest creation: "
f"{source.waveform_path}"
)
return np.asarray(
speech * np.float32(source.speech_scale),
dtype=np.float32,
)
def _build_disfluency_turn_speech(
self,
*,
record: dict[str, Any],
voice: AudioSource,
assignment: SourceBackgroundAssignment,
subtype: str,
gap_steps: int,
marker_id: str = "",
repair_style: str = "",
provisional_audio_dir: str = "",
provisional_instruction: str = "",
provisional_instruction_zh: str = "",
) -> tuple[AudioSource, SourceBackgroundAssignment, np.ndarray, dict[str, Any]]:
"""Compose one clean multi-segment utterance over later background mix."""
voice_name = self._voice_name(voice)
key = (str(record["audio_dir"]), voice_name)
by_role = self.disfluency_instruction_segments.get(key)
if by_role is None or set(by_role) != {"prefix", "suffix"}:
raise RuntimeError(f"Missing split instruction speech for {key}")
prefix_source = by_role["prefix"]
suffix_source = by_role["suffix"]
prefix = self._read_disfluency_audio(prefix_source)
suffix = self._read_disfluency_audio(suffix_source)
gap_steps = int(gap_steps)
if not 1 <= gap_steps <= 7:
raise ValueError(f"Invalid disfluency gap steps: {gap_steps}")
gap_samples = gap_steps * self.audio_stride_samples
if subtype == "self-repair" and not repair_style:
# Historical v23 callers always used an explicit repair cue and
# predate the style field. Preserve that exact construction.
repair_style = "explicit-cue"
component_rows: list[dict[str, Any]] = []
def assemble(parts: list[tuple[str, np.ndarray, str]]) -> np.ndarray:
offset = 0
arrays: list[np.ndarray] = []
for role, values, text in parts:
array = np.asarray(values, dtype=np.float32).reshape(-1)
arrays.append(array)
component_rows.append(
{
"role": role,
"text": text,
"start_sample": int(offset),
"end_sample": int(offset + array.size),
}
)
offset += int(array.size)
return np.concatenate(arrays).astype(np.float32, copy=False)
if subtype == "silent-pause":
speech = assemble(
[
("final-prefix", prefix, prefix_source.text),
("background-only-pause", np.zeros(gap_samples, np.float32), ""),
("final-suffix", suffix, suffix_source.text),
]
)
marker_source = None
elif subtype in {"filled-pause", "self-repair"}:
explicit_marker = (
subtype == "filled-pause" or repair_style == "explicit-cue"
)
marker_source = None
marker: np.ndarray | None = None
if explicit_marker:
marker_category = (
"hesitation_filler"
if subtype == "filled-pause"
else "repair_cue"
)
marker_candidates = self.disfluency_marker_catalog.get(
voice_name,
{},
).get(marker_category, ())
if not marker_candidates:
raise RuntimeError(
f"No {marker_category} for split={self.split} "
f"voice={voice_name}"
)
marker_source = next(
(
candidate
for candidate in marker_candidates
if candidate.segment_id == str(marker_id)
),
None,
)
if marker_source is None:
raise RuntimeError(
f"Unknown {marker_category} marker {marker_id!r} for "
f"split={self.split} voice={voice_name}"
)
marker = self._read_disfluency_audio(marker_source)
minimum_span = int(
marker.size + round(0.16 * self.audio_sampling_rate)
)
required_gap_steps = int(
math.ceil(minimum_span / self.audio_stride_samples)
)
if gap_steps < required_gap_steps:
raise RuntimeError(
"Stamped hesitation gap is too short for its marker: "
f"gap={gap_steps} required={required_gap_steps} "
f"path={marker_source.waveform_path}"
)
leading = (gap_samples - int(marker.size)) // 2
trailing = gap_samples - int(marker.size) - leading
elif subtype != "self-repair" or repair_style != "implicit-restart":
raise RuntimeError(
f"Invalid marker-free disfluency: {subtype}/{repair_style}"
)
if subtype == "filled-pause":
assert marker is not None and marker_source is not None
speech = assemble(
[
("final-prefix", prefix, prefix_source.text),
("hesitation-background-before", np.zeros(leading, np.float32), ""),
("hesitation-filler", marker, marker_source.text),
("hesitation-background-after", np.zeros(trailing, np.float32), ""),
("final-suffix", suffix, suffix_source.text),
]
)
else:
provisional_key = (str(provisional_audio_dir), voice_name)
provisional_roles = self.disfluency_instruction_segments.get(
provisional_key
)
if provisional_roles is None:
raise RuntimeError(
"Self-repair provisional command lacks regenerated prefix: "
f"{provisional_key}"
)
provisional_prefix_source = provisional_roles["prefix"]
provisional_prefix = self._read_disfluency_audio(
provisional_prefix_source
)
if assignment.original_audio_path is None:
raise RuntimeError("Self-repair needs the complete final TTS")
final_speech = _read_resampled_speech_float(
assignment.original_audio_path,
self.audio_sampling_rate,
)
if final_speech.size != int(assignment.speech_samples):
raise RuntimeError(
f"Final self-repair TTS changed: {assignment.original_audio_path}"
)
final_speech = np.asarray(
final_speech * np.float32(assignment.speech_scale),
dtype=np.float32,
)
if repair_style == "implicit-restart":
repair_parts = [
(
"provisional-prefix",
provisional_prefix,
provisional_prefix_source.text,
),
(
"repair-implicit-background",
np.zeros(gap_samples, np.float32),
"",
),
(
"complete-final-instruction",
final_speech,
instruction_text_for_voice(record, voice),
),
]
else:
assert marker is not None and marker_source is not None
repair_parts = [
(
"provisional-prefix",
provisional_prefix,
provisional_prefix_source.text,
),
(
"repair-background-before",
np.zeros(leading, np.float32),
"",
),
("repair-cue", marker, marker_source.text),
(
"repair-background-after",
np.zeros(trailing, np.float32),
"",
),
(
"complete-final-instruction",
final_speech,
instruction_text_for_voice(record, voice),
),
]
speech = assemble(repair_parts)
else:
raise ValueError(f"Unsupported disfluency subtype: {subtype!r}")
peak = float(np.max(np.abs(speech), initial=0.0))
if not np.isfinite(speech).all() or peak >= 1.0:
raise RuntimeError(
f"Invalid/clipping disfluency speech peak={peak:g} subtype={subtype}"
)
window_count = self._runtime_listen_window_count(int(speech.size))
custom_voice = AudioSource(
source_key=voice.source_key,
clips=(),
waveform_path=voice.waveform_path,
waveform_count=window_count,
)
custom_assignment = SourceBackgroundAssignment(
source_key=assignment.source_key,
background_key=assignment.background_key,
listen_start_index=assignment.listen_start_index,
listen_window_count=window_count,
background_window_count=assignment.background_window_count,
initial_idle_max_steps=assignment.initial_idle_max_steps,
max_motion_steps=assignment.max_motion_steps,
safety_steps=assignment.safety_steps,
mixed_audio_path=assignment.mixed_audio_path,
original_audio_path=assignment.original_audio_path,
speech_samples=int(speech.size),
speech_scale=1.0,
)
metadata = {
"subtype": subtype,
"turn_index": -1,
"gap_steps": int(gap_steps),
"gap_seconds": gap_steps * self.audio_stride_samples
/ float(self.audio_sampling_rate),
"voice_name": voice_name,
"marker_id": (
marker_source.segment_id if marker_source is not None else None
),
"marker_text": marker_source.text if marker_source is not None else "",
"repair_style": str(repair_style),
"provisional_instruction": str(provisional_instruction),
"provisional_instruction_zh": str(provisional_instruction_zh),
"final_instruction": str(record["instruction"]),
"final_instruction_zh": str(record["instruction_zh"]),
"speech_samples": int(speech.size),
"listen_window_count": int(window_count),
"components": component_rows,
}
return custom_voice, custom_assignment, speech, metadata
def _sample_backchannel_speech_start(
self,
task_sequence: np.ndarray,
*,
reaction_frames: int,
) -> int:
"""Place a short acknowledgement wholly inside active task motion."""
frames = int(np.asarray(task_sequence).shape[0])
stride = int(self.action_exec_horizon)
minimum = max(
int(
math.ceil(
self.backchannel_min_start_seconds
* self.motion_fps
/ stride
)
)
* stride,
int(math.ceil(self.turn_transition_frames / stride)) * stride,
)
requested_maximum = int(
math.floor(
self.backchannel_max_start_seconds
* self.motion_fps
/ stride
)
) * stride
minimum_tail = max(
int(
math.ceil(
self.backchannel_min_remaining_seconds * self.motion_fps
)
),
stride,
)
latest = (
(frames - int(reaction_frames) - minimum_tail) // stride
) * stride
maximum = min(requested_maximum, latest)
if maximum < minimum:
raise RuntimeError(
"Motion is too short for an in-motion backchannel: "
f"frames={frames} reaction={reaction_frames}"
)
return int(random.choice(tuple(range(minimum, maximum + 1, stride))))
def _control_command_min_start_frames(self) -> int:
"""First valid command onset after the task-entry transition."""
stride = int(self.action_exec_horizon)
requested = int(
math.ceil(
float(
getattr(
self,
"control_min_start_seconds",
getattr(self, "interruption_min_start_seconds", 2.0),
)
)
* self.motion_fps
/ stride
)
) * stride
transition_complete = int(
math.ceil(self.turn_transition_frames / stride)
) * stride
return max(stride, requested, transition_complete)
def _control_max_reaction_frames(
self,
intent: str,
voice_name: str,
) -> int:
candidates = self.control_command_catalog.get(str(intent), {}).get(
str(voice_name),
(),
)
if not candidates:
return 0
return max(
self._runtime_listen_window_count(int(source.speech_samples))
* self.action_exec_horizon
for source in candidates
)
def _control_motion_is_eligible(
self,
record: dict[str, Any],
*,
intent: str,
voice_name: str,
) -> bool:
"""Whether every phrase for this voice fits inside real task motion.
The normal instruction-to-motion transition is part of the generated
task sequence. A control command may start only after that transition,
must be heard completely, and must leave the configured real-motion
tail before the original end. Filtering is task-local: it never
removes a primitive from the preserved turn-taking/interruption pools.
"""
reaction_frames = self._control_max_reaction_frames(
intent,
voice_name,
)
if reaction_frames <= 0:
return False
task_frames = self._estimated_runtime_motion_frames(record)
max_motion_steps = int(getattr(self, "max_motion_steps", 0))
if max_motion_steps > 0:
# The smoke/debug path truncates the loaded motion before session
# assembly. Eligibility must use that same runtime length;
# otherwise an originally long primitive can pass the index-time
# filter but fail later when the complete Stop utterance no longer
# fits inside its truncated task sequence.
debug_frames = max(
self.prefix_frames + 2,
max_motion_steps * self.action_exec_horizon,
)
task_frames = min(task_frames, debug_frames)
if not bool(getattr(self, "dynamic_task_entry_blend", False)):
task_frames += max(self.turn_transition_frames - 1, 1)
minimum_tail = max(
int(
math.ceil(
float(
getattr(
self,
"control_min_remaining_seconds",
self.action_exec_horizon / float(self.motion_fps),
)
)
* self.motion_fps
)
),
self.action_exec_horizon,
)
required = (
self._control_command_min_start_frames()
+ reaction_frames
+ minimum_tail
)
return task_frames >= required
@staticmethod
def _control_motion_activity(
task_sequence: np.ndarray,
) -> np.ndarray:
"""Generic per-frame whole-body motion magnitude.
The score has no action-name/category assumptions: joint RMS change,
root translation and root rotation all contribute. It is used only
to rank otherwise valid stop-command times within the same source.
"""
sequence = np.asarray(task_sequence, dtype=np.float32)
if sequence.ndim != 2 or sequence.shape[1] != 36:
raise ValueError(
f"Control timing expects a (T,36) motion, got {sequence.shape}"
)
count = int(sequence.shape[0])
if count <= 1:
return np.zeros(count, dtype=np.float32)
joint_delta = np.diff(sequence[:, :29], axis=0)
root_delta = np.diff(sequence[:, 29:32], axis=0)
quaternions = _normalise_motion_quaternions(sequence)[:, 32:36]
quaternion_dot = np.abs(
np.sum(quaternions[1:] * quaternions[:-1], axis=-1)
)
angular_delta = 2.0 * np.arccos(
np.clip(quaternion_dot, 0.0, 1.0)
)
delta_score = (
np.sqrt(np.mean(np.square(joint_delta), axis=-1))
+ 2.0 * np.linalg.norm(root_delta, axis=-1)
+ 0.25 * angular_delta
).astype(np.float32)
return np.concatenate([delta_score[:1], delta_score], axis=0)
def _sample_control_speech_start(
self,
task_sequence: np.ndarray,
*,
reaction_frames: int,
) -> tuple[int, dict[str, float | int | bool]]:
"""Choose an aligned, preferably active mid-motion command start."""
frames = int(np.asarray(task_sequence).shape[0])
stride = int(self.action_exec_horizon)
minimum = self._control_command_min_start_frames()
requested_maximum = int(
math.floor(
float(
getattr(
self,
"control_max_start_seconds",
self.interruption_max_start_seconds,
)
)
* self.motion_fps
/ stride
)
) * stride
minimum_tail = max(
int(
math.ceil(
float(
getattr(
self,
"control_min_remaining_seconds",
stride / float(self.motion_fps),
)
)
* self.motion_fps
)
),
stride,
)
# The configured tail is measured after the full command is audible,
# not merely after its onset. The activity selector below then keeps
# otherwise legal Stop examples away from already-stationary tails.
latest = (
(frames - int(reaction_frames) - minimum_tail) // stride
) * stride
maximum = min(requested_maximum, latest)
if maximum < minimum:
raise RuntimeError(
"Motion is too short for a complete control command: "
f"frames={frames} reaction={reaction_frames}"
)
candidates = tuple(range(minimum, maximum + 1, stride))
diagnostics: dict[str, float | int | bool] = {
"candidate_count": len(candidates),
"active_candidate_count": len(candidates),
"activity_score": 0.0,
"activity_reference": 0.0,
"activity_fallback": False,
"minimum_remaining_frames": int(minimum_tail),
}
if not bool(
getattr(self, "control_activity_aware_timing", False)
):
return int(random.choice(candidates)), diagnostics
activity = self._control_motion_activity(task_sequence)
reference = float(np.quantile(activity, 0.90))
active_floor = max(reference * 0.15, 1e-6)
ranked: list[tuple[float, int]] = []
active: list[tuple[float, int]] = []
for start in candidates:
response = start + int(reaction_frames)
onset_score = float(np.mean(activity[start : start + stride]))
speech_score = float(np.mean(activity[start:response]))
response_score = float(
np.mean(
activity[
max(response - stride, start) : min(
response + stride,
frames,
)
]
)
)
# Both the command onset and the point at which the complete
# command becomes actionable must be moving. The middle term
# breaks ties without allowing one active instant to hide a
# stationary boundary.
score = min(onset_score, response_score) + 0.25 * speech_score
ranked.append((score, start))
if onset_score >= active_floor and response_score >= active_floor:
active.append((score, start))
pool = active if active else ranked
best_score = max(score for score, _ in pool)
near_best = [
(score, start)
for score, start in pool
if score >= 0.75 * best_score
]
selected_score, selected_start = random.choice(near_best)
diagnostics.update(
{
"active_candidate_count": len(active),
"activity_score": float(selected_score),
"activity_reference": reference,
"activity_fallback": not bool(active),
}
)
return int(selected_start), diagnostics
def _safe_hold_boundary(
self,
task_sequence: np.ndarray,
*,
earliest_frame: int,
) -> int:
"""Find the first supported, upright, low-velocity Qwen boundary.
This uses the same joint/root velocity proxy that creates TextOp foot
contacts. It therefore lets a backflip finish and land, while a pose
that is already balanced can stop earlier. If no earlier supported
window exists, the complete source motion is retained and its real end
pose is held; no neutral-idle pose is substituted.
"""
sequence = np.asarray(task_sequence, dtype=np.float32)
count = int(sequence.shape[0])
stride = int(self.action_exec_horizon)
window = int(self.safe_hold_stability_window_frames)
first = max(
int(math.ceil(int(earliest_frame) / stride) * stride),
window,
)
if count <= first:
return count
joint_delta = np.diff(sequence[:, :29], axis=0, prepend=sequence[:1, :29])
root_delta = np.diff(sequence[:, 29:32], axis=0, prepend=sequence[:1, 29:32])
contacts = estimate_two_foot_contacts(joint_delta, root_delta)
quat = _normalise_motion_quaternions(sequence)[:, 32:36]
upright = 1.0 - 2.0 * (
np.square(quat[:, 1]) + np.square(quat[:, 2])
)
for boundary in range(first, count, stride):
start = max(boundary - window, 1)
stop = boundary
if stop - start < max(window // 2, 2):
continue
supported = np.any(contacts[start:stop] > 0.5, axis=-1)
joint_speed = np.linalg.norm(joint_delta[start:stop], axis=-1)
horizontal_speed = np.linalg.norm(root_delta[start:stop, :2], axis=-1)
vertical_speed = np.abs(root_delta[start:stop, 2])
if (
float(np.mean(supported)) >= 0.8
and float(np.min(upright[start:stop])) >= 0.55
and float(np.quantile(joint_speed, 0.8)) <= 0.08
and float(np.quantile(horizontal_speed, 0.8)) <= 0.025
and float(np.quantile(vertical_speed, 0.8)) <= 0.025
):
return int(boundary)
return count
def _nearest_safe_recovery_boundary(
self,
task_sequence: np.ndarray,
*,
earliest_frame: int,
) -> tuple[int, np.ndarray]:
"""Choose a nearby support phase and its closest audited stand pose.
The search is action-agnostic. It considers every frame in the next
one-second transition budget, preferring upright double-support frames
and then minimizing whole-body distance and velocity to the audited
neutral-pose manifold. Looking at every frame matters for cyclic
gaits: a 0.4-second sampling grid can skip the brief double-support
phase entirely. This avoids both a hard-coded locomotion/jump taxonomy
and an unsafe instant morph from a flight or single-support frame.
"""
sequence = np.asarray(task_sequence, dtype=np.float32)
count = int(len(sequence))
if count <= 0:
raise ValueError("Safe recovery requires a non-empty task sequence")
first = min(max(int(earliest_frame), 1), count)
if not bool(getattr(self, "safe_stationary_anchors", False)):
pose = sequence[first - 1]
return first, self._sample_stationary_anchor(pose)
last = min(count, first + int(self.turn_transition_frames))
boundaries = range(first, last + 1)
joint_delta = np.diff(
sequence[:, :29], axis=0, prepend=sequence[:1, :29]
)
root_delta = np.diff(
sequence[:, 29:32], axis=0, prepend=sequence[:1, 29:32]
)
contacts = estimate_two_foot_contacts(joint_delta, root_delta)
quat = _normalise_motion_quaternions(sequence)[:, 32:36]
upright = 1.0 - 2.0 * (
np.square(quat[:, 1]) + np.square(quat[:, 2])
)
ranked: list[tuple[tuple[float, ...], int, np.ndarray]] = []
for boundary in boundaries:
frame = boundary - 1
pose = sequence[frame]
safe_anchor, pose_distance = self._nearest_safe_stationary_anchor(
pose
)
support_count = int(np.count_nonzero(contacts[frame] > 0.5))
support_rank = 0 if support_count == 2 else 1 if support_count == 1 else 2
upright_penalty = max(0.80 - float(upright[frame]), 0.0)
velocity_penalty = (
2.0 * float(np.linalg.norm(joint_delta[frame, :12]))
+ 4.0 * float(np.linalg.norm(root_delta[frame]))
)
wait_fraction = float(boundary - first) / max(last - first, 1)
rank = (
float(support_rank),
float(upright_penalty > 0.0),
float(pose_distance + velocity_penalty + 0.05 * wait_fraction),
)
ranked.append((rank, boundary, safe_anchor))
_, boundary, anchor = min(ranked, key=lambda item: item[0])
return int(boundary), anchor.astype(np.float32, copy=False)
def _stream_action_rows(
self,
sequence: np.ndarray,
*,
start: int,
) -> tuple[
np.ndarray,
np.ndarray,
np.ndarray,
np.ndarray,
np.ndarray,
np.ndarray,
]:
"""Return v26's 30 future frames after the current state.
``start`` indexes the observed/current frame. Action token zero is
therefore ``sequence[start + 1]``; the tail is repeated only for
padding and is masked out. Keeping this distinction is important for
TextOp root deltas and for strict train/inference RTC alignment.
"""
if sequence.ndim != 2 or sequence.shape[1] != 36 or not len(sequence):
raise ValueError(f"Invalid streaming sequence shape: {sequence.shape}")
start = int(start)
if start < 0 or start >= len(sequence):
raise IndexError(
"Dense RTC state index is outside the source timeline: "
f"start={start} sequence={len(sequence)}"
)
future_indices = np.arange(
start + 1,
start + 1 + self.action_chunk_size,
dtype=np.int64,
)
valid = future_indices < len(sequence)
clipped = np.clip(future_indices, 0, len(sequence) - 1)
actions = sequence[clipped].astype(np.float32, copy=True)
state = sequence[start].astype(np.float32, copy=True)
action_reference_state = state.copy()
# Two prior action tokens need three absolute poses:
# x[t-2] -> x[t-1], x[t-1] -> x[t].
# At the source boundary, repeat x[0] so the initial velocity is zero
# without ever borrowing history from another source.
context_indices = np.clip(
np.arange(start - 2, start + 1, dtype=np.int64),
0,
len(sequence) - 1,
)
smoothness_context_reference_state = sequence[
context_indices[0]
].astype(np.float32, copy=True)
smoothness_context_actions = sequence[
context_indices[1:]
].astype(np.float32, copy=True)
action_mask = valid.astype(np.float32, copy=False)
if not np.any(action_mask[self.prefix_frames:] > 0.0):
raise IndexError(
"Dense RTC action target has no supervised frame after the "
f"maximum prefix: start={start} sequence={len(sequence)}"
)
return (
actions,
action_mask,
state,
action_reference_state,
smoothness_context_actions,
smoothness_context_reference_state,
)
def _build_session_audio_windows(
self,
*,
turn_audio: list[dict[str, Any]],
semantic_window_count: int,
) -> tuple[tuple[np.ndarray, ...], str, int]:
"""Mix every turn into one uninterrupted background waveform.
Existing assignment rows are reused only for the original clean TTS
path and its audited gain. Their old single-turn background offsets
are intentionally not concatenated. One new contiguous background
slice covers the complete session, so every overlapping 0.6 s window
shares bit-identical PCM with its neighbours.
"""
count = int(semantic_window_count)
if count <= 0:
raise ValueError("A session needs at least one semantic window")
choices: list[tuple[BackgroundWaveformSource, int]] = []
total_slots = 0
for source in sorted(
self.background_catalog.values(),
key=lambda item: item.source_key,
):
if not isinstance(source, BackgroundWaveformSource):
continue
slots = int(source.window_count) - count + 1
if slots > 0:
choices.append((source, slots))
total_slots += slots
if total_slots <= 0:
raise RuntimeError(
"No one-piece office background can cover the complete "
f"multi-turn session: windows={count}"
)
slot = random.randrange(total_slots)
background: BackgroundWaveformSource | None = None
background_start = 0
for candidate, slots in choices:
if slot < slots:
background = candidate
background_start = int(slot)
break
slot -= slots
if background is None:
raise AssertionError("Background slot selection fell through")
flat_count = (
(count - 1) * self.audio_stride_samples
+ self.audio_window_samples
)
flat = _read_pcm16_wave_segment(
background.waveform_path,
start_sample=background_start * self.audio_stride_samples,
sample_count=flat_count,
expected_rate=self.audio_sampling_rate,
)
prefix_samples = self.audio_window_samples - self.audio_stride_samples
previous_speech_end = -1
for payload in turn_audio:
assignment = payload["assignment"]
if not isinstance(assignment, SourceBackgroundAssignment):
raise TypeError("Invalid multi-turn speech assignment")
speech_start_frame = int(
payload.get("speech_start_frame", payload["listen_start_frame"])
)
if speech_start_frame % self.audio_sample_step_frames != 0:
raise RuntimeError(
"A speech boundary must align with one Qwen update: "
f"frame={speech_start_frame}"
)
if assignment.original_audio_path is None:
raise RuntimeError(
f"Missing clean TTS waveform for {assignment.source_key}"
)
payload_speech = payload.get("speech_waveform")
speech = (
np.asarray(payload_speech, dtype=np.float32).reshape(-1)
if payload_speech is not None
else _read_resampled_speech_float(
assignment.original_audio_path,
self.audio_sampling_rate,
)
)
if speech.size != int(assignment.speech_samples):
raise RuntimeError(
"Speech waveform changed after production assignment: "
f"source={assignment.source_key} actual={speech.size} "
f"expected={assignment.speech_samples}"
)
start = (
prefix_samples
+ speech_start_frame
// self.audio_sample_step_frames
* self.audio_stride_samples
)
end = start + int(speech.size)
if start < previous_speech_end:
raise RuntimeError(
"Two command waveforms overlap inside one session; "
"speech may overlap motion, but commands may not overlap "
"each other"
)
if end > flat_count:
raise RuntimeError(
"Multi-turn speech exceeds its continuous background: "
f"source={assignment.source_key} end={end} flat={flat_count}"
)
flat[start:end] += speech * np.float32(assignment.speech_scale)
previous_speech_end = end
if float(np.max(np.abs(flat), initial=0.0)) >= 1.0:
raise RuntimeError(
"Multi-turn speech/background mix clips; refusing to hide it "
"with output clipping"
)
flat = (
np.rint(flat * 32768.0)
.clip(-32768, 32767)
.astype("<i2")
.astype(np.float32)
/ np.float32(32768.0)
)
windows = tuple(
flat[
offset * self.audio_stride_samples :
offset * self.audio_stride_samples
+ self.audio_window_samples
]
for offset in range(count)
)
if any(window.shape != (self.audio_window_samples,) for window in windows):
raise AssertionError("Incomplete multi-turn audio window")
return windows, background.source_key, background_start
def __getitem__(self, index: int) -> dict[str, Any]:
"""Build one source; validation augmentation is repeatable per row."""
deterministic = bool(
getattr(self, "deterministic_eval", True)
and getattr(self, "split", "train") == "val"
)
if not deterministic:
return self._build_source(index)
sessions = getattr(self, "sessions", None)
session = (
sessions[int(index)]
if sessions is not None
else (self.records[int(index)],)
)
row_fingerprint = sum(
(turn_index + 1) * int(record["row_id"])
for turn_index, record in enumerate(session)
)
deterministic_index = int(
session[0].get("_session_pool_index", index)
)
seed = (
int(getattr(self, "split_seed", 42)) * 1_000_003
+ row_fingerprint * 97_409
+ deterministic_index
) & ((1 << 64) - 1)
random_state = random.getstate()
random.seed(seed)
try:
return self._build_source(index)
finally:
random.setstate(random_state)
def _build_source(self, index: int) -> dict[str, Any]:
if bool(getattr(self, "multi_turn_sessions", False)):
return self._build_session_source(index)
return self._build_single_source(index)
def _build_session_source(self, index: int) -> dict[str, Any]:
"""Build one 1--4 turn session with continuous action, audio and KV."""
session = tuple(dict(record) for record in self.sessions[int(index)])
if not self.session_min_turns <= len(session) <= self.session_max_turns:
raise RuntimeError(
f"Invalid runtime session size: {len(session)}"
)
interaction_modes = {
str(record.get("_session_interaction_mode", "turn-taking"))
for record in session
}
if len(interaction_modes) != 1:
raise RuntimeError(
f"One session mixes interaction modes: {interaction_modes}"
)
interaction_mode = interaction_modes.pop()
if interaction_mode == "interruption" and len(session) < 2:
raise RuntimeError("Command interruption cannot be a single-source session")
if interaction_mode not in {
"turn-taking",
"interruption",
"emergency-stop",
"safe-hold",
"backchannel",
"hesitation",
"pause",
}:
raise RuntimeError(
f"Unsupported session interaction mode: {interaction_mode!r}"
)
if interaction_mode == "interruption" and not bool(
getattr(self, "interruption_sessions", False)
):
raise RuntimeError(
"An interruption session reached a dataset with the feature disabled"
)
session_control_turn_index = session[0].get(
"_session_control_turn_index"
)
session_control_has_followup = bool(
session[0].get("_session_control_has_followup", False)
)
session_backchannel_turn_index = session[0].get(
"_session_backchannel_turn_index"
)
session_disfluency_turn_index = session[0].get(
"_session_disfluency_turn_index"
)
session_disfluency_subtype = session[0].get(
"_session_disfluency_subtype"
)
session_disfluency_gap_steps = session[0].get(
"_session_disfluency_gap_steps"
)
session_disfluency_marker_id = session[0].get(
"_session_disfluency_marker_id"
)
session_disfluency_repair_style = session[0].get(
"_session_disfluency_repair_style",
"",
)
if session_disfluency_turn_index is not None:
if any(
record.get("_session_disfluency_turn_index")
!= session_disfluency_turn_index
or record.get("_session_disfluency_subtype")
!= session_disfluency_subtype
or record.get("_session_disfluency_gap_steps")
!= session_disfluency_gap_steps
or record.get("_session_disfluency_marker_id")
!= session_disfluency_marker_id
or record.get("_session_disfluency_repair_style", "")
!= session_disfluency_repair_style
for record in session
):
raise RuntimeError("Disfluency-event placement drifted within session")
session_disfluency_turn_index = int(session_disfluency_turn_index)
if (
interaction_mode not in {"hesitation", "pause"}
or not 0 <= session_disfluency_turn_index < len(session)
or session_disfluency_subtype
not in {"silent-pause", "filled-pause", "self-repair"}
or not 1 <= int(session_disfluency_gap_steps or 0) <= 7
):
raise RuntimeError(
"Invalid hesitation/pause event: "
f"mode={interaction_mode} turn={session_disfluency_turn_index} "
f"subtype={session_disfluency_subtype}"
)
if session_disfluency_subtype == "self-repair":
if session_disfluency_repair_style not in {
"explicit-cue",
"implicit-restart",
}:
raise RuntimeError(
"Invalid self-repair style: "
f"{session_disfluency_repair_style!r}"
)
if (
session_disfluency_repair_style == "explicit-cue"
) != bool(session_disfluency_marker_id):
raise RuntimeError(
"Self-repair cue/style mismatch: "
f"style={session_disfluency_repair_style!r} "
f"marker={session_disfluency_marker_id!r}"
)
elif session_disfluency_repair_style:
raise RuntimeError(
"Only self-repair may carry a repair style: "
f"subtype={session_disfluency_subtype!r} "
f"style={session_disfluency_repair_style!r}"
)
elif interaction_mode in {"hesitation", "pause"}:
raise RuntimeError("Hesitation/pause session has no event placement")
if session_backchannel_turn_index is not None:
if any(
record.get("_session_backchannel_turn_index")
!= session_backchannel_turn_index
for record in session
):
raise RuntimeError(
"Backchannel-event placement drifted within session"
)
session_backchannel_turn_index = int(
session_backchannel_turn_index
)
if (
interaction_mode != "backchannel"
or not 0 <= session_backchannel_turn_index < len(session)
):
raise RuntimeError(
"Invalid backchannel-event placement: "
f"mode={interaction_mode} "
f"turn={session_backchannel_turn_index}"
)
elif interaction_mode == "backchannel":
raise RuntimeError("Backchannel session has no event placement")
if session_control_turn_index is not None:
if any(
record.get("_session_control_turn_index")
!= session_control_turn_index
or bool(
record.get("_session_control_has_followup", False)
)
!= session_control_has_followup
for record in session
):
raise RuntimeError("Control-event placement drifted within session")
session_control_turn_index = int(session_control_turn_index)
if not 0 <= session_control_turn_index < len(session):
raise RuntimeError(
"Control-event turn is outside its session: "
f"turn={session_control_turn_index} turns={len(session)}"
)
if session_control_has_followup != (
session_control_turn_index + 1 < len(session)
):
raise RuntimeError(
"Control follow-up flag disagrees with event placement"
)
sequence_parts: list[np.ndarray] = []
control_parts: list[np.ndarray] = []
progress_parts: list[np.ndarray] = []
original_frame_parts: list[np.ndarray] = []
turn_index_parts: list[np.ndarray] = []
turn_audio: list[dict[str, Any]] = []
idle_motion_names: list[str | None] = []
listen_motion_names: list[str | None] = []
task_transition_frames: list[int] = []
recovery_transition_frames: list[int] = []
inter_turn_idle_steps: list[int] = []
listen_reaction_frames: list[int] = []
listen_reaction_idle_motion_names: list[str | None] = []
speech_onset_samples: list[int] = []
interruption_start_frames: list[int] = []
interruption_original_frames: list[int] = []
interruption_remaining_frames: list[int] = []
interruption_terminal_hold_frames: list[int] = []
control_command_events: list[dict[str, Any]] = []
backchannel_events: list[dict[str, Any]] = []
disfluency_events: list[dict[str, Any]] = []
def append_part(
values: np.ndarray,
*,
control_label: int,
progress: np.ndarray,
original_frames: np.ndarray,
turn_index: int,
) -> None:
array = np.asarray(values, dtype=np.float32)
count = int(array.shape[0])
if array.shape != (count, 36):
raise ValueError(f"Invalid session motion part: {array.shape}")
if progress.shape != (count, 5):
raise ValueError(
f"Invalid session progress part: {progress.shape}"
)
if original_frames.shape != (count,):
raise ValueError(
"Invalid session original-frame part: "
f"{original_frames.shape}"
)
if count <= 0:
return
sequence_parts.append(array)
control_parts.append(
np.full(count, int(control_label), dtype=np.int64)
)
progress_parts.append(np.asarray(progress, dtype=np.float32))
original_frame_parts.append(
np.asarray(original_frames, dtype=np.int64)
)
turn_index_parts.append(
np.full(count, int(turn_index), dtype=np.int64)
)
motions: list[np.ndarray] = []
voices: list[AudioSource] = []
assignments: list[SourceBackgroundAssignment] = []
for record in session:
motion = self._load_motion(record["motion_dir"])
if motion.shape[0] <= 0:
raise RuntimeError(f"Empty motion: {record['motion_dir']}")
if self.max_motion_steps > 0:
debug_frames = max(
self.prefix_frames + 2,
self.max_motion_steps * self.action_exec_horizon,
)
motion = motion[: min(len(motion), debug_frames)]
record["actual_frames"] = int(motion.shape[0])
voice = self._voice_for_record(record)
assignment = self.background_assignments.get(voice.source_key)
if assignment is None:
raise RuntimeError(
f"No raw-audio assignment for {voice.source_key}"
)
motions.append(motion)
voices.append(voice)
assignments.append(assignment)
session_voice_name = session[0].get("_session_voice_name")
session_voice_variant = session[0].get("_session_voice_variant")
if session_voice_name is not None and any(
record.get("_session_voice_name") != session_voice_name
for record in session
):
raise RuntimeError("One session unexpectedly contains multiple voices")
# Load and measure every utterance before assembling motion. An
# interruption cut must leave enough real old-motion frames to cover
# the next utterance's causal reaction period.
turn_speeches: list[np.ndarray | None] = []
turn_onset_samples: list[int] = []
turn_reaction_frames: list[int] = []
turn_disfluency_metadata: list[dict[str, Any] | None] = [
None for _ in session
]
for turn_index, (record, voice, assignment) in enumerate(
zip(session, voices, assignments)
):
speech: np.ndarray | None = None
onset_sample = 0
reaction_frames = 0
if bool(getattr(self, "causal_listen_timing", False)):
if turn_index == session_disfluency_turn_index:
(
voice,
assignment,
speech,
event_metadata,
) = self._build_disfluency_turn_speech(
record=record,
voice=voice,
assignment=assignment,
subtype=str(session_disfluency_subtype),
gap_steps=int(session_disfluency_gap_steps),
marker_id=str(session_disfluency_marker_id or ""),
repair_style=str(
session_disfluency_repair_style or ""
),
provisional_audio_dir=str(
record.get("_session_provisional_audio_dir", "")
),
provisional_instruction=str(
record.get("_session_provisional_instruction", "")
),
provisional_instruction_zh=str(
record.get(
"_session_provisional_instruction_zh",
"",
)
),
)
voices[turn_index] = voice
assignments[turn_index] = assignment
turn_disfluency_metadata[turn_index] = event_metadata
else:
if assignment.original_audio_path is None:
raise RuntimeError(
"Causal listen timing needs the clean TTS waveform: "
f"{assignment.source_key}"
)
speech = _read_resampled_speech_float(
assignment.original_audio_path,
self.audio_sampling_rate,
)
if speech.size != int(assignment.speech_samples):
raise RuntimeError(
"Speech waveform changed after production assignment: "
f"source={assignment.source_key} actual={speech.size} "
f"expected={assignment.speech_samples}"
)
onset_sample = detect_clean_speech_onset_sample(
speech,
self.audio_sampling_rate,
)
prefix_samples = (
self.audio_window_samples - self.audio_stride_samples
)
onset_steps = max(
1,
int(
math.ceil(
(prefix_samples + onset_sample)
/ self.audio_stride_samples
)
),
)
reaction_frames = (
onset_steps + self.listen_reaction_steps
) * self.action_exec_horizon
turn_speeches.append(speech)
turn_onset_samples.append(int(onset_sample))
turn_reaction_frames.append(int(reaction_frames))
anchor = self._sample_stationary_anchor(motions[0][0])
initial_idle_steps = random.randint(0, self.initial_idle_max_steps)
initial_idle_frames = initial_idle_steps * self.action_exec_horizon
idle_sequence, idle_name = self._sample_stationary_sequence(
"idle",
frame_count=initial_idle_frames,
anchor_pose=anchor,
)
idle_motion_names.append(idle_name)
append_part(
idle_sequence,
control_label=IDLE_TOKEN_ID,
progress=self._stationary_progress_rows(initial_idle_frames),
original_frames=np.zeros(initial_idle_frames, dtype=np.int64),
turn_index=0,
)
pending_task_sequence: np.ndarray | None = None
pending_task_original_frames: np.ndarray | None = None
pending_task_cursor = 0
for turn_index, (record, motion, voice, assignment) in enumerate(
zip(session, motions, voices, assignments)
):
if sum(len(part) for part in sequence_parts) % self.action_exec_horizon:
raise RuntimeError(
"A listen phase would start inside one Qwen window"
)
speech_start_frame = sum(len(part) for part in sequence_parts)
speech = turn_speeches[turn_index]
onset_sample = turn_onset_samples[turn_index]
reaction_frames = turn_reaction_frames[turn_index]
listen_entry_pose: np.ndarray | None = None
if bool(getattr(self, "causal_listen_timing", False)):
if interaction_mode == "interruption" and turn_index > 0:
if (
pending_task_sequence is None
or pending_task_original_frames is None
):
raise AssertionError(
"Interrupted speech has no preceding motion tail"
)
reaction_stop = min(
pending_task_cursor + reaction_frames,
len(pending_task_sequence),
)
continued_motion = pending_task_sequence[
pending_task_cursor:reaction_stop
]
continued_original = pending_task_original_frames[
pending_task_cursor:reaction_stop
]
previous_motion_frames = int(
session[turn_index - 1]["actual_frames"]
)
append_part(
continued_motion,
control_label=MOTION_TOKEN_ID,
progress=_progress_rows(
continued_original,
previous_motion_frames,
),
original_frames=continued_original,
turn_index=turn_index - 1,
)
reaction_hold_frames = max(
pending_task_cursor + reaction_frames
- len(pending_task_sequence),
0,
)
anchor = (
continued_motion[-1].copy()
if len(continued_motion)
else pending_task_sequence[-1].copy()
)
if reaction_hold_frames:
# The speech onset was still at least the configured
# 1.5 s before the original task end. If unusually
# long leading silence delays the causal listen point
# beyond that tail, keep the completed old pose under
# </motion> rather than crashing a worker or looking
# into future speech. The following quintic listen
# entry remains the only control-state transition.
append_part(
np.repeat(
anchor[None, :],
reaction_hold_frames,
axis=0,
),
control_label=MOTION_TOKEN_ID,
progress=self._completed_motion_progress_rows(
reaction_hold_frames
),
original_frames=np.full(
reaction_hold_frames,
previous_motion_frames - 1,
dtype=np.int64,
),
turn_index=turn_index - 1,
)
interruption_terminal_hold_frames.append(
int(reaction_hold_frames)
)
listen_entry_pose = anchor.copy()
if bool(
getattr(self, "safe_stationary_anchors", False)
):
safe_anchor = self._sample_stationary_anchor(anchor)
anchor = _align_task_motion_to_anchor_heading(
safe_anchor[None, :],
anchor,
)[0]
pending_task_sequence = None
pending_task_original_frames = None
pending_task_cursor = 0
listen_reaction_idle_motion_names.append(None)
else:
reaction_idle, reaction_name = (
self._sample_stationary_sequence(
"idle",
frame_count=reaction_frames,
anchor_pose=anchor,
)
)
append_part(
reaction_idle,
control_label=IDLE_TOKEN_ID,
progress=self._stationary_progress_rows(
reaction_frames
),
original_frames=np.zeros(
reaction_frames,
dtype=np.int64,
),
turn_index=turn_index,
)
listen_reaction_idle_motion_names.append(reaction_name)
listen_start_frame = sum(len(part) for part in sequence_parts)
listen_frames = voice.window_count * self.action_exec_horizon
listen_kwargs: dict[str, Any] = {}
if bool(getattr(self, "causal_listen_timing", False)):
listen_kwargs["blend_frames"] = self.turn_transition_frames
if listen_entry_pose is not None:
# The first listen row is exactly the actually executed old
# motion pose; the following rows converge into the nearest
# audited two-support standing/listening trajectory.
listen_kwargs["entry_pose"] = listen_entry_pose
listen_sequence, listen_name = self._sample_stationary_sequence(
"listen",
frame_count=listen_frames,
anchor_pose=anchor,
**listen_kwargs,
)
listen_motion_names.append(listen_name)
append_part(
listen_sequence,
control_label=LISTEN_TOKEN_ID,
progress=self._stationary_progress_rows(listen_frames),
original_frames=np.zeros(listen_frames, dtype=np.int64),
turn_index=turn_index,
)
turn_audio.append(
{
"voice": voice,
"assignment": assignment,
"speech_start_frame": speech_start_frame,
"listen_start_frame": listen_start_frame,
"speech_waveform": speech,
"speech_onset_sample": onset_sample,
"listen_reaction_frames": reaction_frames,
}
)
speech_onset_samples.append(int(onset_sample))
listen_reaction_frames.append(int(reaction_frames))
if turn_disfluency_metadata[turn_index] is not None:
event = dict(turn_disfluency_metadata[turn_index] or {})
uncertainty_components = [
component
for component in event["components"]
if component["role"] in DISFLUENCY_UNCERTAINTY_ROLES
]
final_components = [
component
for component in event["components"]
if component["role"]
in {"final-suffix", "complete-final-instruction"}
]
if not uncertainty_components or len(final_components) != 1:
raise RuntimeError(
"Disfluency timeline lacks uncertainty/final components: "
f"{event['components']}"
)
def sample_to_frame(sample: int, *, ceil: bool) -> int:
value = int(sample) * self.motion_fps
frames = (
(value + self.audio_sampling_rate - 1)
// self.audio_sampling_rate
if ceil
else value // self.audio_sampling_rate
)
return int(speech_start_frame + frames)
uncertainty_start_sample = min(
int(component["start_sample"])
for component in uncertainty_components
)
final_component = final_components[0]
event.update(
{
"turn_index": int(turn_index),
"speech_start_frame": int(speech_start_frame),
"listen_start_frame": int(listen_start_frame),
"motion_start_frame": int(
listen_start_frame + listen_frames
),
"uncertainty_start_frame": sample_to_frame(
uncertainty_start_sample,
ceil=False,
),
"final_intent_start_frame": sample_to_frame(
int(final_component["start_sample"]),
ceil=False,
),
"final_intent_complete_frame": sample_to_frame(
int(final_component["end_sample"]),
ceil=True,
),
"expected_control_during_gap": LISTEN_TOKEN_ID,
}
)
disfluency_events.append(event)
entry_frames = (
self.turn_transition_frames
if bool(getattr(self, "causal_listen_timing", False))
else self.stationary_blend_frames
if turn_index == 0
else self.turn_transition_frames
)
task_sequence, entry_frames = self._build_task_sequence(
motion,
stationary_anchor=(
listen_sequence[-1]
if bool(getattr(self, "stationary_listen_hold", False)) and len(listen_sequence)
else anchor
),
transition_frames=entry_frames,
)
task_transition_frames.append(entry_frames)
task_original_frames = self._task_original_frame_indices(
sequence_frames=len(task_sequence),
motion_frames=len(motion),
transition_frames=entry_frames,
)
if (
interaction_mode == "backchannel"
and turn_index == session_backchannel_turn_index
):
voice_name = str(session_voice_name or self._voice_name(voice))
backchannel, backchannel_speech, backchannel_reaction_frames = (
self._sample_backchannel(voice_name)
)
backchannel_start = self._sample_backchannel_speech_start(
task_sequence,
reaction_frames=backchannel_reaction_frames,
)
backchannel_global_start = (
sum(len(part) for part in sequence_parts)
+ backchannel_start
)
backchannel_assignment = SourceBackgroundAssignment(
source_key=(
f"backchannel/{backchannel.split}/"
f"{backchannel.phrase_id}/{backchannel.voice_name}"
),
background_key="session-selected",
listen_start_index=0,
listen_window_count=self._runtime_listen_window_count(
backchannel.speech_samples
),
background_window_count=0,
initial_idle_max_steps=0,
max_motion_steps=0,
safety_steps=0,
original_audio_path=backchannel.waveform_path,
speech_samples=backchannel.speech_samples,
speech_scale=backchannel.speech_scale,
)
turn_audio.append(
{
"voice": None,
"assignment": backchannel_assignment,
"speech_start_frame": backchannel_global_start,
"listen_start_frame": backchannel_global_start,
"speech_waveform": backchannel_speech,
"speech_onset_sample": (
detect_clean_speech_onset_sample(
backchannel_speech,
self.audio_sampling_rate,
)
),
"listen_reaction_frames": (
backchannel_reaction_frames
),
}
)
backchannel_events.append(
{
"turn_index": int(turn_index),
"category": backchannel.category,
"phrase_id": backchannel.phrase_id,
"split": backchannel.split,
"voice_name": backchannel.voice_name,
"text": backchannel.text,
"speech_start_frame": int(
backchannel_global_start
),
"reaction_frames": int(
backchannel_reaction_frames
),
"source_motion_speech_start_frame": int(
backchannel_start
),
"source_motion_remaining_after_heard_frames": int(
len(task_sequence)
- backchannel_start
- backchannel_reaction_frames
),
"expected_control_id": MOTION_TOKEN_ID,
}
)
if (
interaction_mode in {"emergency-stop", "safe-hold"}
and (
session_control_turn_index is None
or turn_index == session_control_turn_index
)
):
voice_name = str(session_voice_name or self._voice_name(voice))
command, command_speech, command_reaction_frames = (
self._sample_control_command(
interaction_mode,
voice_name,
)
)
command_start, command_timing = (
self._sample_control_speech_start(
task_sequence,
reaction_frames=command_reaction_frames,
)
)
append_part(
task_sequence[:command_start],
control_label=MOTION_TOKEN_ID,
progress=_progress_rows(
task_original_frames[:command_start],
len(motion),
),
original_frames=task_original_frames[:command_start],
turn_index=turn_index,
)
command_global_start = sum(
len(part) for part in sequence_parts
)
command_assignment = SourceBackgroundAssignment(
source_key=(
f"control/{command.intent}/{command.phrase_id}/"
f"{command.voice_name}"
),
background_key="session-selected",
listen_start_index=0,
listen_window_count=self._runtime_listen_window_count(
command.speech_samples
),
background_window_count=0,
initial_idle_max_steps=0,
max_motion_steps=0,
safety_steps=0,
original_audio_path=command.waveform_path,
speech_samples=command.speech_samples,
speech_scale=command.speech_scale,
)
turn_audio.append(
{
"voice": None,
"assignment": command_assignment,
"speech_start_frame": command_global_start,
"listen_start_frame": command_global_start,
"speech_waveform": command_speech,
"speech_onset_sample": detect_clean_speech_onset_sample(
command_speech,
self.audio_sampling_rate,
),
"listen_reaction_frames": command_reaction_frames,
}
)
reaction_end = min(
command_start + command_reaction_frames,
len(task_sequence),
)
append_part(
task_sequence[command_start:reaction_end],
control_label=MOTION_TOKEN_ID,
progress=_progress_rows(
task_original_frames[command_start:reaction_end],
len(motion),
),
original_frames=task_original_frames[
command_start:reaction_end
],
turn_index=turn_index,
)
if reaction_end < command_start + command_reaction_frames:
missing = command_start + command_reaction_frames - reaction_end
append_part(
np.repeat(task_sequence[-1:], missing, axis=0),
control_label=MOTION_TOKEN_ID,
progress=self._completed_motion_progress_rows(missing),
original_frames=np.full(
missing,
len(motion) - 1,
dtype=np.int64,
),
turn_index=turn_index,
)
reaction_end = len(task_sequence)
if interaction_mode == "safe-hold":
hold_boundary = self._safe_hold_boundary(
task_sequence,
earliest_frame=reaction_end,
)
if hold_boundary > reaction_end:
append_part(
task_sequence[reaction_end:hold_boundary],
control_label=MOTION_TOKEN_ID,
progress=_progress_rows(
task_original_frames[
reaction_end:hold_boundary
],
len(motion),
),
original_frames=task_original_frames[
reaction_end:hold_boundary
],
turn_index=turn_index,
)
anchor = task_sequence[
min(max(hold_boundary - 1, 0), len(task_sequence) - 1)
].copy()
# If the natural stable point is not exactly on the 0.4 s
# grid, keep its real pose under </motion> until the next
# decision boundary; the first </idle> frame is identical.
alignment = (
-sum(len(part) for part in sequence_parts)
) % self.action_exec_horizon
if alignment:
append_part(
np.repeat(anchor[None, :], alignment, axis=0),
control_label=MOTION_TOKEN_ID,
progress=self._completed_motion_progress_rows(alignment),
original_frames=np.full(
alignment,
len(motion) - 1,
dtype=np.int64,
),
turn_index=turn_index,
)
next_anchor = anchor.copy()
recovery_transition_frames.append(0)
else:
recovery_boundary, next_anchor = (
self._nearest_safe_recovery_boundary(
task_sequence,
earliest_frame=reaction_end,
)
)
if recovery_boundary > reaction_end:
safe_tail = task_sequence[
reaction_end:recovery_boundary
]
append_part(
safe_tail,
control_label=IDLE_TOKEN_ID,
progress=self._stationary_progress_rows(
len(safe_tail)
),
original_frames=task_original_frames[
reaction_end:recovery_boundary
],
turn_index=turn_index,
)
stop_pose = task_sequence[recovery_boundary - 1].copy()
next_anchor = _align_task_motion_to_anchor_heading(
next_anchor[None, :],
stop_pose,
)[0]
recovery_frames = int(self.turn_transition_frames)
recovery_frames += (
-(
sum(len(part) for part in sequence_parts)
+ recovery_frames
)
) % self.action_exec_horizon
recovery = self._build_pose_transition(
stop_pose,
next_anchor,
recovery_frames + 1,
)[1:]
append_part(
recovery,
control_label=IDLE_TOKEN_ID,
progress=self._stationary_progress_rows(len(recovery)),
original_frames=np.full(
len(recovery),
int(task_original_frames[recovery_boundary - 1]),
dtype=np.int64,
),
turn_index=turn_index,
)
recovery_transition_frames.append(
int(recovery_boundary - reaction_end + len(recovery))
)
has_followup = turn_index + 1 < len(session)
idle_steps = self._sample_control_post_idle_steps(
interaction_mode,
has_followup=has_followup,
)
if has_followup:
inter_turn_idle_steps.append(idle_steps)
idle_turn_index = turn_index + 1
else:
terminal_idle_steps = idle_steps
idle_turn_index = turn_index
idle_frames = idle_steps * self.action_exec_horizon
if interaction_mode == "safe-hold":
post_control_idle = np.repeat(
next_anchor[None, :],
idle_frames,
axis=0,
)
idle_name = "held_actual_stable_pose"
else:
post_control_idle, idle_name = (
self._sample_stationary_sequence(
"idle",
frame_count=idle_frames,
anchor_pose=next_anchor,
)
)
idle_motion_names.append(idle_name)
append_part(
post_control_idle,
control_label=IDLE_TOKEN_ID,
progress=self._stationary_progress_rows(idle_frames),
original_frames=np.zeros(idle_frames, dtype=np.int64),
turn_index=idle_turn_index,
)
control_command_events.append(
{
"turn_index": int(turn_index),
"intent": command.intent,
"phrase_id": command.phrase_id,
"voice_name": command.voice_name,
"text": command.text,
"speech_start_frame": int(command_global_start),
"reaction_frames": int(command_reaction_frames),
"source_motion_speech_start_frame": int(command_start),
"source_motion_response_frame": int(
command_start + command_reaction_frames
),
"source_motion_remaining_at_speech_start_frames": int(
len(task_sequence) - command_start
),
"source_motion_remaining_at_response_frames": int(
len(task_sequence)
- command_start
- command_reaction_frames
),
**command_timing,
"held_original_pose": interaction_mode == "safe-hold",
}
)
anchor = next_anchor
continue
if (
interaction_mode == "interruption"
and turn_index + 1 < len(session)
):
prefix_frames = self._sample_interruption_prefix_frames(
task_sequence,
next_reaction_frames=turn_reaction_frames[turn_index + 1],
)
append_part(
task_sequence[:prefix_frames],
control_label=MOTION_TOKEN_ID,
progress=_progress_rows(
task_original_frames[:prefix_frames],
len(motion),
),
original_frames=task_original_frames[:prefix_frames],
turn_index=turn_index,
)
interruption_start_frames.append(
sum(len(part) for part in sequence_parts)
)
interruption_original_frames.append(
int(task_original_frames[prefix_frames - 1])
)
interruption_remaining_frames.append(
int(len(task_sequence) - prefix_frames)
)
pending_task_sequence = task_sequence
pending_task_original_frames = task_original_frames
pending_task_cursor = prefix_frames
anchor = task_sequence[prefix_frames - 1].copy()
continue
append_part(
task_sequence,
control_label=MOTION_TOKEN_ID,
progress=_progress_rows(task_original_frames, len(motion)),
original_frames=task_original_frames,
turn_index=turn_index,
)
# Complete the task, then settle into a new neutral stance at the
# task's actual final XY/yaw. The transition length is extended by
# at most 19 frames so the next token boundary lands exactly on a
# Qwen update instead of becoming impossible scalar supervision.
task_end = task_sequence[-1]
next_anchor = self._sample_stationary_anchor(task_end)
next_anchor = _align_task_motion_to_anchor_heading(
next_anchor[None, :],
task_end,
)[0]
timeline_length = sum(len(part) for part in sequence_parts)
recovery_frames = self.turn_transition_frames
recovery_frames += (
-(
timeline_length
+ recovery_frames
- 1
)
) % self.action_exec_horizon
recovery = self._build_pose_transition(
task_end,
next_anchor,
recovery_frames,
)[1:]
recovery_transition_frames.append(int(recovery_frames))
append_part(
recovery,
control_label=MOTION_TOKEN_ID,
progress=self._completed_motion_progress_rows(len(recovery)),
original_frames=np.full(
len(recovery),
len(motion) - 1,
dtype=np.int64,
),
turn_index=turn_index,
)
if sum(len(part) for part in sequence_parts) % self.action_exec_horizon:
raise AssertionError("Recovery did not align the Qwen boundary")
if turn_index + 1 < len(session):
idle_steps = self._sample_interturn_idle_steps()
inter_turn_idle_steps.append(idle_steps)
idle_turn_index = turn_index + 1
else:
idle_steps = random.randint(
1,
max(self.initial_idle_max_steps, 1),
)
terminal_idle_steps = idle_steps
idle_turn_index = turn_index
idle_frames = idle_steps * self.action_exec_horizon
turn_idle, turn_idle_name = self._sample_stationary_sequence(
"idle",
frame_count=idle_frames,
anchor_pose=next_anchor,
)
idle_motion_names.append(turn_idle_name)
append_part(
turn_idle,
control_label=IDLE_TOKEN_ID,
progress=self._stationary_progress_rows(idle_frames),
original_frames=np.zeros(idle_frames, dtype=np.int64),
turn_index=idle_turn_index,
)
anchor = next_anchor
if pending_task_sequence is not None:
raise AssertionError("Final interruption motion was left pending")
full_sequence = np.concatenate(sequence_parts, axis=0)
timeline_control_labels = np.concatenate(control_parts, axis=0)
timeline_progress = np.concatenate(progress_parts, axis=0)
timeline_original_frames = np.concatenate(
original_frame_parts,
axis=0,
)
timeline_turn_indices = np.concatenate(turn_index_parts, axis=0)
if not (
len(full_sequence)
== len(timeline_control_labels)
== len(timeline_progress)
== len(timeline_original_frames)
== len(timeline_turn_indices)
):
raise AssertionError("Multi-turn frame timelines have different lengths")
valid_dense_starts = max(
len(full_sequence) - self.prefix_frames - 1,
0,
)
semantic_window_count = int(
math.ceil(valid_dense_starts / self.action_exec_horizon)
)
if semantic_window_count <= 0:
raise RuntimeError("Multi-turn session is too short for RTC")
if (
self.max_stream_steps > 0
and semantic_window_count > self.max_stream_steps
):
raise RuntimeError(
"Session exceeds the complete-KV context guard: "
f"windows={semantic_window_count} limit={self.max_stream_steps}"
)
turn_audio.sort(key=lambda payload: int(payload["speech_start_frame"]))
normal_turn_audio = [
payload for payload in turn_audio if payload["voice"] is not None
]
if len(normal_turn_audio) != len(session):
raise AssertionError(
"Normal instruction audio metadata drifted from session turns: "
f"audio={len(normal_turn_audio)} turns={len(session)}"
)
audio_windows, background_key, background_start = (
self._build_session_audio_windows(
turn_audio=turn_audio,
semantic_window_count=semantic_window_count,
)
)
row_ids = [int(record["row_id"]) for record in session]
source_id = (
f"session{int(index):06d}::rows"
+ "-".join(f"{row_id:06d}" for row_id in row_ids)
)
source_id += f"::mode={interaction_mode}"
if session_voice_name is not None:
source_id += (
f"::voice={session_voice_name}"
f"::variant={int(session_voice_variant)}"
)
steps: list[dict[str, Any]] = []
action_phase_counts = {"idle": 0, "listen": 0, "motion": 0}
phase_names = {
IDLE_TOKEN_ID: "idle",
LISTEN_TOKEN_ID: "listen",
MOTION_TOKEN_ID: "motion",
}
final_start = len(full_sequence) - self.prefix_frames - 2
for semantic_index, semantic_input in enumerate(audio_windows):
target_base = semantic_index * self.action_exec_horizon
stop = min(
target_base + self.action_exec_horizon,
final_start + 1,
)
if stop <= target_base:
break
for target_start in range(
target_base,
stop,
self.action_sample_step_frames,
):
action_substep = target_start - target_base
update_semantic_cache = action_substep == 0
(
actions,
action_mask,
state,
reference,
smoothness_context_actions,
smoothness_context_reference_state,
) = self._stream_action_rows(
full_sequence,
start=target_start,
)
action_indices = np.arange(
target_start + 1,
target_start + 1 + self.action_chunk_size,
dtype=np.int64,
)
clipped = np.clip(
action_indices,
0,
len(full_sequence) - 1,
)
control_id = int(timeline_control_labels[target_start])
phase = phase_names[control_id]
active_turn = int(timeline_turn_indices[target_start])
semantic_kwargs: dict[str, Any] = {}
if update_semantic_cache:
semantic_kwargs["audio_waveform"] = semantic_input
step = self._raw_step(
**semantic_kwargs,
actions=actions,
action_mask=action_mask,
state=state,
action_reference_state=reference,
smoothness_context_actions=smoothness_context_actions,
smoothness_context_reference_state=(
smoothness_context_reference_state
),
action_control_labels=timeline_control_labels[clipped],
future_progress=timeline_progress[clipped],
control_progress=(
timeline_progress[target_start - 1]
if target_start > 0
else self._stationary_progress_rows(1)[0]
),
previous_control_label=(
int(timeline_control_labels[target_start - 1])
if target_start > 0
else 3
),
phase=phase,
source_id=source_id,
step_index=len(steps),
frame_id=int(timeline_original_frames[target_start]),
semantic_frame_id=target_base,
semantic_window_index=semantic_index,
action_substep=action_substep,
record=session[active_turn],
instruction_text=instruction_text_for_voice(
session[active_turn],
voices[active_turn],
),
instruction_history_texts=tuple(
instruction_text_for_voice(
session[history_turn],
voices[history_turn],
)
for history_turn in range(active_turn)
),
update_semantic_cache=update_semantic_cache,
control_transition=bool(
update_semantic_cache
and target_start > 0
and timeline_control_labels[target_start]
!= timeline_control_labels[target_start - 1]
),
)
step["turn_index"] = active_turn
step["interaction_mode"] = interaction_mode
step["background_window_index"] = (
background_start + semantic_index
)
steps.append(step)
action_phase_counts[phase] += 1
listen_steps = sum(voice.window_count for voice in voices)
semantic_phase_labels = timeline_control_labels[
np.arange(semantic_window_count) * self.action_exec_horizon
]
duration_targets = [
step
for step in steps
if bool(step["motion_duration_target_mask"])
]
duration_target_turns = [
int(step["turn_index"])
for step in duration_targets
]
expected_duration_turns = list(range(len(session)))
if duration_target_turns != expected_duration_turns:
raise RuntimeError(
"Each multi-turn primitive must expose exactly one causal "
"motion-duration target before transforms: "
f"source={source_id} expected_turns={expected_duration_turns} "
f"actual_turns={duration_target_turns}"
)
text_targets = [
step
for step in steps
if bool(step["latest_instruction_target_mask"])
]
text_target_turns = [int(step["turn_index"]) for step in text_targets]
if text_target_turns != expected_duration_turns:
raise RuntimeError(
"Each multi-turn primitive must expose exactly one causal "
"latest-instruction text target: "
f"source={source_id} expected_turns={expected_duration_turns} "
f"actual_turns={text_target_turns}"
)
if any(
step["latest_instruction_text"]
!= instruction_text_for_voice(session[turn], voices[turn])
for step, turn in zip(text_targets, expected_duration_turns)
):
raise RuntimeError(
f"Voice-specific instruction text drifted in {source_id}"
)
memory_targets = [
step
for step in steps
if bool(step["instruction_memory_target_mask"])
]
expected_memory_turns = list(range(1, len(session)))
memory_target_turns = [
int(step["turn_index"]) for step in memory_targets
]
if memory_target_turns != expected_memory_turns:
raise RuntimeError(
"Each non-initial multi-turn primitive must expose exactly "
"one historical-instruction target: "
f"source={source_id} expected_turns={expected_memory_turns} "
f"actual_turns={memory_target_turns}"
)
expected_memory_slots = [
"previous" if turn == 1 or turn % 2 == 1 else "first"
for turn in expected_memory_turns
]
expected_memory_texts = [
instruction_text_for_voice(
session[turn - 1 if slot == "previous" else 0],
voices[turn - 1 if slot == "previous" else 0],
)
for turn, slot in zip(
expected_memory_turns,
expected_memory_slots,
)
]
if any(
step["instruction_memory_slot"] != expected_slot
or step["instruction_memory_text"] != expected_text
for step, expected_slot, expected_text in zip(
memory_targets,
expected_memory_slots,
expected_memory_texts,
)
):
raise RuntimeError(
f"Historical-instruction target drifted in {source_id}"
)
return {
"source_id": source_id,
"source_audio_key": "+".join(voice.source_key for voice in voices),
"manifest_row_id": row_ids[0],
"turn_count": len(session),
"turn_row_ids": row_ids,
"turn_motion_names": [record["filename"] for record in session],
"turn_audio_keys": [voice.source_key for voice in voices],
"session_voice_name": session_voice_name,
"session_voice_variant": session_voice_variant,
"interaction_mode": interaction_mode,
# Exact microphone ledger for independent VQA overlays. The
# microphone has a 0.2 s prefix before the action frame clock.
"microphone_speech_intervals": [
{
"start_seconds": 0.2 + int(payload["speech_start_frame"]) / self.motion_fps,
"end_seconds": 0.2 + int(payload["speech_start_frame"]) / self.motion_fps
+ int(payload["assignment"].speech_samples) / self.audio_sampling_rate,
}
for payload in turn_audio
],
"turn_listen_start_frames": [
int(payload["listen_start_frame"])
for payload in normal_turn_audio
],
"turn_speech_start_frames": [
int(payload["speech_start_frame"])
for payload in normal_turn_audio
],
"turn_speech_onset_samples": speech_onset_samples,
"turn_listen_reaction_frames": listen_reaction_frames,
"turn_listen_reaction_idle_motion_names": (
listen_reaction_idle_motion_names
),
"listen_transition_frames": (
self.turn_transition_frames
if bool(getattr(self, "causal_listen_timing", False))
else self.stationary_blend_frames
),
"motion_name": "+".join(record["filename"] for record in session),
"idle_motion_name": idle_motion_names,
"listen_motion_name": listen_motion_names,
"task_transition_frames": task_transition_frames,
"recovery_transition_frames": recovery_transition_frames,
"inter_turn_idle_steps": inter_turn_idle_steps,
"interruption_start_frames": interruption_start_frames,
"interruption_original_frames": interruption_original_frames,
"interruption_remaining_frames": interruption_remaining_frames,
"interruption_terminal_hold_frames": (
interruption_terminal_hold_frames
),
"control_command_events": control_command_events,
"backchannel_events": backchannel_events,
"backchannel_turn_index": session_backchannel_turn_index,
"disfluency_events": disfluency_events,
"disfluency_turn_index": session_disfluency_turn_index,
"disfluency_subtype": session_disfluency_subtype,
"control_turn_index": session_control_turn_index,
"control_has_followup": session_control_has_followup,
"terminal_idle_steps": int(terminal_idle_steps),
"white_noise_start_index": background_start,
"background_key": background_key,
"background_listen_start_index": (
background_start
+ int(normal_turn_audio[0]["listen_start_frame"])
// self.action_exec_horizon
),
"idle_steps": int(
np.count_nonzero(semantic_phase_labels == IDLE_TOKEN_ID)
),
"listen_steps": int(listen_steps),
"motion_steps": int(
np.count_nonzero(semantic_phase_labels == MOTION_TOKEN_ID)
),
"semantic_steps": semantic_window_count,
"action_idle_steps": action_phase_counts["idle"],
"action_listen_steps": action_phase_counts["listen"],
"action_motion_steps": action_phase_counts["motion"],
"action_steps": len(steps),
"steps": steps,
}
def _build_single_source(self, index: int) -> dict[str, Any]:
record = dict(self.records[int(index)])
# ``getattr`` keeps lightweight unit fixtures and the retained legacy
# cached-hidden compatibility path valid; formal HFDv2 instances set
# this explicitly in ``__init__``.
online_audio = bool(getattr(self, "online_audio", False))
motion = self._load_motion(record["motion_dir"])
if motion.shape[0] <= 0:
raise RuntimeError(f"Empty motion: {record['motion_dir']}")
record["actual_frames"] = int(motion.shape[0])
task_start_pose = motion[0].copy()
stationary_anchor = self._sample_stationary_anchor(task_start_pose)
task_sequence, task_transition_frames = self._build_task_sequence(
motion,
stationary_anchor=stationary_anchor,
)
voices = self._record_voices(record)
voice = self._select_voice(voices, row_id=int(record["row_id"]))
source_id = f"row{record['row_id']:06d}::{voice.source_key}"
steps: list[dict[str, Any]] = []
idle_step_count = random.randint(0, self.initial_idle_max_steps)
listen_step_count = voice.window_count
continuous_background = bool(
getattr(self, "continuous_background", False)
)
assignment = (
self.background_assignments[voice.source_key]
if continuous_background
else None
)
noise_start_index = (
int(assignment.listen_start_index)
if assignment is not None
else random.randrange(len(self.white_noise_hidden))
)
# One dense row per current state, with the v26 validity rule above.
motion_action_step_count = max(
int(task_sequence.shape[0]) - self.prefix_frames - 1,
0,
)
if self.max_motion_steps > 0:
motion_action_step_count = min(
motion_action_step_count,
self.max_motion_steps * self.action_exec_horizon,
)
motion_step_count = int(
math.ceil(motion_action_step_count / self.action_exec_horizon)
)
if motion_step_count <= 0:
raise RuntimeError(
"Task motion is too short for one complete 30-frame RTC target: "
f"source={source_id} frames={task_sequence.shape[0]}"
)
if (
assignment is not None
and motion_step_count > assignment.max_motion_steps
):
raise RuntimeError(
"Motion is longer than its continuous-background reservation: "
f"source={voice.source_key} actual_steps={motion_step_count} "
f"reserved={assignment.max_motion_steps}"
)
if assignment is not None:
if online_audio:
idle_background_inputs = (
self._load_background_waveform_window_block(
assignment.background_key,
assignment.listen_start_index - idle_step_count,
idle_step_count,
)
)
motion_background_inputs = (
self._load_background_waveform_window_block(
assignment.background_key,
assignment.listen_start_index + listen_step_count,
motion_step_count,
)
)
else:
idle_background_inputs = self._load_background_window_block(
assignment.background_key,
assignment.listen_start_index - idle_step_count,
idle_step_count,
)
motion_background_inputs = self._load_background_window_block(
assignment.background_key,
assignment.listen_start_index + listen_step_count,
motion_step_count,
)
else:
idle_background_inputs = ()
motion_background_inputs = ()
if online_audio:
if assignment is None:
raise RuntimeError(
"Dense online audio requires a continuous background "
f"assignment: {voice.source_key}"
)
voice_inputs: tuple[torch.Tensor | np.ndarray, ...] = (
self._load_mixed_waveform_window_block(
voice,
assignment,
)
)
else:
voice_inputs = self._load_voice_hidden(voice)
if len(voice_inputs) != listen_step_count:
raise RuntimeError(
f"Semantic window count mismatch for {voice.source_key}: "
f"{len(voice_inputs)} != {listen_step_count}"
)
total_stream_steps = (
idle_step_count + listen_step_count + motion_step_count
)
max_stream_steps = int(getattr(self, "max_stream_steps", 0))
if max_stream_steps > 0 and total_stream_steps > max_stream_steps:
raise RuntimeError(
"Source exceeds the configured complete-KV context guard: "
f"source={source_id} windows={total_stream_steps} "
f"limit={max_stream_steps}. The source was expected to "
"be removed by the worst-case construction filter."
)
idle_sequence, idle_motion_name = self._sample_stationary_sequence(
"idle",
frame_count=idle_step_count * self.action_exec_horizon,
anchor_pose=stationary_anchor,
)
listen_sequence, listen_motion_name = self._sample_stationary_sequence(
"listen",
frame_count=listen_step_count * self.action_exec_horizon,
anchor_pose=stationary_anchor,
)
if bool(getattr(self, "stationary_listen_hold", False)) and len(listen_sequence):
task_sequence, task_transition_frames = self._build_task_sequence(
motion, stationary_anchor=listen_sequence[-1],
)
pretask_sequence = np.concatenate(
[idle_sequence, listen_sequence],
axis=0,
)
full_sequence = np.concatenate(
[pretask_sequence, task_sequence],
axis=0,
)
timeline_control_labels = np.concatenate(
[
np.full(
len(idle_sequence),
IDLE_TOKEN_ID,
dtype=np.int64,
),
np.full(
len(listen_sequence),
LISTEN_TOKEN_ID,
dtype=np.int64,
),
np.full(
len(task_sequence),
MOTION_TOKEN_ID,
dtype=np.int64,
),
]
)
if len(timeline_control_labels) != len(full_sequence):
raise RuntimeError(
"Full-duplex frame/token timeline length mismatch: "
f"tokens={len(timeline_control_labels)} "
f"motion={len(full_sequence)}"
)
task_original_frames = self._task_original_frame_indices(
sequence_frames=int(task_sequence.shape[0]),
motion_frames=int(motion.shape[0]),
transition_frames=task_transition_frames,
)
timeline_original_frames = np.concatenate(
[
np.zeros(len(pretask_sequence), dtype=np.int64),
task_original_frames,
]
)
action_phase_counts = {"idle": 0, "listen": 0, "motion": 0}
semantic_window_index = 0
def append_dense_group(
*,
phase: str,
semantic_input: torch.Tensor | np.ndarray,
target_base: int,
semantic_frame_id: int,
background_index: int,
) -> None:
nonlocal semantic_window_index
# ``target_base`` indexes the current state. The last allowed
# state leaves one real target after a maximal RTC prefix.
final_start = len(full_sequence) - self.prefix_frames - 2
stop = min(
int(target_base) + self.action_exec_horizon,
final_start + 1,
)
if stop <= int(target_base):
raise RuntimeError(
"Semantic window has no complete dense RTC action target: "
f"source={source_id} phase={phase} base={target_base} "
f"timeline={len(full_sequence)}"
)
for target_start in range(
int(target_base),
stop,
int(getattr(self, "action_sample_step_frames", 1)),
):
action_substep = target_start - int(target_base)
update_semantic_cache = action_substep == 0
(
actions,
action_mask,
state,
action_reference_state,
smoothness_context_actions,
smoothness_context_reference_state,
) = self._stream_action_rows(full_sequence, start=target_start)
semantic_kwargs: dict[str, Any] = {}
if update_semantic_cache:
semantic_kwargs = (
{"audio_waveform": semantic_input}
if online_audio
else {"hidden_states": semantic_input}
)
action_indices = np.arange(
int(target_start) + 1,
int(target_start) + 1 + self.action_chunk_size,
dtype=np.int64,
)
action_control_labels = timeline_control_labels[
np.clip(action_indices, 0, len(timeline_control_labels) - 1)
]
progress_indices = timeline_original_frames[
np.clip(action_indices, 0, len(timeline_original_frames) - 1)
]
step = self._raw_step(
**semantic_kwargs,
actions=actions,
action_mask=action_mask,
state=state,
action_reference_state=action_reference_state,
smoothness_context_actions=smoothness_context_actions,
smoothness_context_reference_state=(
smoothness_context_reference_state
),
action_control_labels=action_control_labels,
future_progress=_progress_rows(
progress_indices,
int(motion.shape[0]),
),
control_progress=(
_progress_rows(
[int(timeline_original_frames[target_start - 1])],
int(motion.shape[0]),
)[0]
if target_start > 0
and int(timeline_control_labels[target_start - 1])
== MOTION_TOKEN_ID
else self._stationary_progress_rows(1)[0]
),
previous_control_label=(
int(timeline_control_labels[target_start - 1])
if target_start > 0
else 3
),
phase=phase,
source_id=source_id,
step_index=len(steps),
frame_id=(
int(
timeline_original_frames[
min(int(target_start), len(timeline_original_frames) - 1)
]
)
if phase == "motion"
else int(target_start)
),
semantic_frame_id=int(semantic_frame_id),
semantic_window_index=semantic_window_index,
action_substep=action_substep,
record=record,
instruction_text=instruction_text_for_voice(record, voice),
update_semantic_cache=update_semantic_cache,
control_transition=bool(
update_semantic_cache
and int(target_start) > 0
and timeline_control_labels[int(target_start)]
!= timeline_control_labels[int(target_start) - 1]
),
)
step["background_window_index"] = int(background_index)
steps.append(step)
action_phase_counts[phase] += 1
semantic_window_index += 1
for idle_step in range(idle_step_count):
target_base = idle_step * self.action_exec_horizon
if assignment is not None:
background_index = (
assignment.listen_start_index
- idle_step_count
+ idle_step
)
semantic_input = idle_background_inputs[idle_step]
else:
background_index = (
noise_start_index + idle_step
) % len(self.white_noise_hidden)
semantic_input = self.white_noise_hidden[background_index]
append_dense_group(
phase="idle",
semantic_input=semantic_input,
target_base=target_base,
semantic_frame_id=target_base,
background_index=background_index,
)
for clip_index, semantic_input in enumerate(voice_inputs):
semantic_frame_id = (
voice.clips[clip_index].frame_id
if (
voice.waveform_path is None
and voice.packed_path is None
)
else clip_index * self.audio_sample_step_frames
)
target_base = (
idle_step_count + clip_index
) * self.action_exec_horizon
background_index = noise_start_index + clip_index
append_dense_group(
phase="listen",
semantic_input=semantic_input,
target_base=target_base,
semantic_frame_id=semantic_frame_id,
background_index=background_index,
)
for motion_step in range(motion_step_count):
target_base = (
len(pretask_sequence)
+ motion_step * self.action_exec_horizon
)
if assignment is not None:
background_index = (
assignment.listen_start_index
+ listen_step_count
+ motion_step
)
semantic_input = motion_background_inputs[motion_step]
else:
background_index = (
noise_start_index + idle_step_count + motion_step
) % len(self.white_noise_hidden)
semantic_input = self.white_noise_hidden[background_index]
append_dense_group(
phase="motion",
semantic_input=semantic_input,
target_base=target_base,
semantic_frame_id=motion_step * self.action_exec_horizon,
background_index=background_index,
)
duration_targets = [
step
for step in steps
if bool(step["motion_duration_target_mask"])
]
if len(duration_targets) != 1:
raise RuntimeError(
"Each single-turn source must expose exactly one causal "
"motion-duration target before transforms: "
f"source={source_id} actual={len(duration_targets)}"
)
text_targets = [
step
for step in steps
if bool(step["latest_instruction_target_mask"])
]
if len(text_targets) != 1 or text_targets[0][
"latest_instruction_text"
] != instruction_text_for_voice(record, voice):
raise RuntimeError(
"Each single-turn source must expose exactly one exact, "
f"voice-specific text target: source={source_id}"
)
return {
"source_id": source_id,
"source_audio_key": voice.source_key,
"manifest_row_id": int(record["row_id"]),
"motion_name": record["filename"],
"idle_motion_name": idle_motion_name,
"listen_motion_name": listen_motion_name,
"task_transition_frames": task_transition_frames,
"white_noise_start_index": noise_start_index,
"background_key": (
assignment.background_key
if assignment is not None
else "legacy-cycle"
),
"background_listen_start_index": noise_start_index,
"idle_steps": idle_step_count,
"listen_steps": listen_step_count,
"motion_steps": motion_step_count,
"semantic_steps": (
idle_step_count + listen_step_count + motion_step_count
),
"action_idle_steps": action_phase_counts["idle"],
"action_listen_steps": action_phase_counts["listen"],
"action_motion_steps": action_phase_counts["motion"],
"action_steps": len(steps),
"steps": steps,
}
class FullDuplexTransformedDataset(torch.utils.data.Dataset):
"""Apply the existing SEED TextOp/normalisation path step by step."""
_STREAM_KEYS = (
"source_id",
"source_audio_key",
"phase",
"control_label",
"control_token",
"control_transition",
"action_control_labels",
"update_semantic_cache",
"step_index",
"frame_id",
"semantic_frame_id",
"semantic_window_index",
"action_substep",
"turn_index",
"interaction_mode",
# SeedPosttrainRepackTransform deliberately builds a compact new
# dictionary. Duration supervision is stream metadata, so restore it
# explicitly just like control labels instead of silently losing every
# target before the trainer sees it.
"motion_duration_target_seconds",
"motion_duration_target_mask",
"latest_instruction_text",
"latest_instruction_target_mask",
"instruction_memory_text",
"instruction_memory_slot",
"instruction_memory_target_mask",
# Dimensionless causal progress and the discrete previous-token ID
# bypass the floating-point action/state transforms and are restored
# verbatim below.
"control_progress",
"previous_control_label",
)
def __init__(self, raw_dataset: FullDuplexAudioCommandDataset, transform, transform_kwargs=None):
self.raw_dataset = raw_dataset
self.transform = transform
self.transform_kwargs = dict(transform_kwargs or {})
def __len__(self) -> int:
return len(self.raw_dataset)
def estimated_stream_steps(self, index: int) -> int:
return self.raw_dataset.estimated_stream_steps(index)
def source_modality(self, index: int) -> str:
method = getattr(self.raw_dataset, "source_modality", None)
return method(index) if method is not None else "audio_action"
def __getitem__(self, index: int) -> dict[str, Any]:
source = self.raw_dataset[index]
transformed_steps = []
for raw_step in source["steps"]:
if bool(raw_step["update_semantic_cache"]):
transformed = self.transform(
dict(raw_step),
**self.transform_kwargs,
)
else:
# Repack and normalize every dense action target, but do not
# run the expensive Qwen audio processor for the 19 samples
# that deliberately reuse the preceding semantic condition.
transformed = self.transform.repack(
dict(raw_step),
**self.transform_kwargs,
)
transformed = self.transform.field(
transformed,
**self.transform_kwargs,
)
for key in self._STREAM_KEYS:
if key == "control_transition":
transformed[key] = bool(
raw_step.get("control_transition", False)
)
elif key == "action_control_labels":
transformed[key] = raw_step.get(
"action_control_labels",
np.full(
len(raw_step["actions"]),
int(raw_step["control_label"]),
dtype=np.int64,
),
)
elif key == "turn_index":
transformed[key] = int(raw_step.get("turn_index", 0))
elif key == "interaction_mode":
transformed[key] = str(
raw_step.get("interaction_mode", "turn-taking")
)
elif key == "motion_duration_target_seconds":
transformed[key] = float(raw_step[key])
elif key == "motion_duration_target_mask":
transformed[key] = bool(raw_step[key])
elif key == "latest_instruction_text":
transformed[key] = str(
raw_step.get("latest_instruction_text", "")
)
elif key == "latest_instruction_target_mask":
transformed[key] = bool(
raw_step.get("latest_instruction_target_mask", False)
)
elif key == "instruction_memory_text":
transformed[key] = str(
raw_step.get("instruction_memory_text", "")
)
elif key == "instruction_memory_slot":
transformed[key] = str(
raw_step.get("instruction_memory_slot", "")
)
elif key == "instruction_memory_target_mask":
transformed[key] = bool(
raw_step.get(
"instruction_memory_target_mask",
False,
)
)
elif key == "control_progress":
transformed[key] = np.asarray(
raw_step.get(
"control_progress",
raw_step.get(
"progress",
np.asarray(
[0.0, 1.0, 0.0, 0.0, 1.0],
dtype=np.float32,
),
),
),
dtype=np.float32,
).reshape(5)
elif key == "previous_control_label":
transformed[key] = int(
raw_step.get("previous_control_label", 3)
)
else:
transformed[key] = raw_step[key]
transformed_steps.append(transformed)
raw_duration_targets = sum(
bool(step["motion_duration_target_mask"])
for step in source["steps"]
)
transformed_duration_targets = sum(
bool(step["motion_duration_target_mask"])
for step in transformed_steps
)
if transformed_duration_targets != raw_duration_targets:
raise RuntimeError(
"Motion-duration supervision changed during data transforms: "
f"source={source['source_id']} raw={raw_duration_targets} "
f"transformed={transformed_duration_targets}"
)
raw_text_targets = sum(
bool(step.get("latest_instruction_target_mask", False))
for step in source["steps"]
)
transformed_text_targets = sum(
bool(step["latest_instruction_target_mask"])
for step in transformed_steps
)
if transformed_text_targets != raw_text_targets:
raise RuntimeError(
"Latest-instruction text supervision changed during data "
f"transforms: source={source['source_id']} "
f"raw={raw_text_targets} transformed={transformed_text_targets}"
)
raw_memory_targets = sum(
bool(step.get("instruction_memory_target_mask", False))
for step in source["steps"]
)
transformed_memory_targets = sum(
bool(step["instruction_memory_target_mask"])
for step in transformed_steps
)
if transformed_memory_targets != raw_memory_targets:
raise RuntimeError(
"Historical-instruction supervision changed during data "
f"transforms: source={source['source_id']} "
f"raw={raw_memory_targets} "
f"transformed={transformed_memory_targets}"
)
result = {**source, "steps": transformed_steps}
if "multimodal_overlay" in result:
from psi.data.seed.live_multimodal_overlay import attach_live_packets
return attach_live_packets(result)
return result
from __future__ import annotations
import hashlib
import json
import math
import random
import re
import sqlite3
import wave
from collections import OrderedDict
from dataclasses import dataclass
from pathlib import Path
from typing import Any, Iterable
import numpy as np
import pandas as pd
import torch
from safetensors import safe_open
from safetensors.torch import load_file
from scipy.signal import resample_poly
from psi.data.seed.omni_hidden_cache import (
load_hidden_states_slice,
load_hidden_states_source,
)
from psi.data.seed.motion_repr import (
estimate_two_foot_contacts,
quat_mul_wxyz,
quat_to_yaw_wxyz,
rpy_to_quat_wxyz,
wrap_angle,
)
from psi.data.seed.source_split import motion_source_id, select_source_row_ids
from psi.data.seed.storage_paths import relocate_hfd_storage_path
LISTEN_TOKEN_ID = 0
MOTION_TOKEN_ID = 1
IDLE_TOKEN_ID = 2
CONTROL_TOKEN_NAMES = ("</listen>", "</motion>", "</idle>")
CANONICAL_MOTION_MANIFEST_FORMAT = "hfd_canonical_motion_per_instruction_v1"
HESITATION_PAUSE_MANIFEST_FORMAT = "hfd_hesitation_pause_tts_v1"
DISFLUENCY_UNCERTAINTY_ROLES = frozenset(
{
"background-only-pause",
"hesitation-background-before",
"hesitation-filler",
"hesitation-background-after",
"repair-background-before",
"repair-cue",
"repair-background-after",
"repair-implicit-background",
}
)
# The continuous-audio corpus is exported at the deployment cadence: one
# 0.6-second window every 20 motion frames (0.4 seconds).
EXPORTED_AUDIO_STRIDE_FRAMES = 20
# These are deliberately exact metadata labels, not fuzzy keyword matches.
# SEED contains unsafe names such as idle_handstand, injured_*_idle and
# crouch_*_idle. The selected labels were manually audited against every
# natural-language description and kinematically checked for root drift.
LISTEN_MOTION_CONTENT_NAMES = frozenset({"listening_R"})
IDLE_MOTION_CONTENT_NAMES = frozenset({"neutral_idle"})
class HiddenSlice:
frame_id: int
shard_path: Path
offset: int
length: int
class AudioSource:
source_key: str
clips: tuple[HiddenSlice, ...]
packed_path: Path | None = None
packed_count: int = 0
# HFDv2 reads the exact continuous mixed waveform online. This field is
# mutually exclusive with the two cached-hidden layouts above.
waveform_path: Path | None = None
waveform_count: int = 0
def window_count(self) -> int:
if self.waveform_path is not None:
return int(self.waveform_count)
return (
int(self.packed_count)
if self.packed_path is not None
else len(self.clips)
)
def instruction_text_for_voice(
record: dict[str, Any],
voice: AudioSource,