code-HFD / src /psi /data /seed /full_duplex_audio_command_dataset.py
Tinghong-Ye22's picture
Add files using upload-large-folder tool
06f9561 verified
Raw History Blame Contribute Delete
367 kB
from __future__ import annotations import hashlib import json import math import random import re import sqlite3 import wave from collections import OrderedDict from dataclasses import dataclass from pathlib import Path from typing import Any, Iterable import numpy as np import pandas as pd import torch from safetensors import safe_open from safetensors.torch import load_file from scipy.signal import resample_poly from psi.data.seed.omni_hidden_cache import ( load_hidden_states_slice, load_hidden_states_source, ) from psi.data.seed.motion_repr import ( estimate_two_foot_contacts, quat_mul_wxyz, quat_to_yaw_wxyz, rpy_to_quat_wxyz, wrap_angle, ) from psi.data.seed.source_split import motion_source_id, select_source_row_ids from psi.data.seed.storage_paths import relocate_hfd_storage_path LISTEN_TOKEN_ID = 0 MOTION_TOKEN_ID = 1 IDLE_TOKEN_ID = 2 CONTROL_TOKEN_NAMES = ("</listen>", "</motion>", "</idle>") CANONICAL_MOTION_MANIFEST_FORMAT = "hfd_canonical_motion_per_instruction_v1" HESITATION_PAUSE_MANIFEST_FORMAT = "hfd_hesitation_pause_tts_v1" DISFLUENCY_UNCERTAINTY_ROLES = frozenset( { "background-only-pause", "hesitation-background-before", "hesitation-filler", "hesitation-background-after", "repair-background-before", "repair-cue", "repair-background-after", "repair-implicit-background", } ) # The continuous-audio corpus is exported at the deployment cadence: one # 0.6-second window every 20 motion frames (0.4 seconds). EXPORTED_AUDIO_STRIDE_FRAMES = 20 # These are deliberately exact metadata labels, not fuzzy keyword matches. # SEED contains unsafe names such as idle_handstand, injured_*_idle and # crouch_*_idle. The selected labels were manually audited against every # natural-language description and kinematically checked for root drift. LISTEN_MOTION_CONTENT_NAMES = frozenset({"listening_R"}) IDLE_MOTION_CONTENT_NAMES = frozenset({"neutral_idle"}) @dataclass(frozen=True) class HiddenSlice: frame_id: int shard_path: Path offset: int length: int @dataclass(frozen=True) class AudioSource: source_key: str clips: tuple[HiddenSlice, ...] packed_path: Path | None = None packed_count: int = 0 # HFDv2 reads the exact continuous mixed waveform online. This field is # mutually exclusive with the two cached-hidden layouts above. waveform_path: Path | None = None waveform_count: int = 0 @property def window_count(self) -> int: if self.waveform_path is not None: return int(self.waveform_count) return ( int(self.packed_count) if self.packed_path is not None else len(self.clips) ) def instruction_text_for_voice( record: dict[str, Any], voice: AudioSource, ) -> str: """Return the exact text used to synthesize this voice waveform. Qwen3-TTS uses ``content_short_description`` for ``*_en.wav`` and ``content_short_description_zh`` for ``*_zh.wav``. The action metadata is English-only, so selecting ``record['instruction']`` unconditionally would give every Chinese waveform a false auxiliary transcript target. """ voice_name = Path(voice.source_key).name if voice_name.endswith("_zh"): key = "instruction_zh" elif voice_name.endswith("_en"): key = "instruction" else: # Retain lightweight synthetic fixtures and legacy private caches that # predate the production ``*_en``/``*_zh`` filename contract. key = "instruction" text = str(record.get(key, "")).strip() if not text: raise RuntimeError( "The selected TTS voice has no exact transcript label: " f"row={record.get('row_id')} voice={voice.source_key} key={key}" ) return text def has_known_tts_translation_mismatch( instruction: str, instruction_zh: str, ) -> bool: """Identify the audited ``retro`` -> ``后退`` TTS mistranslation.""" return bool( re.search(r"\bretro\b", str(instruction), flags=re.IGNORECASE) and "后退" in str(instruction_zh) ) def _load_canonical_motion_manifest( path: str | Path, ) -> tuple[dict[tuple[str, str], str], frozenset[str]]: """Load one globally selected motion for every bilingual instruction. Selection happens before the source-level train/validation split. This prevents the same instruction from mapping to one trajectory in train and a different trajectory in validation, which would silently reintroduce the ambiguity that this opt-in ablation is intended to remove. """ manifest_path = Path(path) if not manifest_path.is_file(): raise FileNotFoundError( f"Missing canonical-motion manifest: {manifest_path}" ) with manifest_path.open("r", encoding="utf-8") as handle: payload = json.load(handle) if payload.get("format") != CANONICAL_MOTION_MANIFEST_FORMAT: raise RuntimeError( "Unsupported canonical-motion manifest format: " f"{payload.get('format')!r}" ) entries = payload.get("selections") if not isinstance(entries, list) or not entries: raise RuntimeError( f"Canonical-motion manifest has no selections: {manifest_path}" ) result: dict[tuple[str, str], str] = {} selected_filenames: set[str] = set() for entry in entries: if not isinstance(entry, dict): raise RuntimeError( f"Invalid canonical-motion selection row: {entry!r}" ) key = ( str(entry.get("instruction", "")).strip(), str(entry.get("instruction_zh", "")).strip(), ) filename = str(entry.get("filename", "")).strip() if not all(key) or not filename: raise RuntimeError( f"Incomplete canonical-motion selection row: {entry!r}" ) if key in result: raise RuntimeError( f"Duplicate canonical instruction pair: {key!r}" ) if filename in selected_filenames: raise RuntimeError( "One canonical motion cannot represent two different exact " f"instructions: {filename}" ) result[key] = filename selected_filenames.add(filename) chinese_to_keys: dict[str, list[tuple[str, str]]] = {} for key in result: chinese_to_keys.setdefault(key[1], []).append(key) ambiguous_chinese = frozenset( text for text, keys in chinese_to_keys.items() if len(keys) > 1 ) declared_exclusions = payload.get( "excluded_ambiguous_chinese_instructions", [], ) if not isinstance(declared_exclusions, list) or any( not isinstance(text, str) or not text.strip() for text in declared_exclusions ): raise RuntimeError( "Canonical-motion Chinese exclusions must be non-empty strings" ) declared_exclusions = frozenset( text.strip() for text in declared_exclusions ) if declared_exclusions != ambiguous_chinese: raise RuntimeError( "Canonical-motion manifest must exclude every Chinese prompt that " "still maps to multiple English instructions: " f"expected={sorted(ambiguous_chinese)[:3]} " f"declared={sorted(declared_exclusions)[:3]}" ) return result, ambiguous_chinese def load_canonical_motion_selection( path: str | Path, ) -> dict[tuple[str, str], str]: """Return the audited bilingual-instruction to motion mapping.""" selection, _ = _load_canonical_motion_manifest(path) return selection @dataclass(frozen=True) class StationaryMotionSource: filename: str motion_dir: Path declared_frames: int content_name: str @dataclass(frozen=True) class BackgroundHiddenSource: source_key: str packed_path: Path window_count: int offsets: tuple[int, ...] @dataclass(frozen=True) class BackgroundWaveformSource: source_key: str waveform_path: Path window_count: int @dataclass(frozen=True) class SourceBackgroundAssignment: source_key: str background_key: str listen_start_index: int listen_window_count: int background_window_count: int initial_idle_max_steps: int max_motion_steps: int safety_steps: int mixed_audio_path: Path | None = None original_audio_path: Path | None = None speech_samples: int = 0 speech_scale: float = 1.0 @dataclass(frozen=True) class ControlCommandSource: """One audited emergency/hold command rendition used inside a session.""" intent: str phrase_id: str voice_name: str language: str text: str waveform_path: Path speech_samples: int speech_scale: float = 1.0 @dataclass(frozen=True) class BackchannelSource: """One short same-speaker acknowledgement overlaid during motion.""" category: str phrase_id: str split: str voice_name: str language: str text: str waveform_path: Path speech_samples: int speech_scale: float = 1.0 @dataclass(frozen=True) class DisfluencyAudioSource: """One regenerated instruction segment or split-specific discourse cue.""" category: str segment_id: str split: str voice_name: str language: str text: str waveform_path: Path speech_samples: int speech_scale: float = 1.0 audio_dir: str = "" instruction: str = "" instruction_zh: str = "" segment_role: str = "" def _normalise_audio_dir(value: str) -> str: """Map an old absolute Qwen3-TTS path to the portable hidden-cache key.""" parts = Path(str(value).replace("\\", "/")).parts try: start = parts.index("seed_audio") except ValueError: return Path(str(value)).as_posix().strip("/") return Path(*parts[start:]).as_posix() def _pcm16_wave_num_samples(path: Path, expected_rate: int) -> int: if not path.is_file(): raise FileNotFoundError(f"Missing online HFDv2 waveform: {path}") with wave.open(str(path), "rb") as handle: if ( handle.getnchannels() != 1 or handle.getsampwidth() != 2 or handle.getframerate() != int(expected_rate) ): raise ValueError( "HFDv2 online waveforms must be mono PCM16 at " f"{expected_rate} Hz: {path}" ) return int(handle.getnframes()) def _read_pcm16_wave_segment( path: Path, *, start_sample: int, sample_count: int, expected_rate: int, ) -> np.ndarray: start = int(start_sample) count = int(sample_count) if start < 0 or count < 0: raise ValueError( f"Invalid waveform slice at {path}: start={start}, count={count}" ) with wave.open(str(path), "rb") as handle: if ( handle.getnchannels() != 1 or handle.getsampwidth() != 2 or handle.getframerate() != int(expected_rate) ): raise ValueError( "HFDv2 online waveforms must be mono PCM16 at " f"{expected_rate} Hz: {path}" ) if start + count > int(handle.getnframes()): raise IndexError( f"Waveform slice exceeds {path}: start={start}, count={count}, " f"samples={handle.getnframes()}" ) handle.setpos(start) raw = handle.readframes(count) pcm = np.frombuffer(raw, dtype="<i2") if pcm.shape != (count,): raise RuntimeError( f"Short PCM read at {path}: got={pcm.shape} expected={(count,)}" ) # Match the existing online Qwen inference loader's PCM16 normalization. return (pcm.astype(np.float32) / 32768.0).copy() def _read_resampled_speech_float(path: Path, expected_rate: int) -> np.ndarray: """Decode the TTS PCM16 WAV exactly like the offline corpus builder.""" if not path.is_file(): raise FileNotFoundError(f"Missing original speech waveform: {path}") with wave.open(str(path), "rb") as handle: channels = int(handle.getnchannels()) sample_width = int(handle.getsampwidth()) source_rate = int(handle.getframerate()) compression = str(handle.getcomptype()) frames = int(handle.getnframes()) raw = handle.readframes(frames) if ( channels != 1 or sample_width != 2 or source_rate <= 0 or compression != "NONE" ): raise ValueError( "Expected uncompressed mono PCM16 TTS WAV: " f"path={path} channels={channels} width={sample_width} " f"rate={source_rate} compression={compression}" ) speech = ( np.frombuffer(raw, dtype="<i2").astype(np.float32) / np.float32(32768.0) ) if source_rate != int(expected_rate): divisor = math.gcd(source_rate, int(expected_rate)) speech = resample_poly( speech, int(expected_rate) // divisor, source_rate // divisor, ).astype(np.float32, copy=False) expected = int(round(frames * int(expected_rate) / float(source_rate))) if speech.size > expected: speech = speech[:expected] elif speech.size < expected: speech = np.pad(speech, (0, expected - speech.size)) if speech.size <= 0 or not np.isfinite(speech).all(): raise RuntimeError(f"Invalid decoded speech source: {path}") return np.ascontiguousarray(speech, dtype=np.float32) def detect_clean_speech_onset_sample( waveform: np.ndarray, sampling_rate: int, ) -> int: """Locate the first sustained speech energy in one clean TTS waveform. The detector is deliberately applied before office background is mixed in. A relative RMS threshold handles different speaker gains, while three adjacent 10 ms hops reject isolated PCM clicks. The returned sample is only used to move a control boundary to a later 0.4 s Qwen update; it does not trim, stretch or otherwise modify the speech waveform. """ values = np.asarray(waveform, dtype=np.float32).reshape(-1) rate = int(sampling_rate) if rate <= 0 or values.size <= 0 or not np.isfinite(values).all(): raise ValueError( "Speech onset detection needs a finite non-empty waveform and " f"positive sample rate, got samples={values.size} rate={rate}" ) frame = max(int(round(0.020 * rate)), 1) hop = max(int(round(0.010 * rate)), 1) if values.size < frame: rms = np.asarray( [float(np.sqrt(np.mean(np.square(values, dtype=np.float64))))], dtype=np.float64, ) else: starts = np.arange(0, values.size - frame + 1, hop, dtype=np.int64) rms = np.asarray( [ np.sqrt( np.mean( np.square(values[start : start + frame], dtype=np.float64) ) ) for start in starts ], dtype=np.float64, ) floor = float(np.quantile(rms, 0.10)) peak = float(np.quantile(rms, 0.95)) threshold = max(2.5e-3, floor + 0.08 * max(peak - floor, 0.0)) active = rms >= threshold if active.size >= 3: sustained = np.convolve( active.astype(np.int8), np.ones(3, dtype=np.int8), mode="valid", ) matches = np.flatnonzero(sustained == 3) if matches.size: return min(int(matches[0]) * hop, values.size - 1) matches = np.flatnonzero(active) return min(int(matches[0]) * hop, values.size - 1) if matches.size else 0 def _map_seed_motion_dir(root_dir: Path, move_g1_path: str) -> Path: rel = str(move_g1_path).strip().replace("\\", "/") rel = rel.replace("g1/csv/", "g1/csv_sonic/") rel = rel.replace("/csv/", "/csv_sonic/") if rel.endswith(".csv"): rel = rel[:-4] if rel.startswith("dataset/seed/"): rel = rel[len("dataset/seed/") :] if rel.startswith("seed/"): rel = rel[len("seed/") :] path = Path(rel) return path if path.is_absolute() else root_dir / path def _progress_rows(frame_indices: Iterable[int], total_frames: int) -> np.ndarray: denominator = float(max(int(total_frames) - 1, 1)) rows = [] for frame_index in frame_indices: progress = float(min(max(int(frame_index), 0), total_frames - 1)) / denominator rows.append( [ progress, max(0.0, 1.0 - progress), progress, math.sin(2.0 * math.pi * progress), math.cos(2.0 * math.pi * progress), ] ) return np.asarray(rows, dtype=np.float32) def _normalise_motion_quaternions(motion: np.ndarray) -> np.ndarray: result = np.asarray(motion, dtype=np.float32).copy() quat = result[:, 32:36] norms = np.linalg.norm(quat, axis=-1, keepdims=True) fallback = np.zeros_like(quat) fallback[:, 0] = 1.0 result[:, 32:36] = np.where( norms > 1e-8, quat / np.maximum(norms, 1e-8), fallback, ) return result def _blend_pose_rows( anchor: np.ndarray, motion: np.ndarray, weight: np.ndarray, ) -> np.ndarray: """Blend absolute G1 poses while keeping quaternion signs and norms valid.""" anchor_rows = np.broadcast_to( np.asarray(anchor, dtype=np.float32).reshape(1, 36), motion.shape, ).copy() moving = np.asarray(motion, dtype=np.float32).copy() # q and -q are the same rotation. Flip before linear blending so an # equivalent sign choice cannot pass through a zero quaternion. quat_dot = (anchor_rows[:, 32:36] * moving[:, 32:36]).sum(axis=-1) moving[quat_dot < 0.0, 32:36] *= -1.0 alpha = np.asarray(weight, dtype=np.float32).reshape(-1, 1) blended = anchor_rows * (1.0 - alpha) + moving * alpha return _normalise_motion_quaternions(blended) def _align_task_motion_to_anchor_heading( motion: np.ndarray, stationary_anchor: np.ndarray, ) -> np.ndarray: """Rigidly express a task trajectory in the anchor's world heading. TextOp observes local root deltas and yaw deltas, not absolute world yaw. Requiring a task recorded at one arbitrary world heading to start from an independently sampled idle heading would therefore create an unobservable turn target. A single world-frame yaw transform preserves the task's local motion while making its first heading and horizontal origin match the current stationary robot state. Root height keeps its original transition semantics. """ task = np.asarray(motion, dtype=np.float32) anchor = np.asarray(stationary_anchor, dtype=np.float32).reshape(-1) if task.ndim != 2 or task.shape[1] != 36 or task.shape[0] <= 0: raise ValueError(f"Invalid task motion shape: {task.shape}") if anchor.shape != (36,): raise ValueError(f"Invalid stationary anchor shape: {anchor.shape}") if not np.isfinite(task).all() or not np.isfinite(anchor).all(): raise ValueError("Task motion and stationary anchor must be finite") aligned = _normalise_motion_quaternions(task) task_yaw = float(quat_to_yaw_wxyz(aligned[0, 32:36])) anchor_yaw = float(quat_to_yaw_wxyz(anchor[32:36])) yaw_delta = float( wrap_angle(np.asarray(anchor_yaw - task_yaw, dtype=np.float32)) ) relative_root = aligned[:, 29:31] - aligned[0:1, 29:31] cos_yaw = math.cos(yaw_delta) sin_yaw = math.sin(yaw_delta) rotated_x = cos_yaw * relative_root[:, 0] - sin_yaw * relative_root[:, 1] rotated_y = sin_yaw * relative_root[:, 0] + cos_yaw * relative_root[:, 1] aligned[:, 29] = anchor[29] + rotated_x aligned[:, 30] = anchor[30] + rotated_y yaw_quat = rpy_to_quat_wxyz(0.0, 0.0, yaw_delta) aligned[:, 32:36] = quat_mul_wxyz(yaw_quat, aligned[:, 32:36]) return _normalise_motion_quaternions(aligned) class FullDuplexAudioCommandDataset(torch.utils.data.Dataset): """Return one complete spoken-command source and its streaming motion. A dataset item remains one ordered source so the DataLoader cannot shuffle its windows. Action targets advance by one 50 Hz frame, while a new 0.6 s audio window is encoded only every 20 frames. The 19 intervening action samples reuse that history-conditioned Qwen output without extending KV. """ def __init__( self, *, hidden_root: str, metadata_jsonl: str, motion_metadata_csv: str, motion_root: str, white_noise_hidden_dir: str = "", background_hidden_root: str = "", background_assignment_path: str = "", continuous_audio_root: str = "", online_audio: bool = False, background_source_name: str = "hfd_continuous_office_background", background_task_name: str = "audio_command_text", split: str = "train", val_ratio: float = 0.1, split_seed: int = 42, split_source_namespace: str = "seed", source_name: str = "qwen3_tts_seed_audio_command", task_name: str = "audio_command_text", action_chunk_size: int = 30, action_exec_horizon: int = 20, motion_fps: int = 50, motion_metadata_fps: int = 120, audio_sampling_rate: int = 16000, audio_window_seconds: float = 0.6, audio_sample_step_frames: int = 20, action_sample_step_frames: int = 1, motion_cache_size: int = 256, fixed_voice_per_record: bool = True, max_voice_windows: int = 30, max_voice_median_ratio: float = 3.0, deterministic_eval: bool = True, limit_sources: int = 0, max_motion_steps: int = 0, initial_idle_max_steps: int = 100, stationary_blend_frames: int = 20, stationary_heading_alignment: bool = False, safe_stationary_anchors: bool = False, stationary_idle_residual_retargeting: bool = False, stationary_listen_hold: bool = False, dynamic_task_entry_blend: bool = False, multi_turn_sessions: bool = False, session_min_turns: int = 2, session_max_turns: int = 4, session_voice_variants: int = 0, interruption_sessions: bool = False, interruption_ratio: float = 0.5, interruption_min_start_seconds: float = 2.0, interruption_max_start_seconds: float = 6.0, interruption_min_remaining_seconds: float = 1.5, independent_task_pools: bool = False, turn_taking_session_variants: int = 2, command_interruption_session_variants: int = 2, emergency_stop_session_variants: int = 1, safe_hold_session_variants: int = 1, backchannel_session_variants: int = 0, hesitation_session_variants: int = 0, pause_session_variants: int = 0, interruption_task_session_ratio: float = 0.0, control_task_session_ratio: float = 0.0, backchannel_task_session_ratio: float = 0.0, hesitation_task_window_ratio: float = 0.0, pause_task_window_ratio: float = 0.0, control_followup_ratio: float = 0.0, control_post_idle_min_steps: int = 1, control_post_idle_max_steps: int = 0, control_post_idle_sampling: str = "legacy_uniform", control_min_start_seconds: float = 2.0, control_max_start_seconds: float = 6.0, control_min_remaining_seconds: float = 0.4, control_activity_aware_timing: bool = False, filter_known_tts_translation_mismatches: bool = False, canonical_motion_manifest: str = "", control_command_manifest: str = "", backchannel_command_manifest: str = "", hesitation_pause_manifest: str = "", new_task_max_rows_per_instruction_train: int = 32, new_task_max_rows_per_instruction_val: int = 8, hesitation_filled_ratio: float = 0.60, self_repair_implicit_ratio: float = 0.0, backchannel_min_start_seconds: float = 1.2, backchannel_max_start_seconds: float = 6.0, backchannel_min_remaining_seconds: float = 1.2, safe_hold_stability_window_frames: int = 20, turn_transition_frames: int = 50, causal_listen_timing: bool = False, max_speech_onset_steps: int = 3, listen_reaction_steps: int = 1, max_stream_steps: int = 1923, long_session_drop_ratio: float = 0.0, ) -> None: self.hidden_root = relocate_hfd_storage_path(hidden_root) self.metadata_jsonl = relocate_hfd_storage_path(metadata_jsonl) self.motion_metadata_csv = relocate_hfd_storage_path( motion_metadata_csv ) self.motion_root = relocate_hfd_storage_path(motion_root) self.white_noise_hidden_dir = ( relocate_hfd_storage_path(white_noise_hidden_dir) if str(white_noise_hidden_dir) else None ) self.background_hidden_root = ( relocate_hfd_storage_path(background_hidden_root) if str(background_hidden_root) else None ) self.background_assignment_path = ( relocate_hfd_storage_path(background_assignment_path) if str(background_assignment_path) else None ) self.continuous_audio_root = ( relocate_hfd_storage_path(continuous_audio_root) if str(continuous_audio_root) else ( self.background_assignment_path.parent.parent if self.background_assignment_path is not None else None ) ) self.online_audio = bool(online_audio) self.background_source_name = str(background_source_name) self.background_task_name = str(background_task_name) self.split = str(split) self.val_ratio = float(val_ratio) self.split_seed = int(split_seed) self.split_source_namespace = str(split_source_namespace) self.source_name = str(source_name) self.task_name = str(task_name) self.action_chunk_size = int(action_chunk_size) self.action_exec_horizon = int(action_exec_horizon) self.prefix_frames = self.action_chunk_size - self.action_exec_horizon self.motion_fps = int(motion_fps) self.motion_metadata_fps = int(motion_metadata_fps) self.audio_sampling_rate = int(audio_sampling_rate) self.audio_window_seconds = float(audio_window_seconds) self.audio_sample_step_frames = int(audio_sample_step_frames) self.action_sample_step_frames = int(action_sample_step_frames) self.motion_cache_size = max(int(motion_cache_size), 0) self.fixed_voice_per_record = bool(fixed_voice_per_record) self.max_voice_windows = int(max_voice_windows) self.max_voice_median_ratio = float(max_voice_median_ratio) self.deterministic_eval = bool(deterministic_eval) self.max_motion_steps = max(int(max_motion_steps), 0) self.initial_idle_max_steps = max(int(initial_idle_max_steps), 0) self.stationary_blend_frames = max(int(stationary_blend_frames), 1) self.stationary_heading_alignment = bool( stationary_heading_alignment ) self.safe_stationary_anchors = bool(safe_stationary_anchors) self.stationary_idle_residual_retargeting = bool( stationary_idle_residual_retargeting ) self.stationary_listen_hold = bool(stationary_listen_hold) self.dynamic_task_entry_blend = bool(dynamic_task_entry_blend) self.multi_turn_sessions = bool(multi_turn_sessions) self.session_min_turns = int(session_min_turns) self.session_max_turns = int(session_max_turns) self.session_voice_variants = int(session_voice_variants) self.interruption_sessions = bool(interruption_sessions) self.interruption_ratio = float(interruption_ratio) self.interruption_min_start_seconds = float( interruption_min_start_seconds ) self.interruption_max_start_seconds = float( interruption_max_start_seconds ) self.interruption_min_remaining_seconds = float( interruption_min_remaining_seconds ) self.independent_task_pools = bool(independent_task_pools) self.task_session_variants = { "turn-taking": int(turn_taking_session_variants), "interruption": int(command_interruption_session_variants), "emergency-stop": int(emergency_stop_session_variants), "safe-hold": int(safe_hold_session_variants), "backchannel": int(backchannel_session_variants), "hesitation": int(hesitation_session_variants), "pause": int(pause_session_variants), } self.interruption_task_session_ratio = float( interruption_task_session_ratio ) self.control_task_session_ratio = float( control_task_session_ratio ) self.backchannel_task_session_ratio = float( backchannel_task_session_ratio ) self.hesitation_task_window_ratio = float( hesitation_task_window_ratio ) self.pause_task_window_ratio = float(pause_task_window_ratio) self.control_followup_ratio = float(control_followup_ratio) self.control_post_idle_min_steps = max( int(control_post_idle_min_steps), 1, ) configured_post_idle_max = int(control_post_idle_max_steps) self.control_post_idle_max_steps_explicit = ( configured_post_idle_max > 0 ) self.control_post_idle_max_steps = ( configured_post_idle_max if self.control_post_idle_max_steps_explicit else max(self.initial_idle_max_steps, 1) ) self.control_post_idle_sampling = str( control_post_idle_sampling ).strip() if self.control_post_idle_sampling not in { "legacy_uniform", "short_heavy_long_tail", }: raise ValueError( "Unsupported control_post_idle_sampling: " f"{self.control_post_idle_sampling!r}" ) self.control_min_start_seconds = float(control_min_start_seconds) self.control_max_start_seconds = float(control_max_start_seconds) self.control_min_remaining_seconds = float( control_min_remaining_seconds ) self.control_activity_aware_timing = bool( control_activity_aware_timing ) self.filter_known_tts_translation_mismatches = bool( filter_known_tts_translation_mismatches ) self.canonical_motion_manifest = ( relocate_hfd_storage_path(canonical_motion_manifest) if str(canonical_motion_manifest).strip() else None ) self.control_command_manifest = ( relocate_hfd_storage_path(control_command_manifest) if str(control_command_manifest).strip() else None ) self.backchannel_command_manifest = ( relocate_hfd_storage_path(backchannel_command_manifest) if str(backchannel_command_manifest).strip() else None ) self.hesitation_pause_manifest = ( relocate_hfd_storage_path(hesitation_pause_manifest) if str(hesitation_pause_manifest).strip() else None ) self.new_task_max_rows_per_instruction_train = max( int(new_task_max_rows_per_instruction_train), 1, ) self.new_task_max_rows_per_instruction_val = max( int(new_task_max_rows_per_instruction_val), 1, ) self.hesitation_filled_ratio = float(hesitation_filled_ratio) self.self_repair_implicit_ratio = float( self_repair_implicit_ratio ) self.backchannel_min_start_seconds = float( backchannel_min_start_seconds ) self.backchannel_max_start_seconds = float( backchannel_max_start_seconds ) self.backchannel_min_remaining_seconds = float( backchannel_min_remaining_seconds ) self.safe_hold_stability_window_frames = max( int(safe_hold_stability_window_frames), 2, ) self.turn_transition_frames = max(int(turn_transition_frames), 2) self.causal_listen_timing = bool(causal_listen_timing) self.max_speech_onset_steps = max(int(max_speech_onset_steps), 1) self.listen_reaction_steps = max(int(listen_reaction_steps), 0) self.max_stream_steps = max(int(max_stream_steps), 0) self.long_session_drop_ratio = float(long_session_drop_ratio) if not ( math.isfinite(self.long_session_drop_ratio) and 0.0 <= self.long_session_drop_ratio < 1.0 ): raise ValueError( "long_session_drop_ratio must be finite and within [0, 1), " f"got {self.long_session_drop_ratio}" ) if self.long_session_drop_ratio > 0.0 and not self.multi_turn_sessions: raise ValueError( "Long-session tail filtering requires multi_turn_sessions=True" ) if not ( 1 <= self.session_min_turns <= self.session_max_turns <= 4 ): raise ValueError( "Sessions require 1 <= min_turns <= max_turns <= 4, " f"got {self.session_min_turns}..{self.session_max_turns}" ) if self.session_voice_variants < 0: raise ValueError( "session_voice_variants must be non-negative, got " f"{self.session_voice_variants}" ) if self.session_voice_variants > 0 and not self.multi_turn_sessions: raise ValueError( "Session voice variants require multi_turn_sessions=True" ) if self.interruption_sessions: if self.session_max_turns < 2: raise ValueError("Command interruption requires at least two source turns") if not self.multi_turn_sessions: raise ValueError( "Interruption sessions require multi_turn_sessions=True" ) if not self.causal_listen_timing: raise ValueError( "Interruption sessions require causal_listen_timing=True" ) if not self.independent_task_pools and self.session_voice_variants != 2: raise ValueError( "The 1:1 turn-taking/interruption design requires exactly " "two train session variants" ) if self.independent_task_pools: invalid_weights = { name: count for name, count in self.task_session_variants.items() if count < 0 } if invalid_weights or self.task_session_variants["turn-taking"] < 1: raise ValueError( "Independent task variant counts must be non-negative " "and retain turn-taking: " f"{self.task_session_variants}" ) if ( self.task_session_variants["emergency-stop"] > 0 or self.task_session_variants["safe-hold"] > 0 ) and ( self.control_command_manifest is None or not self.control_command_manifest.is_file() ): raise FileNotFoundError( "Emergency-stop/safe-hold data requires the generated " f"control-command manifest: {self.control_command_manifest}" ) if self.task_session_variants["backchannel"] > 0 and ( self.backchannel_command_manifest is None or not self.backchannel_command_manifest.is_file() ): raise FileNotFoundError( "Backchannel data requires the generated same-speaker " "manifest: " f"{self.backchannel_command_manifest}" ) disfluency_modes_enabled = any( self.task_session_variants[mode] > 0 for mode in ("hesitation", "pause") ) if disfluency_modes_enabled and ( self.hesitation_pause_manifest is None or not self.hesitation_pause_manifest.is_file() ): raise FileNotFoundError( "Hesitation/pause data requires the regenerated segment " f"manifest: {self.hesitation_pause_manifest}" ) enabled_control_modes = sum( int(self.task_session_variants[mode] > 0) for mode in ("emergency-stop", "safe-hold") ) if not ( math.isfinite(self.interruption_task_session_ratio) and 0.0 <= self.interruption_task_session_ratio < 1.0 and ( self.interruption_task_session_ratio == 0.0 or self.task_session_variants["interruption"] > 0 ) ): raise ValueError( "Interruption task session ratio must be zero or target " "an enabled interruption pool: " f"ratio={self.interruption_task_session_ratio}" ) if not ( math.isfinite(self.backchannel_task_session_ratio) and 0.0 <= self.backchannel_task_session_ratio < 1.0 and ( self.backchannel_task_session_ratio == 0.0 or self.task_session_variants["backchannel"] > 0 ) ): raise ValueError( "Backchannel task session ratio must be zero or target " "an enabled backchannel pool: " f"ratio={self.backchannel_task_session_ratio}" ) new_window_ratios = { "hesitation": self.hesitation_task_window_ratio, "pause": self.pause_task_window_ratio, } invalid_new_ratios = { mode: ratio for mode, ratio in new_window_ratios.items() if ( not math.isfinite(ratio) or ratio < 0.0 or ratio >= 1.0 or ( ratio > 0.0 and self.task_session_variants[mode] <= 0 ) ) } if invalid_new_ratios or sum(new_window_ratios.values()) >= 1.0: raise ValueError( "Invalid hesitation/pause final window ratios: " f"{new_window_ratios} invalid={invalid_new_ratios}" ) if not ( math.isfinite(self.hesitation_filled_ratio) and 0.0 < self.hesitation_filled_ratio < 1.0 ): raise ValueError( "hesitation_filled_ratio must be strictly within (0, 1), " f"got {self.hesitation_filled_ratio}" ) if not ( math.isfinite(self.self_repair_implicit_ratio) and 0.0 <= self.self_repair_implicit_ratio <= 1.0 ): raise ValueError( "self_repair_implicit_ratio must be within [0, 1], " f"got {self.self_repair_implicit_ratio}" ) if not ( math.isfinite(self.control_task_session_ratio) and 0.0 <= self.control_task_session_ratio and self.control_task_session_ratio * enabled_control_modes + self.interruption_task_session_ratio + self.backchannel_task_session_ratio < 1.0 ): raise ValueError( "Control task session ratio must leave non-control " "sessions in the mixture: " f"ratio={self.control_task_session_ratio} " f"modes={enabled_control_modes}" ) backchannel_timing = ( self.backchannel_min_start_seconds, self.backchannel_max_start_seconds, self.backchannel_min_remaining_seconds, ) if ( not all( math.isfinite(value) and value > 0.0 for value in backchannel_timing ) or self.backchannel_min_start_seconds > self.backchannel_max_start_seconds ): raise ValueError( "Invalid backchannel timing: " f"start={self.backchannel_min_start_seconds}.." f"{self.backchannel_max_start_seconds}s remaining=" f"{self.backchannel_min_remaining_seconds}s" ) if not ( math.isfinite(self.control_followup_ratio) and 0.0 <= self.control_followup_ratio <= 1.0 ): raise ValueError( "control_followup_ratio must be within [0, 1], got " f"{self.control_followup_ratio}" ) if ( self.control_post_idle_min_steps > self.control_post_idle_max_steps ): raise ValueError( "Control post-idle minimum exceeds the configured " "idle range: " f"minimum={self.control_post_idle_min_steps} " f"maximum={self.control_post_idle_max_steps}" ) control_timing = ( self.control_min_start_seconds, self.control_max_start_seconds, self.control_min_remaining_seconds, ) if ( not all( math.isfinite(value) and value > 0.0 for value in control_timing ) or self.control_min_start_seconds > self.control_max_start_seconds ): raise ValueError( "Invalid control-command timing: " f"start={self.control_min_start_seconds}.." f"{self.control_max_start_seconds}s remaining=" f"{self.control_min_remaining_seconds}s" ) if not self.independent_task_pools and not math.isclose( self.interruption_ratio, 0.5, rel_tol=0.0, abs_tol=1e-12, ): raise ValueError( "Interruption v1 requires an exact 1:1 behavior ratio" ) timing = ( self.interruption_min_start_seconds, self.interruption_max_start_seconds, self.interruption_min_remaining_seconds, ) if ( not all(math.isfinite(value) and value > 0.0 for value in timing) or self.interruption_min_start_seconds > self.interruption_max_start_seconds ): raise ValueError( "Invalid interruption timing: " f"start={self.interruption_min_start_seconds}.." f"{self.interruption_max_start_seconds}s remaining=" f"{self.interruption_min_remaining_seconds}s" ) if ( self.multi_turn_sessions and self.initial_idle_max_steps * self.audio_sample_step_frames > int(round(3.0 * self.motion_fps)) ): raise ValueError( "Initial idle must stay within three seconds at the Qwen " f"cadence, got steps={self.initial_idle_max_steps}" ) if self.causal_listen_timing and self.listen_reaction_steps < 1: raise ValueError( "Causal listen timing requires at least one post-onset Qwen " "reaction step" ) if self.motion_metadata_fps <= 0: raise ValueError( "motion_metadata_fps must be positive, got " f"{self.motion_metadata_fps}" ) if self.max_voice_windows < 0: raise ValueError( "max_voice_windows must be non-negative, got " f"{self.max_voice_windows}" ) if self.max_voice_windows > 0 and self.max_voice_median_ratio < 1.0: raise ValueError( "max_voice_median_ratio must be at least 1 when voice " f"filtering is enabled, got {self.max_voice_median_ratio}" ) if EXPORTED_AUDIO_STRIDE_FRAMES % self.audio_sample_step_frames != 0: raise ValueError( "Runtime audio stride must divide the exported 20-frame " f"stride, got {self.audio_sample_step_frames}" ) self.exported_stride_scale = ( EXPORTED_AUDIO_STRIDE_FRAMES // self.audio_sample_step_frames ) self._motion_cache: OrderedDict[str, np.ndarray] = OrderedDict() if self.online_audio: self.continuous_background = ( self.continuous_audio_root is not None and self.background_assignment_path is not None ) if not self.continuous_background: raise ValueError( "HFDv2 online audio requires continuous_audio_root and " "background_assignment_path" ) else: self.continuous_background = ( self.background_hidden_root is not None and self.background_assignment_path is not None ) if (self.background_hidden_root is None) != ( self.background_assignment_path is None ): raise ValueError( "Continuous background requires both background_hidden_root " "and background_assignment_path" ) if self.action_chunk_size != 30 or self.action_exec_horizon != 20: raise ValueError( "Window-step HFD timing requires a 30-frame window and " "a 20-frame execute horizon; " f"got chunk={self.action_chunk_size}, execute={self.action_exec_horizon}" ) if ( self.motion_fps != 50 or self.audio_sampling_rate != 16000 or not math.isclose(self.audio_window_seconds, 0.6) or self.audio_sample_step_frames != 20 or self.action_sample_step_frames != 1 ): raise ValueError( "Dense HFD timing requires 50 fps actions at one-frame stride " "and 16 kHz Qwen audio at 0.6 s window / 20-frame stride; " f"got fps={self.motion_fps}, rate={self.audio_sampling_rate}, " f"window={self.audio_window_seconds}, audio_stride=" f"{self.audio_sample_step_frames}, action_stride=" f"{self.action_sample_step_frames}" ) if not self.online_audio: raise ValueError( "Dense action sliding cannot reuse the offline hidden cache; " "online_audio=True is required" ) for path, label in ( (self.metadata_jsonl, "audio-command manifest"), (self.motion_metadata_csv, "motion metadata"), ): if not path.is_file(): raise FileNotFoundError(f"Missing {label}: {path}") raw_audio_catalog = ( self._load_waveform_audio_catalog() if self.online_audio else self._load_audio_catalog() ) ( self.audio_catalog, self.voice_outliers_filtered, ) = self._filter_voice_duration_outliers(raw_audio_catalog) # Session construction and its conservative length audit repeatedly # resolve one fixed speaker identity for every turn. Index the small # audio catalog once instead of rebuilding ``Path(...).name`` and # linearly scanning all voice variants millions of times on every # distributed rank. This is only a lookup acceleration: the catalog, # selected AudioSource objects and deterministic session policy stay # byte-for-byte identical. self._audio_catalog_by_voice_name = { audio_dir: { self._voice_name(voice): voice for voice in voices } for audio_dir, voices in self.audio_catalog.items() } if any( len(self._audio_catalog_by_voice_name[audio_dir]) != len(voices) for audio_dir, voices in self.audio_catalog.items() ): raise RuntimeError( "Duplicate speaker/language identity in HFD audio catalog" ) if self.continuous_background: self.background_catalog = ( self._load_background_waveform_catalog() if self.online_audio else self._load_background_catalog() ) self.background_assignments = self._load_background_assignments() self.white_noise_hidden = () else: self.background_catalog = {} self.background_assignments = {} self.white_noise_hidden = self._load_white_noise_hidden() self.control_command_catalog = ( self._load_control_command_catalog() if self.independent_task_pools and ( self.task_session_variants["emergency-stop"] > 0 or self.task_session_variants["safe-hold"] > 0 ) else {} ) self.backchannel_catalog = ( self._load_backchannel_catalog() if self.independent_task_pools and self.task_session_variants["backchannel"] > 0 else {} ) ( self.disfluency_instruction_segments, self.disfluency_marker_catalog, ) = ( self._load_hesitation_pause_catalog() if self.independent_task_pools and any( self.task_session_variants[mode] > 0 for mode in ("hesitation", "pause") ) else ({}, {}) ) self.stationary_motion_pools = self._load_stationary_motion_pools() self._stationary_motion_paths = frozenset( str(source.motion_dir) for pool in self.stationary_motion_pools.values() for source in pool ) self._safe_idle_motion_paths = frozenset( str(source.motion_dir) for source in self.stationary_motion_pools["idle"] ) self._stationary_anchor_poses = np.empty((0, 36), dtype=np.float32) self._stationary_joint_center = np.zeros(29, dtype=np.float32) self._stationary_joint_scale = np.ones(29, dtype=np.float32) self.stationary_audit_summary: dict[str, Any] = { "enabled": False, "accepted_clips": len(self.stationary_motion_pools["idle"]), "rejected_clips": 0, "anchor_frames": 0, } if self.safe_stationary_anchors: self._initialise_safe_stationary_anchors() self.records = self._load_records() records_before_context_filter = len(self.records) if self.max_stream_steps > 0: self.records = [ record for record in self.records if self._record_max_stream_steps(record) <= self.max_stream_steps ] records_filtered_for_context = ( records_before_context_filter - len(self.records) ) if limit_sources > 0: self.records = self.records[: int(limit_sources)] if not self.records: raise RuntimeError(f"No full-duplex sources remain for split={self.split!r}") sessions_filtered_for_context, sessions_filtered_for_background = ( self._prepare_session_index() ) listen_counts = [ source.window_count for sources in self.audio_catalog.values() for source in sources ] interaction_session_counts: dict[str, int] = {} interaction_primitive_counts: dict[str, int] = {} for session in self.sessions: mode = str( session[0].get("_session_interaction_mode", "turn-taking") ) interaction_session_counts[mode] = ( interaction_session_counts.get(mode, 0) + 1 ) interaction_primitive_counts[mode] = ( interaction_primitive_counts.get(mode, 0) + len(session) ) interaction_summary = ",".join( f"{mode}:{interaction_session_counts[mode]}/" f"{interaction_primitive_counts[mode]}/" f"{100.0 * interaction_session_counts[mode] / len(self.sessions):.3f}%" for mode in sorted(interaction_session_counts) ) filtered_control_summary = ",".join( f"{mode}:{count}" for mode, count in sorted( getattr(self, "control_task_rows_filtered", {}).items() ) ) or "none" canonical_motion_summary = getattr( self, "canonical_motion_summary", {"enabled": False}, ) print( "[FULL DUPLEX DATA] " f"split={self.split} rows={len(self.records):,} sessions={len(self.sessions):,} " f"turns={self.session_min_turns}-{self.session_max_turns} " f"wav_sources={sum(len(v) for v in self.audio_catalog.values()):,} " f"voice_outliers_filtered={self.voice_outliers_filtered:,} " f"translation_mismatch_rows=" f"{getattr(self, 'known_translation_mismatch_rows', 0):,} " f"translation_mismatch_audio_dirs=" f"{getattr(self, 'known_translation_mismatch_audio_dirs', 0):,} " f"translation_voice_variants_excluded=" f"{getattr(self, 'known_translation_voice_variants_excluded', 0):,} " f"canonical_motion={canonical_motion_summary} " f"listen_clips={sum(listen_counts):,} " f"background_mode={'continuous' if self.continuous_background else 'legacy_cycle'} " f"background_sessions={len(self.background_catalog):,} " f"background_windows={sum(item.window_count for item in self.background_catalog.values()):,} " f"input_layout={'raw_waveform_qwen' if self.online_audio else self.hidden_layout} " f"cache_scope={'one_session' if self.multi_turn_sessions else 'one_source'} voice_policy=" f"{'fixed_per_session' if self.session_voice_variants > 0 else 'fixed_per_record' if self.fixed_voice_per_record else 'random'} " f"session_voice_variants={self.session_voice_variants or 1} " f"voice_reuse_after_coverage=" f"{getattr(self, 'voice_reuse_after_coverage', {})} " f"interaction_sessions/primitives/ratio={interaction_summary} " f"control_task_uses_filtered={filtered_control_summary} " f"interruption_session_target=" f"{self.interruption_task_session_ratio:g} " f"control_session_ratio={self.control_task_session_ratio:g} " f"backchannel_session_target=" f"{self.backchannel_task_session_ratio:g} " f"backchannel_start_seconds=" f"{self.backchannel_min_start_seconds:g}-" f"{self.backchannel_max_start_seconds:g} " f"backchannel_min_remaining_seconds=" f"{self.backchannel_min_remaining_seconds:g} " f"control_followup_ratio={self.control_followup_ratio:g} " f"control_post_idle_min_steps={self.control_post_idle_min_steps} " f"control_post_idle_max_steps={self.control_post_idle_max_steps} " f"control_post_idle_sampling={self.control_post_idle_sampling} " f"control_start_seconds=" f"{self.control_min_start_seconds:g}-" f"{self.control_max_start_seconds:g} " f"control_min_remaining_seconds=" f"{self.control_min_remaining_seconds:g} " f"control_activity_aware=" f"{int(self.control_activity_aware_timing)} " f"interruption_ratio=" f"{self.interruption_ratio if self.interruption_sessions else 0.0:g} " f"interruption_start_seconds=" f"{self.interruption_min_start_seconds:g}-" f"{self.interruption_max_start_seconds:g} " f"interruption_min_remaining_seconds=" f"{self.interruption_min_remaining_seconds:g} " f"idle_pool={len(self.stationary_motion_pools['idle']):,} " f"listen_pool={len(self.stationary_motion_pools['listen']):,} " f"safe_idle_clips=" f"{self.stationary_audit_summary['accepted_clips']:,}/" f"{len(self.stationary_motion_pools['idle']):,} " f"safe_idle_anchor_frames=" f"{self.stationary_audit_summary['anchor_frames']:,} " f"long_session_drop_ratio={self.long_session_drop_ratio:g} " f"long_session_filter={self.long_session_filter_summary or 'disabled'} " f"dynamic_task_entry_blend={int(self.dynamic_task_entry_blend)} " f"initial_idle_steps=uniform[0,{self.initial_idle_max_steps}]", f"max_stream_steps={self.max_stream_steps or 'unbounded'} " f"context_filtered_rows={records_filtered_for_context:,} " f"context_filtered_sessions={sessions_filtered_for_context:,} " f"background_filtered_sessions={sessions_filtered_for_background:,} " f"unsafe_turn_rows_filtered={getattr(self, 'unsafe_turn_rows_filtered', 0):,} " f"turn_transition_frames={self.turn_transition_frames} " f"causal_listen_timing={int(self.causal_listen_timing)} " f"listen_reaction_steps={self.listen_reaction_steps}", flush=True, ) def _prepare_session_index(self): from psi.data.seed.startup_index_cache import cached_index # Hash the *loaded, validated contents*, not just manifest names/mtime. # This includes every constructor option, records, waveform geometry, # task inventories and safe idle anchors. Augmentation still runs in # __getitem__; no RNG, audio, motion item or Qwen KV is persisted here. inputs = {key: value for key, value in vars(self).items() if key != "_motion_cache"} before = set(vars(self)) def build(): stats = self._build_session_index() self._estimated_stream_lengths = tuple( self._estimate_stream_steps(index) for index in range(len(self)) ) state = {key: value for key, value in vars(self).items() if key not in before and key != "_session_length_cache"} return state, stats state, stats = cached_index( f"audio-sessions-{self.split}", inputs, build, code_paths=(Path(__file__),), ) vars(self).update(state) # IDs of tuples are process-local and must never survive serialization. self._session_length_cache = {} if len(self._estimated_stream_lengths) != len(self): raise RuntimeError("Startup session index/length count mismatch") return stats def _build_session_index(self): self.sessions = self._compose_sessions(self.records) self._session_length_cache = {} sessions_before_context_filter = len(self.sessions) if self.multi_turn_sessions and self.max_stream_steps > 0: self.sessions = tuple( session for session in self.sessions if self._cached_session_max_stream_steps(session) <= self.max_stream_steps ) context_filtered = sessions_before_context_filter - len(self.sessions) sessions_before_background_filter = len(self.sessions) max_background_windows = max( (int(source.window_count) for source in self.background_catalog.values() if isinstance(source, BackgroundWaveformSource)), default=0, ) if self.multi_turn_sessions and max_background_windows > 0: self.sessions = tuple( session for session in self.sessions if self._cached_session_max_stream_steps(session) <= max_background_windows ) background_filtered = sessions_before_background_filter - len(self.sessions) if (self.multi_turn_sessions and self.independent_task_pools and ( self.control_task_session_ratio > 0.0 or self.interruption_task_session_ratio > 0.0 or self.backchannel_task_session_ratio > 0.0 or self.hesitation_task_window_ratio > 0.0 or self.pause_task_window_ratio > 0.0 )): if (self.backchannel_task_session_ratio > 0.0 or self.hesitation_task_window_ratio > 0.0 or self.pause_task_window_ratio > 0.0): self.sessions = self._balance_backchannel_task_sessions(self.sessions) else: self.sessions = self._balance_control_task_sessions(self.sessions) self.long_session_filter_summary = {} if self.long_session_drop_ratio > 0.0: self.sessions = self._drop_longest_sessions_by_mode(self.sessions) if not self.sessions: raise RuntimeError(f"No full-duplex sessions remain for split={self.split!r}") return context_filtered, background_filtered @property def hidden_index_path(self) -> Path: return ( self.hidden_root / "audio" / self.source_name / self.task_name / "shards" / "index.rank00.sqlite" ) @property def source_hidden_index_paths(self) -> tuple[Path, ...]: source_dir = ( self.hidden_root / "audio" / self.source_name / self.task_name / "sources" ) return tuple(sorted(source_dir.glob("index.ws*.rank*.sqlite"))) @property def audio_window_samples(self) -> int: return int(round(self.audio_window_seconds * self.audio_sampling_rate)) @property def audio_stride_samples(self) -> int: return int( round( self.audio_sample_step_frames / float(self.motion_fps) * self.audio_sampling_rate ) ) def _runtime_listen_window_count(self, speech_samples: int) -> int: prefix_samples = self.audio_window_samples - self.audio_stride_samples return int( math.ceil( (prefix_samples + int(speech_samples)) / float(self.audio_stride_samples) ) ) def _filter_voice_duration_outliers( self, catalog: dict[str, tuple[AudioSource, ...]], ) -> tuple[dict[str, tuple[AudioSource, ...]], int]: """Remove broken TTS duration outliers before sampling and grouping. Every command normally has several renditions with similar duration. A small number of generated WAVs contain tens of seconds of unrelated audio. Those sources otherwise dominate hundreds of dense optimizer steps and leave the other GPUs executing padded rows. Keep a generous absolute cap and a per-command robust cap; retain the shortest voice as a defensive fallback if a custom corpus contains only long renditions. """ max_windows = int(getattr(self, "max_voice_windows", 0)) if max_windows <= 0: return dict(catalog), 0 median_ratio = float(self.max_voice_median_ratio) output: dict[str, tuple[AudioSource, ...]] = {} filtered_count = 0 for audio_dir, voices in catalog.items(): if not voices: continue median_windows = float( np.median([voice.window_count for voice in voices]) ) relative_floor = min(max_windows, 15) directory_cap = min( max_windows, max( relative_floor, int(math.ceil(median_windows * median_ratio)), ), ) retained = tuple( voice for voice in voices if voice.window_count <= directory_cap ) if not retained: retained = (min(voices, key=lambda voice: voice.window_count),) output[audio_dir] = retained filtered_count += len(voices) - len(retained) return output, filtered_count def _load_waveform_audio_catalog( self, ) -> dict[str, tuple[AudioSource, ...]]: """Index the exact mixed WAVs without touching cached Qwen hidden data.""" if self.background_assignment_path is None: raise ValueError("background_assignment_path is not configured") by_directory: dict[str, list[AudioSource]] = {} with self.background_assignment_path.open( "r", encoding="utf-8", ) as handle: for line in handle: if not line.strip(): continue row = json.loads(line) if row.get("format") != "hfd_continuous_background_assignment_v1": raise RuntimeError( "Unsupported HFDv2 waveform assignment row: " f"{row.get('source_key')}" ) source_key = str(row["source_key"]) waveform_path = relocate_hfd_storage_path( str(row["mixed_audio_path"]) ) exported_count = int(row["listen_window_count"]) speech_samples = int(row.get("speech_samples", 0)) count = self._runtime_listen_window_count(speech_samples) if count <= 0: raise RuntimeError( "Mixed waveform/window assignment mismatch: " f"source={source_key} exported_count={exported_count} " f"speech_samples={speech_samples}" ) # Do not open all 41,694 small WAVs on every distributed rank # during dataset construction. The production marker binds the # complete assignment file, while the exact PCM geometry is # validated by _read_pcm16_wave_segment when a selected source # is first consumed. This changes only fail timing for a broken # file and removes hundreds of thousands of CPFS metadata I/Os # from eight-rank startup. audio_dir = Path(source_key).parent.as_posix() by_directory.setdefault(audio_dir, []).append( AudioSource( source_key=source_key, clips=(), waveform_path=waveform_path, waveform_count=count, ) ) if not by_directory: raise RuntimeError( f"No online mixed waveforms in {self.background_assignment_path}" ) self.hidden_layout = "disabled_online_qwen" return { key: tuple(sorted(values, key=lambda item: item.source_key)) for key, values in by_directory.items() } def _load_background_waveform_catalog( self, ) -> dict[str, BackgroundWaveformSource]: if self.continuous_audio_root is None: raise ValueError("continuous_audio_root is not configured") manifest_path = ( self.continuous_audio_root / "manifests" / "background_sessions.jsonl" ) audio_root = self.continuous_audio_root / "background" if not manifest_path.is_file(): raise FileNotFoundError( f"Missing online background manifest: {manifest_path}" ) output: dict[str, BackgroundWaveformSource] = {} with manifest_path.open("r", encoding="utf-8") as handle: for line in handle: if not line.strip(): continue row = json.loads(line) source_key = str(row["filename"]) path = relocate_hfd_storage_path(str(row["audio_path"])) waveform_path = path if path.is_absolute() else audio_root / path samples = _pcm16_wave_num_samples( waveform_path, self.audio_sampling_rate, ) count = ( 0 if samples < self.audio_window_samples else ( samples - self.audio_window_samples ) // self.audio_stride_samples + 1 ) if count <= 0: raise RuntimeError( f"Background waveform is too short: {waveform_path}" ) output[source_key] = BackgroundWaveformSource( source_key=source_key, waveform_path=waveform_path, window_count=count, ) if not output: raise RuntimeError( f"No HFDv2 online background sessions in {manifest_path}" ) return output def _load_audio_catalog(self) -> dict[str, tuple[AudioSource, ...]]: source_index_paths = self.source_hidden_index_paths if source_index_paths: self.hidden_layout = "source" return self._load_source_audio_catalog(source_index_paths) self.hidden_layout = "legacy_shard" index_path = self.hidden_index_path if not index_path.is_file(): raise FileNotFoundError(f"Missing HFD audio-command SQLite index: {index_path}") conn = sqlite3.connect(f"file:{index_path}?mode=ro", uri=True) try: columns = [row[1] for row in conn.execute("PRAGMA table_info(samples)")] if not {"key", "shard", "offset", "length"}.issubset(columns): raise RuntimeError( f"Expected the project-local fallback SQLite schema at {index_path}; columns={columns}" ) rows = conn.execute( "SELECT key, shard, offset, length FROM samples ORDER BY key" ).fetchall() finally: conn.close() grouped: dict[str, list[HiddenSlice]] = {} pattern = re.compile(r"^(.*)\.f(\d{8})\.safetensors$") for key, shard, offset, length in rows: match = pattern.match(str(key)) if match is None: continue source_key = match.group(1) frame_id = int(match.group(2)) shard_path = (index_path.parent / str(shard)).resolve() grouped.setdefault(source_key, []).append( HiddenSlice(frame_id, shard_path, int(offset), int(length)) ) by_directory: dict[str, list[AudioSource]] = {} for source_key, clips in grouped.items(): clips.sort(key=lambda item: item.frame_id) frame_ids = [item.frame_id for item in clips] # A source is valid only when it begins at zero and is a contiguous # 20-motion-frame stream. This prevents a missing hidden slice # from silently changing the spoken command history. if not frame_ids or frame_ids[0] != 0: continue if frame_ids != list(range(0, len(frame_ids) * 20, 20)): continue audio_dir = Path(source_key).parent.as_posix() by_directory.setdefault(audio_dir, []).append( AudioSource(source_key=source_key, clips=tuple(clips)) ) return { key: tuple(sorted(value, key=lambda source: source.source_key)) for key, value in by_directory.items() } def _load_source_audio_catalog( self, index_paths: tuple[Path, ...], ) -> dict[str, tuple[AudioSource, ...]]: source_rows: dict[str, tuple[Path, int]] = {} required_columns = { "key", "path", "count", "window_seconds", "step_frames", "motion_fps", "sampling_rate", } for index_path in index_paths: conn = sqlite3.connect(f"file:{index_path}?mode=ro", uri=True) try: columns = { str(row[1]) for row in conn.execute("PRAGMA table_info(sources)") } if not required_columns.issubset(columns): raise RuntimeError( f"Invalid HFD source index schema at {index_path}; " f"columns={sorted(columns)}" ) rows = conn.execute( "SELECT key, path, count, window_seconds, step_frames, " "motion_fps, sampling_rate FROM sources ORDER BY key" ).fetchall() finally: conn.close() for ( source_key, relative_path, count, window_seconds, step_frames, motion_fps, sampling_rate, ) in rows: if ( int(count) <= 0 or not math.isclose( float(window_seconds), self.audio_window_seconds, rel_tol=0.0, abs_tol=1e-9, ) or int(step_frames) != self.audio_sample_step_frames or int(motion_fps) != self.motion_fps or int(sampling_rate) != self.audio_sampling_rate ): raise RuntimeError( "HFD source index timing mismatch at " f"{index_path}: key={source_key}, count={count}, " f"window={window_seconds}, step={step_frames}, " f"fps={motion_fps}, rate={sampling_rate}" ) source_path = index_path.parent / str(relative_path) row_value = (source_path, int(count)) previous = source_rows.get(str(source_key)) if previous is not None and previous != row_value: raise RuntimeError( f"Conflicting duplicate HFD source index row: " f"{source_key}: {previous} != {row_value}" ) source_rows[str(source_key)] = row_value by_directory: dict[str, list[AudioSource]] = {} for source_key, (source_path, count) in source_rows.items(): audio_dir = Path(source_key).parent.as_posix() by_directory.setdefault(audio_dir, []).append( AudioSource( source_key=source_key, clips=(), packed_path=source_path, packed_count=count, ) ) if not by_directory: raise RuntimeError( f"No HFD source rows found in indexes: {index_paths}" ) return { key: tuple(sorted(value, key=lambda source: source.source_key)) for key, value in by_directory.items() } def _load_white_noise_hidden(self) -> tuple[torch.Tensor, ...]: if self.white_noise_hidden_dir is None: raise ValueError( "Legacy cyclic background requires white_noise_hidden_dir" ) paths = sorted(self.white_noise_hidden_dir.rglob("*.safetensors")) hidden: list[torch.Tensor] = [] for path in paths: tensors = load_file(str(path), device="cpu") value = tensors.get("hidden_states") if value is None: continue if value.ndim != 2: raise ValueError(f"White-noise hidden must be (tokens, dim), got {value.shape} at {path}") hidden.append(value.contiguous()) if not hidden: raise FileNotFoundError( "No white-noise hidden states found. Run the white-noise preparation step first: " f"{self.white_noise_hidden_dir}" ) return tuple(hidden) def _load_background_catalog( self, ) -> dict[str, BackgroundHiddenSource]: if self.background_hidden_root is None: raise ValueError("background_hidden_root is not configured") source_dir = ( self.background_hidden_root / "audio" / self.background_source_name / self.background_task_name / "sources" ) rows: dict[str, tuple[Path, int]] = {} for index_path in sorted(source_dir.glob("index.ws*.rank*.sqlite")): connection = sqlite3.connect( f"file:{index_path}?mode=ro", uri=True, ) try: values = connection.execute( "SELECT key, path, count, window_seconds, step_frames, " "motion_fps, sampling_rate FROM sources ORDER BY key" ).fetchall() finally: connection.close() for ( source_key, relative_path, count, window_seconds, step_frames, motion_fps, sampling_rate, ) in values: if ( int(count) <= 0 or not math.isclose( float(window_seconds), self.audio_window_seconds, rel_tol=0.0, abs_tol=1e-9, ) or int(step_frames) != self.audio_sample_step_frames or int(motion_fps) != self.motion_fps or int(sampling_rate) != self.audio_sampling_rate ): raise RuntimeError( "Continuous background timing mismatch at " f"{index_path}: key={source_key}" ) row_value = ( index_path.parent / str(relative_path), int(count), ) previous = rows.get(str(source_key)) if previous is not None and previous != row_value: raise RuntimeError( "Conflicting duplicate continuous-background index row: " f"{source_key}: {previous} != {row_value}" ) rows[str(source_key)] = row_value result: dict[str, BackgroundHiddenSource] = {} for source_key, (source_path, count) in rows.items(): if not source_path.is_file(): raise FileNotFoundError( f"Missing continuous background hidden: {source_path}" ) with safe_open( str(source_path), framework="pt", device="cpu", ) as reader: if not {"hidden_states", "offsets", "frame_ids"}.issubset( reader.keys() ): raise RuntimeError( f"Incomplete background hidden source: {source_path}" ) offsets_tensor = reader.get_tensor("offsets").to( dtype=torch.int64 ) frame_ids = reader.get_tensor("frame_ids").to( dtype=torch.int64 ) hidden_shape = tuple( int(value) for value in reader.get_slice("hidden_states").get_shape() ) if offsets_tensor.shape != (count + 1,): raise RuntimeError( f"Background offsets mismatch at {source_path}" ) expected_frames = ( torch.arange(count, dtype=torch.int64) * self.audio_sample_step_frames ) if not torch.equal(frame_ids, expected_frames): raise RuntimeError( f"Background frame IDs are not continuous: {source_path}" ) offsets = tuple(int(value) for value in offsets_tensor.tolist()) if offsets[0] != 0 or any( current <= previous for previous, current in zip(offsets, offsets[1:]) ): raise RuntimeError( f"Invalid background offsets at {source_path}" ) if len(hidden_shape) != 2 or offsets[-1] != hidden_shape[0]: raise RuntimeError( "Background hidden/offset size mismatch at " f"{source_path}: hidden={hidden_shape} offsets_end={offsets[-1]}" ) result[source_key] = BackgroundHiddenSource( source_key=source_key, packed_path=source_path, window_count=count, offsets=offsets, ) if not result: raise RuntimeError( f"No continuous background source indexes under {source_dir}" ) return result def _load_background_assignments( self, ) -> dict[str, SourceBackgroundAssignment]: if self.background_assignment_path is None: raise ValueError("background_assignment_path is not configured") if not self.background_assignment_path.is_file(): raise FileNotFoundError( f"Missing source/background assignments: " f"{self.background_assignment_path}" ) voices = { voice.source_key: voice for directory_voices in self.audio_catalog.values() for voice in directory_voices } result: dict[str, SourceBackgroundAssignment] = {} with self.background_assignment_path.open( "r", encoding="utf-8", ) as handle: for line in handle: if not line.strip(): continue raw = json.loads(line) source_key = str(raw.get("source_key", "")) if raw.get("format") != "hfd_continuous_background_assignment_v1": raise RuntimeError( f"Unsupported background assignment for {source_key}" ) if source_key not in voices: continue background_key = str(raw["background_key"]) background = self.background_catalog.get(background_key) if background is None: raise RuntimeError( f"Unknown background {background_key} for {source_key}" ) exported_listen_start = int( raw["background_listen_start_index"] ) exported_listen_count = int(raw["listen_window_count"]) exported_idle_max = int(raw["initial_idle_max_steps"]) exported_max_motion = int(raw["max_motion_steps"]) exported_safety = int(raw["safety_steps"]) speech_samples = int(raw.get("speech_samples", 0)) runtime_listen_count = self._runtime_listen_window_count( speech_samples ) listen_expansion = ( runtime_listen_count - exported_listen_count * self.exported_stride_scale ) assignment = SourceBackgroundAssignment( source_key=source_key, background_key=background_key, listen_start_index=( exported_listen_start * self.exported_stride_scale ), listen_window_count=runtime_listen_count, # Runtime background geometry is derived from the actual # session WAV, not from a lossy integer conversion. background_window_count=int(background.window_count), initial_idle_max_steps=( exported_idle_max * self.exported_stride_scale ), max_motion_steps=( exported_max_motion * self.exported_stride_scale ), safety_steps=max( exported_safety * self.exported_stride_scale - (self.exported_stride_scale - 1) - max(listen_expansion, 0), 0, ), mixed_audio_path=( relocate_hfd_storage_path( str(raw["mixed_audio_path"]) ) if self.online_audio else None ), original_audio_path=( relocate_hfd_storage_path( str(raw["original_audio_path"]) ) if self.online_audio else None ), speech_samples=speech_samples, speech_scale=float(raw.get("speech_scale", 1.0)), ) if ( exported_listen_count <= 0 or int(raw["background_window_count"]) <= 0 or assignment.listen_window_count != voices[source_key].window_count or assignment.background_window_count != background.window_count or ( not self.multi_turn_sessions and ( assignment.initial_idle_max_steps != self.initial_idle_max_steps or assignment.listen_start_index < self.initial_idle_max_steps ) ) ): raise RuntimeError( f"Source/background assignment mismatch: {source_key}" ) if ( assignment.listen_start_index + assignment.listen_window_count + assignment.max_motion_steps + assignment.safety_steps > assignment.background_window_count ): raise RuntimeError( f"Source/background assignment exceeds one session: " f"{source_key}" ) result[source_key] = assignment missing = sorted(set(voices) - set(result)) if missing: raise RuntimeError( "Continuous background assignments are incomplete: " f"missing={len(missing)} first={missing[:5]}" ) return result def _load_control_command_catalog( self, ) -> dict[str, dict[str, tuple[ControlCommandSource, ...]]]: """Load small TTS controls and verify all eight voice variants.""" path = self.control_command_manifest if path is None or not path.is_file(): raise FileNotFoundError(f"Missing control-command manifest: {path}") expected_voices = { f"{gender}_{speaker}_{language}" for gender in ("female", "male") for speaker in ("01", "02") for language in ("zh", "en") } grouped: dict[str, dict[str, list[ControlCommandSource]]] = {} phrase_voices: dict[tuple[str, str], set[str]] = {} with path.open("r", encoding="utf-8") as handle: for line in handle: if not line.strip(): continue raw = json.loads(line) if raw.get("format") != "hfd_control_command_tts_v1": raise RuntimeError( "Unsupported control-command manifest row: " f"{raw.get('format')!r}" ) intent = str(raw["intent"]) if intent not in {"emergency-stop", "safe-hold"}: raise RuntimeError(f"Unknown control-command intent: {intent}") voice_name = str(raw["voice_name"]) language = str(raw["language"]) if voice_name not in expected_voices or not voice_name.endswith( f"_{language}" ): raise RuntimeError( f"Invalid command voice/language: {voice_name}/{language}" ) waveform_path = relocate_hfd_storage_path( str(raw["audio_path"]) ) speech_samples = int(raw["speech_samples_16k"]) if not waveform_path.is_file() or speech_samples <= 0: raise FileNotFoundError( f"Incomplete control-command audio: {waveform_path}" ) source = ControlCommandSource( intent=intent, phrase_id=str(raw["phrase_id"]), voice_name=voice_name, language=language, text=str(raw["text"]).strip(), waveform_path=waveform_path, speech_samples=speech_samples, speech_scale=float(raw.get("speech_scale", 1.0)), ) if not source.text: raise RuntimeError( f"Empty control-command text: {intent}/{source.phrase_id}" ) grouped.setdefault(intent, {}).setdefault( voice_name, [], ).append(source) phrase_voices.setdefault( (intent, source.phrase_id), set(), ).add(voice_name) incomplete = { key: sorted(expected_voices.difference(voices)) for key, voices in phrase_voices.items() if voices != expected_voices } if incomplete: raise RuntimeError( "Every control-command meaning needs all eight voices: " f"{incomplete}" ) for intent in ("emergency-stop", "safe-hold"): if set(grouped.get(intent, {})) != expected_voices: raise RuntimeError( f"Control-command intent {intent} lacks an eight-voice set" ) return { intent: { voice_name: tuple( sorted(values, key=lambda item: item.phrase_id) ) for voice_name, values in by_voice.items() } for intent, by_voice in grouped.items() } def _load_backchannel_catalog( self, ) -> dict[str, tuple[BackchannelSource, ...]]: """Load held-out short acknowledgements for the active data split. Every phrase is rendered in all eight established session voices. The phrase split, rather than the speaker identity, is held out so validation tests whether the semantic behaviour generalises to unseen acknowledgements without introducing a new-speaker confound. """ path = self.backchannel_command_manifest if path is None or not path.is_file(): raise FileNotFoundError(f"Missing backchannel manifest: {path}") expected_voices = { f"{gender}_{speaker}_{language}" for gender in ("female", "male") for speaker in ("01", "02") for language in ("zh", "en") } grouped: dict[str, list[BackchannelSource]] = {} phrase_voices: dict[tuple[str, str], set[str]] = {} phrase_splits: dict[str, str] = {} split_categories: dict[str, set[str]] = {"train": set(), "val": set()} with path.open("r", encoding="utf-8") as handle: for line in handle: if not line.strip(): continue raw = json.loads(line) if raw.get("format") != "hfd_backchannel_tts_v1": raise RuntimeError( "Unsupported backchannel manifest row: " f"{raw.get('format')!r}" ) category = str(raw["category"]) if category not in {"continuer", "assessment"}: raise RuntimeError( f"Unknown backchannel category: {category!r}" ) phrase_id = str(raw["phrase_id"]) phrase_split = str(raw["split"]) if phrase_split not in {"train", "val"}: raise RuntimeError( f"Invalid backchannel phrase split: {phrase_split!r}" ) previous_split = phrase_splits.setdefault( phrase_id, phrase_split, ) if previous_split != phrase_split: raise RuntimeError( f"Backchannel phrase crosses splits: {phrase_id}" ) split_categories[phrase_split].add(category) voice_name = str(raw["voice_name"]) language = str(raw["language"]) if voice_name not in expected_voices or not voice_name.endswith( f"_{language}" ): raise RuntimeError( "Invalid backchannel voice/language: " f"{voice_name}/{language}" ) waveform_path = relocate_hfd_storage_path( str(raw["audio_path"]) ) speech_samples = int(raw["speech_samples_16k"]) if not waveform_path.is_file() or speech_samples <= 0: raise FileNotFoundError( f"Incomplete backchannel audio: {waveform_path}" ) if speech_samples >= 2 * self.audio_sampling_rate: raise RuntimeError( "Backchannel utterances must stay below two seconds: " f"{waveform_path}" ) source = BackchannelSource( category=category, phrase_id=phrase_id, split=phrase_split, voice_name=voice_name, language=language, text=str(raw["text"]).strip(), waveform_path=waveform_path, speech_samples=speech_samples, speech_scale=float(raw.get("speech_scale", 1.0)), ) if not source.text: raise RuntimeError( f"Empty backchannel text: {source.phrase_id}" ) phrase_voices.setdefault( (phrase_split, phrase_id), set(), ).add(voice_name) if phrase_split == self.split: grouped.setdefault(voice_name, []).append(source) incomplete = { key: sorted(expected_voices.difference(voices)) for key, voices in phrase_voices.items() if voices != expected_voices } if incomplete: raise RuntimeError( "Every backchannel meaning needs all eight voices: " f"{incomplete}" ) if any( categories != {"continuer", "assessment"} for categories in split_categories.values() ): raise RuntimeError( "Train and validation backchannel phrases must each include " f"continuer and assessment categories: {split_categories}" ) if set(grouped) != expected_voices: raise RuntimeError( f"Backchannel split {self.split!r} lacks an eight-voice set" ) return { voice_name: tuple( sorted(values, key=lambda item: item.phrase_id) ) for voice_name, values in grouped.items() } def _load_hesitation_pause_catalog( self, ) -> tuple[ dict[tuple[str, str], dict[str, DisfluencyAudioSource]], dict[str, dict[str, tuple[DisfluencyAudioSource, ...]]], ]: """Load regenerated command halves and held-out hesitation markers. Prefix/suffix rows are shared linguistic material keyed by the original TTS directory and session voice. Filled-pause and repair-cue phrase IDs are train/validation disjoint, preventing validation from succeeding by memorising one fixed ``um`` or ``I mean`` waveform. """ path = self.hesitation_pause_manifest if path is None or not path.is_file(): raise FileNotFoundError(f"Missing hesitation/pause manifest: {path}") manifest_sha256 = hashlib.sha256(path.read_bytes()).hexdigest() production_marker_path = ( path.parent / "reports" / "production_complete.json" ) if not production_marker_path.is_file(): raise FileNotFoundError( "Missing hesitation/pause production marker: " f"{production_marker_path}" ) production_marker = json.loads( production_marker_path.read_text(encoding="utf-8") ) if ( production_marker.get("format") != "hfd_hesitation_pause_production_v1" or production_marker.get("manifest_format") != HESITATION_PAUSE_MANIFEST_FORMAT or production_marker.get("manifest_sha256") != manifest_sha256 or int(production_marker.get("manifest_rows", 0)) <= 0 or int(production_marker.get("voice_count", 0)) != 8 ): raise RuntimeError( "Invalid hesitation/pause production marker: " f"{production_marker_path}" ) expected_by_language = { language: { f"{gender}_{speaker}_{language}" for gender in ("female", "male") for speaker in ("01", "02") } for language in ("en", "zh") } segments: dict[ tuple[str, str], dict[str, DisfluencyAudioSource] ] = {} markers: dict[str, dict[str, list[DisfluencyAudioSource]]] = { voice: {"hesitation_filler": [], "repair_cue": []} for voices in expected_by_language.values() for voice in voices } segment_voice_audit: dict[tuple[str, str, str], set[str]] = {} marker_voice_audit: dict[tuple[str, str, str], set[str]] = {} with path.open("r", encoding="utf-8") as handle: for line in handle: if not line.strip(): continue raw = json.loads(line) if raw.get("format") != HESITATION_PAUSE_MANIFEST_FORMAT: raise RuntimeError( "Unsupported hesitation/pause manifest row: " f"{raw.get('format')!r}" ) category = str(raw.get("category", "")) if category not in { "instruction_segment", "hesitation_filler", "repair_cue", }: raise RuntimeError( f"Unknown hesitation/pause category: {category!r}" ) language = str(raw.get("language", "")) voice_name = str(raw.get("voice_name", "")) if ( language not in expected_by_language or voice_name not in expected_by_language[language] ): raise RuntimeError( "Invalid hesitation/pause voice/language: " f"{voice_name}/{language}" ) waveform_path = relocate_hfd_storage_path(str(raw["audio_path"])) speech_samples = int(raw.get("speech_samples_16k", 0)) # ``finalize`` has already opened and audited every WAV, and # the marker above binds that complete audit to this exact # manifest. Avoid reopening 81k small files on every DDP rank # during startup; the selected WAV is opened and its sample # count rechecked by ``_read_disfluency_audio`` at first use. if speech_samples <= 0: raise RuntimeError( f"Invalid hesitation/pause sample count: {waveform_path}" ) source = DisfluencyAudioSource( category=category, segment_id=str(raw["segment_id"]), split=str(raw.get("split", "shared")), voice_name=voice_name, language=language, text=str(raw.get("text", "")).strip(), waveform_path=waveform_path, speech_samples=speech_samples, speech_scale=float(raw.get("speech_scale", 1.0)), audio_dir=_normalise_audio_dir(str(raw.get("audio_dir", ""))), instruction=str(raw.get("instruction", "")).strip(), instruction_zh=str(raw.get("instruction_zh", "")).strip(), segment_role=str(raw.get("segment_role", "")), ) if not source.text: raise RuntimeError( f"Empty hesitation/pause TTS text: {source.segment_id}" ) if category == "instruction_segment": if ( source.split != "shared" or source.segment_role not in {"prefix", "suffix"} or not source.audio_dir or not source.instruction or not source.instruction_zh ): raise RuntimeError( f"Incomplete instruction segment: {source}" ) key = (source.audio_dir, voice_name) if source.segment_role in segments.setdefault(key, {}): raise RuntimeError( "Duplicate hesitation instruction segment: " f"{key}/{source.segment_role}" ) segments[key][source.segment_role] = source segment_voice_audit.setdefault( (source.audio_dir, source.segment_role, language), set(), ).add(voice_name) else: if source.split not in {"train", "val"}: raise RuntimeError( f"Invalid marker split: {source.split!r}" ) marker_voice_audit.setdefault( (source.split, category, source.segment_id), set(), ).add(voice_name) if source.split == self.split: markers[voice_name][category].append(source) incomplete_segments = { key: sorted(expected_by_language[key[2]].difference(voices)) for key, voices in segment_voice_audit.items() if voices != expected_by_language[key[2]] } incomplete_markers: dict[tuple[str, str, str], list[str]] = {} # Each marker meaning is generated only in its own language, hence four # expected voices rather than all eight bilingual identities. for (marker_split, category, segment_id), voices in marker_voice_audit.items(): languages = {voice.rsplit("_", 1)[-1] for voice in voices} if len(languages) != 1: raise RuntimeError( f"Marker phrase crosses languages: {segment_id}/{languages}" ) expected = expected_by_language[next(iter(languages))] if voices != expected: incomplete_markers[(marker_split, category, segment_id)] = sorted( expected.difference(voices) ) if incomplete_segments or incomplete_markers: raise RuntimeError( "Incomplete hesitation/pause voice coverage: " f"segments={list(incomplete_segments.items())[:3]} " f"markers={list(incomplete_markers.items())[:3]}" ) malformed_pairs = { key: sorted(by_role) for key, by_role in segments.items() if set(by_role) != {"prefix", "suffix"} } if malformed_pairs: raise RuntimeError( "Every splittable command/voice needs prefix and suffix: " f"{list(malformed_pairs.items())[:3]}" ) missing_marker_categories = { voice: [ category for category, values in by_category.items() if not values ] for voice, by_category in markers.items() if any(not values for values in by_category.values()) } if missing_marker_categories: raise RuntimeError( f"Hesitation marker split {self.split!r} is incomplete: " f"{missing_marker_categories}" ) return ( segments, { voice: { category: tuple( sorted(values, key=lambda item: item.segment_id) ) for category, values in by_category.items() } for voice, by_category in markers.items() }, ) def _compose_sessions( self, records: list[dict[str, Any]], ) -> tuple[tuple[dict[str, Any], ...], ...]: """Build deterministic 1--4 turn sessions for one already-split set. Historical versions partition every primitive exactly once and retain the per-record voice. Version 17 creates static voice variants: every turn in one session uses the same speaker/language, and every primitive appears once per train variant with a different voice when available. Validation remains one deterministic voice variant. Version 20 maps the two train variants to turn-taking/interruption. Version 21 restores both complete turn-taking variants and builds every added task in an independent pool, so changing one task's weight cannot remove or reshuffle source usage from another task. Train and validation were separated first, so neither path can cross the split. """ if not self.multi_turn_sessions: return tuple((record,) for record in records) if len(records) < self.session_min_turns: raise RuntimeError( "Multi-turn construction needs at least " f"{self.session_min_turns} primitive sources, got {len(records)}" ) split_salt = 0 if self.split == "train" else 1_000_003 legacy_rng = random.Random(self.split_seed * 97_409 + split_salt) voice_variants = int(getattr(self, "session_voice_variants", 0)) if voice_variants <= 0: return self._partition_session_records(records, legacy_rng) interruption_enabled = bool( getattr(self, "interruption_sessions", False) ) independent_pools = bool( getattr(self, "independent_task_pools", False) ) if interruption_enabled and independent_pools: # Keep the two v19 turn-taking compositions on their historical # salts 0/1. Additional tasks live in disjoint salt ranges and use # their own speaker bookkeeping; this is the key independence # guarantee requested for task weighting. task_offsets = { "turn-taking": 0, "interruption": 100, "emergency-stop": 200, "safe-hold": 300, "backchannel": 400, "hesitation": 500, "pause": 600, } variant_plan = tuple( ( task_offsets[mode] + variant_index, variant_index if self.split == "train" else 0, mode, ) for mode in ( "turn-taking", "interruption", "emergency-stop", "safe-hold", "backchannel", "hesitation", "pause", ) for variant_index in range( int(self.task_session_variants.get(mode, 0)) if self.split == "train" else int(self.task_session_variants.get(mode, 0) > 0) ) ) elif interruption_enabled: if voice_variants != 2: raise RuntimeError( "Interruption v1 requires exactly two session variants" ) # Training keeps the existing two primitive uses: variant 0 is # ordinary turn-taking and variant 1 is interruption. Validation # evaluates both behaviors with the same deterministic voice, so # timing is the only changed variable. legacy_plan = ( ((0, "turn-taking"), (1, "interruption")) if self.split == "train" else ((0, "turn-taking"), (0, "interruption")) ) variant_plan = tuple( (composition_variant, voice_variant, interaction_mode) for composition_variant, (voice_variant, interaction_mode) in enumerate(legacy_plan) ) else: effective_variants = voice_variants if self.split == "train" else 1 variant_plan = tuple( (variant_index, variant_index, "turn-taking") for variant_index in range(effective_variants) ) if independent_pools: used_voices_by_mode: dict[str, dict[int, set[str]]] = { mode: {int(record["row_id"]): set() for record in records} for mode in {item[2] for item in variant_plan} } else: # Historical v17--v20 variants deliberately rotate the voice of a # primitive across their complete shared variant plan. Preserve # that exact behavior for old checkpoints and tests. shared_used_voices = { int(record["row_id"]): set() for record in records } used_voices_by_mode = { mode: shared_used_voices for mode in {item[2] for item in variant_plan} } sessions: list[tuple[dict[str, Any], ...]] = [] expected_uses = 0 control_filtered_uses = { "emergency-stop": 0, "safe-hold": 0, } voice_reuse_after_coverage: dict[str, int] = { mode: 0 for mode in {item[2] for item in variant_plan} } task_local_session_counts: dict[str, int] = {} for composition_variant, voice_variant, interaction_mode in variant_plan: # One action plus Stop/backchannel/disfluency is a real event. # Command interruption needs a SECOND action instruction, however. minimum_turns = max( self.session_min_turns, 2 if interaction_mode == "interruption" else 1, ) used_voices = used_voices_by_mode[interaction_mode] rng = random.Random( self.split_seed * 97_409 + split_salt + (composition_variant + 1) * 15_485_863 ) mode_records = ( self._select_new_task_records(records) if interaction_mode in {"hesitation", "pause"} else list(records) ) shuffled = list(mode_records) rng.shuffle(shuffled) buckets: dict[str, list[dict[str, Any]]] = {} assignments: dict[int, str] = {} for record in shuffled: row_id = int(record["row_id"]) candidates = self._session_voice_names_for_mode( record, interaction_mode, ) available = ( [ name for name in candidates if name not in used_voices[row_id] ] if self.split == "train" else list(candidates) ) if not available: if independent_pools and candidates: # Cover every valid rendition before reusing one when # a task asks for more composition variants than this # rare primitive has voices. Session speaker identity # and independently shuffled neighbours stay intact. available = list(candidates) voice_reuse_after_coverage[interaction_mode] += 1 else: raise RuntimeError( "A primitive has fewer distinct valid voices than " "the requested session variants: " f"row={row_id} mode={interaction_mode} " f"voices={candidates}" ) chooser = random.Random( self.split_seed * 1_000_003 + row_id * 97_409 ) chooser.shuffle(available) voice_name = available[0] assignments[row_id] = voice_name buckets.setdefault(voice_name, []).append(record) # Tasks requiring >=2 turns cannot contain a one-row voice bucket. Move # its row to another still-unused supported voice that already has # at least one row. This preserves both the one-speaker session # contract and distinct per-primitive voices across variants. while minimum_turns > 1: singleton_names = sorted( name for name, bucket in buckets.items() if len(bucket) == 1 ) if not singleton_names: break moved = False for voice_name in singleton_names: bucket = buckets.get(voice_name, []) if len(bucket) != 1: continue record = bucket[0] row_id = int(record["row_id"]) alternatives = [ name for name in self._session_voice_names_for_mode( record, interaction_mode, ) if name != voice_name and name not in used_voices[row_id] and len(buckets.get(name, ())) >= 1 ] if not alternatives and independent_pools: alternatives = [ name for name in self._session_voice_names_for_mode( record, interaction_mode, ) if name != voice_name and len(buckets.get(name, ())) >= 1 ] if not alternatives: continue target_name = min( alternatives, key=lambda name: (len(buckets[name]), name), ) buckets[voice_name] = [] buckets[target_name].append(record) assignments[row_id] = target_name moved = True if not moved: unresolved = { name: [int(record["row_id"]) for record in bucket] for name, bucket in buckets.items() if len(bucket) == 1 } raise RuntimeError( "Cannot form a same-voice multi-turn session from the " f"available renditions: {unresolved}" ) for voice_name in sorted(buckets): bucket = buckets[voice_name] if not bucket: continue selected_bucket = bucket if interaction_mode in {"emergency-stop", "safe-hold"}: selected_bucket = [ record for record in bucket if self._control_motion_is_eligible( record, intent=interaction_mode, voice_name=voice_name, ) ] control_filtered_uses[interaction_mode] += ( len(bucket) - len(selected_bucket) ) # A one-row remainder cannot form the promised 2--4-turn # session. Drop that task-specific use only; the same # primitive remains untouched in turn-taking and ordinary # interruption pools. if len(selected_bucket) < self.session_min_turns: control_filtered_uses[interaction_mode] += len( selected_bucket ) continue elif interaction_mode == "backchannel": selected_bucket = [ record for record in bucket if self._backchannel_motion_is_eligible( record, voice_name=voice_name, ) ] if len(selected_bucket) < self.session_min_turns: continue expected_uses += len(selected_bucket) bucket_sessions = ( self._partition_interruption_session_records( selected_bucket, rng, ) if interaction_mode == "interruption" else self._partition_session_records(selected_bucket, rng) ) for session in bucket_sessions: task_local_index = task_local_session_counts.get( interaction_mode, 0, ) control_turn_index: int | None = None control_has_followup = False backchannel_turn_index: int | None = None disfluency_turn_index: int | None = None disfluency_subtype: str | None = None disfluency_gap_steps: int | None = None disfluency_marker_id: str | None = None disfluency_repair_style: str | None = None provisional_record: dict[str, Any] | None = None if ( interaction_mode in {"emergency-stop", "safe-hold"} and float( getattr(self, "control_task_session_ratio", 0.0) ) > 0.0 ): # Balanced control data has exactly one stop/hold event # per session. A configurable subset places it before # the final turn, leaving a real background pause and # then an ordinary new instruction/action. The rest # keeps the event terminal and still retains a # background-only tail. control_has_followup = ( len(session) > 1 and rng.random() < float( getattr(self, "control_followup_ratio", 0.0) ) ) control_turn_index = ( rng.randrange(len(session) - 1) if control_has_followup else len(session) - 1 ) if interaction_mode == "backchannel": backchannel_turn_index = rng.randrange(len(session)) if interaction_mode in {"hesitation", "pause"}: disfluency_turn_index = rng.randrange(len(session)) disfluency_gap_steps = self._sample_disfluency_gap_steps( rng ) if interaction_mode == "pause": disfluency_subtype = "silent-pause" else: disfluency_subtype = ( "filled-pause" if rng.random() < self.hesitation_filled_ratio else "self-repair" ) if disfluency_subtype == "self-repair": disfluency_repair_style = ( "implicit-restart" if rng.random() < self.self_repair_implicit_ratio else "explicit-cue" ) final_record = session[disfluency_turn_index] final_key = ( str(final_record["instruction"]).strip(), str(final_record["instruction_zh"]).strip(), ) provisional_candidates = [ candidate for candidate in selected_bucket if ( str(candidate["instruction"]).strip(), str(candidate["instruction_zh"]).strip(), ) != final_key and ( str(candidate["audio_dir"]), voice_name, ) in self.disfluency_instruction_segments ] if provisional_candidates: provisional_record = rng.choice( provisional_candidates ) else: # Extremely homogeneous synthetic fixtures # cannot express a semantic self-repair. Keep # the sample useful as filled hesitation # instead of inventing a same-intent repair. disfluency_subtype = "filled-pause" disfluency_repair_style = None if ( disfluency_subtype == "filled-pause" or disfluency_repair_style == "explicit-cue" ): marker_category = ( "hesitation_filler" if disfluency_subtype == "filled-pause" else "repair_cue" ) marker_candidates = ( self.disfluency_marker_catalog.get( voice_name, {}, ).get(marker_category, ()) ) if not marker_candidates: raise RuntimeError( f"No {marker_category} for split={self.split} " f"voice={voice_name}" ) marker_source = rng.choice(marker_candidates) disfluency_marker_id = marker_source.segment_id minimum_span = int( marker_source.speech_samples + round(0.16 * self.audio_sampling_rate) ) disfluency_gap_steps = max( int(disfluency_gap_steps), int( math.ceil( minimum_span / self.audio_stride_samples ) ), ) if disfluency_gap_steps > 7: raise RuntimeError( "A hesitation marker cannot fit the audited " f"2.8 s span: {marker_source.waveform_path}" ) stamped: list[dict[str, Any]] = [] for record in session: row_id = int(record["row_id"]) if assignments[row_id] != voice_name: raise AssertionError("Session voice assignment drifted") copied = dict(record) copied["_session_voice_name"] = voice_name copied["_session_voice_variant"] = voice_variant copied["_session_interaction_mode"] = interaction_mode if control_turn_index is not None: copied["_session_control_turn_index"] = int( control_turn_index ) copied["_session_control_has_followup"] = bool( control_has_followup ) if backchannel_turn_index is not None: copied["_session_backchannel_turn_index"] = int( backchannel_turn_index ) if disfluency_turn_index is not None: copied["_session_disfluency_turn_index"] = int( disfluency_turn_index ) copied["_session_disfluency_subtype"] = str( disfluency_subtype ) copied["_session_disfluency_gap_steps"] = int( disfluency_gap_steps or 1 ) copied["_session_disfluency_marker_id"] = ( disfluency_marker_id ) copied["_session_disfluency_repair_style"] = ( disfluency_repair_style or "" ) if provisional_record is not None: copied["_session_provisional_audio_dir"] = str( provisional_record["audio_dir"] ) copied["_session_provisional_instruction"] = str( provisional_record["instruction"] ) copied[ "_session_provisional_instruction_zh" ] = str(provisional_record["instruction_zh"]) if independent_pools: # Preserve v19's deterministic validation # augmentation for the complete turn-taking pool # even after all independent task sessions are # mixed into one outer dataset index space. copied["_session_pool_index"] = task_local_index stamped.append(copied) sessions.append(tuple(stamped)) task_local_session_counts[interaction_mode] = ( task_local_index + 1 ) if self.split == "train": for row_id, voice_name in assignments.items(): used_voices[row_id].add(voice_name) actual_uses = sum(len(session) for session in sessions) if actual_uses != expected_uses: raise AssertionError( "Static session voice variants lost primitive rows: " f"expected={expected_uses} actual={actual_uses}" ) self.control_task_rows_filtered = control_filtered_uses self.voice_reuse_after_coverage = voice_reuse_after_coverage if interruption_enabled: # Do not leave all turn-taking sessions before all interruption # sessions. Evaluation intentionally consumes only a bounded # prefix, while training groups sources by length; a deterministic # final shuffle lets both paths observe both interaction modes # without changing primitive usage, speakers, or session contents. mix_rng = random.Random( self.split_seed * 1_000_003 + split_salt + 20 * 15_485_863 ) mix_rng.shuffle(sessions) return tuple(sessions) def _balance_control_task_sessions( self, sessions: tuple[tuple[dict[str, Any], ...], ...], ) -> tuple[tuple[dict[str, Any], ...], ...]: """Make enabled interruption/control pools exact final-session ratios. Balancing runs only after context and background filtering, so the printed train/validation proportions describe the data that can actually reach a worker. Turn-taking and ordinary interruption are never removed or reshuffled relative to one another. """ ratio = float(self.control_task_session_ratio) control_modes = tuple( mode for mode in ("emergency-stop", "safe-hold") if int(self.task_session_variants.get(mode, 0)) > 0 ) grouped: dict[str, list[tuple[dict[str, Any], ...]]] = { mode: [] for mode in control_modes } non_control: list[tuple[dict[str, Any], ...]] = [] for session in sessions: mode = str( session[0].get("_session_interaction_mode", "turn-taking") ) if mode in grouped: grouped[mode].append(session) else: non_control.append(session) interruption_ratio = float( getattr(self, "interruption_task_session_ratio", 0.0) ) if interruption_ratio > 0.0: interruption_pool = [ session for session in non_control if str( session[0].get( "_session_interaction_mode", "turn-taking", ) ) == "interruption" ] turn_taking_pool = [ session for session in non_control if str( session[0].get( "_session_interaction_mode", "turn-taking", ) ) != "interruption" ] control_total_ratio = ratio * len(control_modes) turn_taking_ratio = ( 1.0 - control_total_ratio - interruption_ratio ) if turn_taking_ratio <= 0.0: raise ValueError( "Task ratios leave no turn-taking sessions: " f"interruption={interruption_ratio:g} " f"control_total={control_total_ratio:g}" ) target_interruption = int( round( len(turn_taking_pool) * interruption_ratio / turn_taking_ratio ) ) if len(interruption_pool) < target_interruption: raise RuntimeError( "Not enough interruption sessions for requested final " f"ratio={interruption_ratio:g}: " f"target={target_interruption} " f"available={len(interruption_pool)}" ) split_salt = 0 if self.split == "train" else 1_000_003 interruption_rng = random.Random( self.split_seed * 1_000_003 + split_salt + 22 * 15_485_863 ) interruption_rng.shuffle(interruption_pool) non_control = turn_taking_pool + interruption_pool[ :target_interruption ] self.interruption_task_sessions_available = len( interruption_pool ) self.interruption_task_sessions_selected = target_interruption denominator = 1.0 - ratio * len(control_modes) target_per_mode = int(round(len(non_control) * ratio / denominator)) available = {mode: len(values) for mode, values in grouped.items()} insufficient = { mode: count for mode, count in available.items() if count < target_per_mode } if insufficient: raise RuntimeError( "Not enough eligible control sessions for the requested " f"ratio={ratio:g}: target={target_per_mode} " f"available={available}" ) split_salt = 0 if self.split == "train" else 1_000_003 rng = random.Random( self.split_seed * 1_000_003 + split_salt + 21 * 15_485_863 ) selected = list(non_control) for mode in control_modes: pool = list(grouped[mode]) rng.shuffle(pool) selected.extend(pool[:target_per_mode]) rng.shuffle(selected) self.control_task_sessions_available = available self.control_task_sessions_selected = { mode: target_per_mode for mode in control_modes } return tuple(selected) def _balance_backchannel_task_sessions( self, sessions: tuple[tuple[dict[str, Any], ...], ...], ) -> tuple[tuple[dict[str, Any], ...], ...]: """Retain the exact v22 selection, then append v23 listen hard cases. The pre-backchannel v22 mixture used the complete turn-taking pool as its fixed reference. Backchannel v1 keeps that invariant and enlarges the other pools to the requested 60/15/10/15 final proportions. It therefore never obtains a prettier ratio by throwing away established turn-taking supervision. """ legacy_ratios = { "interruption": float(self.interruption_task_session_ratio), "emergency-stop": ( float(self.control_task_session_ratio) if int(self.task_session_variants.get("emergency-stop", 0)) > 0 else 0.0 ), "safe-hold": ( float(self.control_task_session_ratio) if int(self.task_session_variants.get("safe-hold", 0)) > 0 else 0.0 ), "backchannel": float(self.backchannel_task_session_ratio), } turn_ratio = 1.0 - sum(legacy_ratios.values()) if not math.isfinite(turn_ratio) or turn_ratio <= 0.0: raise ValueError( f"Task ratios leave no turn-taking sessions: {legacy_ratios}" ) new_window_ratios = { mode: float(ratio) for mode, ratio in { "hesitation": float( getattr(self, "hesitation_task_window_ratio", 0.0) ), "pause": float( getattr(self, "pause_task_window_ratio", 0.0) ), }.items() if float(ratio) > 0.0 } grouped: dict[str, list[tuple[dict[str, Any], ...]]] = { mode: [] for mode in ( "turn-taking", *legacy_ratios, *new_window_ratios, ) } for session in sessions: mode = str( session[0].get("_session_interaction_mode", "turn-taking") ) if mode not in grouped: raise RuntimeError( f"Unexpected task pool during backchannel balance: {mode}" ) grouped[mode].append(session) turn_pool = grouped["turn-taking"] if not turn_pool: raise RuntimeError("Backchannel mixture has no turn-taking pool") target_total = int(round(len(turn_pool) / turn_ratio)) legacy_targets = { mode: int(round(target_total * ratio)) for mode, ratio in legacy_ratios.items() if ratio > 0.0 } insufficient = { mode: { "target": target, "available": len(grouped[mode]), } for mode, target in legacy_targets.items() if len(grouped[mode]) < target } if insufficient: raise RuntimeError( "Not enough independent sessions for the requested " f"backchannel mixture: {insufficient}" ) split_salt = 0 if self.split == "train" else 1_000_003 rng = random.Random( self.split_seed * 1_000_003 + split_salt + 23 * 15_485_863 ) selected = list(turn_pool) available: dict[str, int] = {} selected_counts: dict[str, int] = {"turn-taking": len(turn_pool)} for mode in legacy_ratios: pool_values = grouped[mode] pool = list(pool_values) available[mode] = len(pool) target = legacy_targets.get(mode, 0) rng.shuffle(pool) selected.extend(pool[:target]) selected_counts[mode] = target legacy_selected = list(selected) if not new_window_ratios: # Preserve the exact v22 path. Besides avoiding unnecessary # length materialisation, this keeps legacy-only unit fixtures # and runs independent of the v23 timeline fields. rng.shuffle(selected) self.task_sessions_available = { "turn-taking": len(turn_pool), **available, } self.task_sessions_selected = selected_counts self.task_windows_selected = {} return tuple(selected) legacy_window_count = sum( self._cached_session_max_stream_steps(session) for session in legacy_selected ) new_ratio_sum = sum(new_window_ratios.values()) if new_ratio_sum >= 1.0: raise ValueError( "Hesitation/pause ratios leave no established task windows: " f"{new_window_ratios}" ) selected_window_counts = { "legacy-total": int(legacy_window_count) } for mode, ratio in new_window_ratios.items(): pool = list(grouped[mode]) available[mode] = len(pool) rng.shuffle(pool) target_windows = int( round(legacy_window_count * ratio / (1.0 - new_ratio_sum)) ) chosen: list[tuple[dict[str, Any], ...]] = [] chosen_windows = 0 for candidate in pool: candidate_windows = self._cached_session_max_stream_steps( candidate ) before_error = abs(target_windows - chosen_windows) after_error = abs( target_windows - chosen_windows - candidate_windows ) if chosen_windows >= target_windows: break if after_error > before_error and chosen: break chosen.append(candidate) chosen_windows += candidate_windows if target_windows > 0 and not chosen: raise RuntimeError( f"No {mode} session can satisfy target window budget " f"{target_windows}" ) if chosen_windows < int(round(target_windows * 0.98)): raise RuntimeError( f"Not enough {mode} windows for requested share: " f"target={target_windows} selected={chosen_windows} " f"available_sessions={len(pool)}" ) selected.extend(chosen) selected_counts[mode] = len(chosen) selected_window_counts[mode] = int(chosen_windows) rng.shuffle(selected) self.task_sessions_available = { "turn-taking": len(turn_pool), **available, } self.task_sessions_selected = selected_counts self.task_windows_selected = selected_window_counts return tuple(selected) def _drop_longest_sessions_by_mode( self, sessions: tuple[tuple[dict[str, Any], ...], ...], ) -> tuple[tuple[dict[str, Any], ...], ...]: """Remove the deterministic longest tail without changing task ratios. Filtering is performed independently inside each interaction mode. A global cutoff would disproportionately remove interruption or control sessions and silently change the requested task mixture. The stable original index breaks equal-length ties, so every rank and every restart builds the same retained set. """ ratio = float(self.long_session_drop_ratio) if ratio <= 0.0: return sessions grouped: dict[str, list[tuple[int, int]]] = {} for index, session in enumerate(sessions): mode = str( session[0].get("_session_interaction_mode", "turn-taking") ) grouped.setdefault(mode, []).append( (index, self._cached_session_max_stream_steps(session)) ) dropped: set[int] = set() summary: dict[str, dict[str, float | int]] = {} for mode, values in sorted(grouped.items()): drop_count = min( int(math.ceil(len(values) * ratio)), max(len(values) - 1, 0), ) ordered = sorted(values, key=lambda item: (item[1], item[0])) selected = ordered[-drop_count:] if drop_count else [] dropped.update(index for index, _ in selected) retained_lengths = [ length for index, length in values if index not in dropped ] summary[mode] = { "before": len(values), "dropped": drop_count, "retained": len(values) - drop_count, "shortest_dropped_steps": ( min(length for _, length in selected) if selected else 0 ), "longest_retained_steps": ( max(retained_lengths) if retained_lengths else 0 ), } self.long_session_filter_summary = summary return tuple( session for index, session in enumerate(sessions) if index not in dropped ) def _interruption_motion_is_eligible( self, record: dict[str, Any], ) -> bool: """Whether a primitive can be interrupted after 2 s with a real tail.""" required_frames = int( math.ceil( ( self.interruption_min_start_seconds + self.interruption_min_remaining_seconds ) * self.motion_fps ) ) return self._estimated_runtime_motion_frames(record) >= required_frames def _partition_interruption_session_records( self, records: list[dict[str, Any]], rng: random.Random, ) -> tuple[tuple[dict[str, Any], ...], ...]: """Put every short primitive last so all interrupted turns are valid.""" eligible = [ record for record in records if self._interruption_motion_is_eligible(record) ] terminal_only = [ record for record in records if not self._interruption_motion_is_eligible(record) ] rng.shuffle(eligible) rng.shuffle(terminal_only) count = len(records) minimum_turns = max(2, self.session_min_turns) if self.session_max_turns < minimum_turns: raise ValueError("Command interruption requires at least two source turns") minimum_sessions = int(math.ceil(count / self.session_max_turns)) maximum_sessions = count // minimum_turns target_sessions = int(round(count / 2.7)) session_count = min( max(target_sessions, minimum_sessions, len(terminal_only)), maximum_sessions, len(eligible), ) if ( session_count < minimum_sessions or session_count < len(terminal_only) or session_count > len(eligible) ): raise RuntimeError( "Cannot form interruption sessions with a valid non-final " "motion in every turn: " f"rows={count} eligible={len(eligible)} " f"terminal_only={len(terminal_only)}" ) final_rows = list(terminal_only) final_rows.extend(eligible[: session_count - len(final_rows)]) nonfinal_rows = eligible[session_count - len(terminal_only) :] rng.shuffle(final_rows) rng.shuffle(nonfinal_rows) buckets: list[list[dict[str, Any]]] = [ [nonfinal_rows.pop()] for _ in range(session_count) ] # Fill the third turn before the fourth, so 2/3-turn sessions dominate. order = list(range(session_count)) rng.shuffle(order) while nonfinal_rows: progressed = False for bucket_index in order: if len(buckets[bucket_index]) >= self.session_max_turns - 1: continue buckets[bucket_index].append(nonfinal_rows.pop()) progressed = True if not nonfinal_rows: break if not progressed: raise AssertionError("Interruption partition exceeded four turns") sessions = [ tuple([*bucket, final_row]) for bucket, final_row in zip(buckets, final_rows) ] rng.shuffle(sessions) if any( not minimum_turns <= len(session) <= self.session_max_turns or any( not self._interruption_motion_is_eligible(record) for record in session[:-1] ) for session in sessions ): raise AssertionError("Invalid interruption session partition") return tuple(sessions) def _partition_session_records( self, records: list[dict[str, Any]], rng: random.Random, ) -> tuple[tuple[dict[str, Any], ...], ...]: """Partition records into valid groups while preserving legacy RNG.""" shuffled = list(records) rng.shuffle(shuffled) sessions: list[tuple[dict[str, Any], ...]] = [] cursor = 0 while cursor < len(shuffled): remaining = len(shuffled) - cursor if remaining <= self.session_max_turns: size = remaining else: size = rng.randint( self.session_min_turns, self.session_max_turns, ) leftover = remaining - size if 0 < leftover < self.session_min_turns: size = remaining - self.session_min_turns if size < self.session_min_turns: # Only a one-row tail can reach here. Move one row from the # previous group while respecting the four-turn ceiling. if not sessions or len(sessions[-1]) <= self.session_min_turns: raise RuntimeError( f"Cannot form valid multi-turn tail of {remaining} rows" ) previous = list(sessions.pop()) borrowed = previous.pop() sessions.append(tuple(previous)) sessions.append((borrowed, *shuffled[cursor:])) cursor = len(shuffled) break sessions.append(tuple(shuffled[cursor : cursor + size])) cursor += size if any( not self.session_min_turns <= len(session) <= self.session_max_turns for session in sessions ): raise AssertionError("Invalid deterministic multi-turn partition") return tuple(sessions) @staticmethod def _voice_name(voice: AudioSource) -> str: """Return the speaker/language identity shared across command dirs.""" return Path(voice.source_key).name def _record_voices( self, record: dict[str, Any], ) -> tuple[AudioSource, ...]: """Return only semantically valid renditions for one motion record.""" voices = self.audio_catalog[record["audio_dir"]] if bool(record.get("_exclude_zh_voices", False)): voices = tuple( voice for voice in voices if not self._voice_name(voice).endswith("_zh") ) if not voices: raise RuntimeError( "No semantically valid TTS rendition remains for " f"row={record.get('row_id')} audio_dir={record.get('audio_dir')}" ) return voices def _session_voice_names(self, record: dict[str, Any]) -> tuple[str, ...]: names = tuple( sorted( { self._voice_name(voice) for voice in self._record_voices(record) } ) ) if not names: raise RuntimeError( f"No valid voice identity for row={record.get('row_id')}" ) return names def _session_voice_names_for_mode( self, record: dict[str, Any], interaction_mode: str, ) -> tuple[str, ...]: names = self._session_voice_names(record) if interaction_mode not in {"hesitation", "pause"}: return names supported = tuple( name for name in names if (str(record["audio_dir"]), name) in self.disfluency_instruction_segments ) if not supported: raise RuntimeError( "A hesitation/pause record has no regenerated split speech: " f"row={record.get('row_id')} audio_dir={record.get('audio_dir')}" ) return supported def _select_new_task_records( self, records: list[dict[str, Any]], ) -> list[dict[str, Any]]: """Cap only new-task duplicate prompts with duration-stratified rows.""" cap = ( self.new_task_max_rows_per_instruction_train if self.split == "train" else self.new_task_max_rows_per_instruction_val ) grouped: dict[tuple[str, str], list[dict[str, Any]]] = {} unsupported = 0 for record in records: if not any( (str(record["audio_dir"]), voice_name) in self.disfluency_instruction_segments for voice_name in self._session_voice_names(record) ): unsupported += 1 continue key = ( str(record["instruction"]).strip(), str(record["instruction_zh"]).strip(), ) grouped.setdefault(key, []).append(record) selected: list[dict[str, Any]] = [] for key in sorted(grouped): values = sorted( grouped[key], key=lambda item: ( int(item.get("actual_frames", 0)), int(item["row_id"]), ), ) if len(values) <= cap: selected.extend(values) continue if cap == 1: selected.append(values[len(values) // 2]) continue indices = [ int(round(position * (len(values) - 1) / float(cap - 1))) for position in range(cap) ] if len(set(indices)) != cap: raise AssertionError( f"Duration-stratified record selection duplicated indices: {key}" ) selected.extend(values[index] for index in indices) selected.sort(key=lambda item: int(item["row_id"])) self.new_task_record_selection_summary = { "input_rows": len(records), "eligible_rows": sum(len(values) for values in grouped.values()), "selected_rows": len(selected), "instruction_pairs": len(grouped), "cap_per_instruction": int(cap), "unsupported_rows": int(unsupported), } if len(selected) < self.session_min_turns: raise RuntimeError( "Too few splittable instruction rows for hesitation/pause: " f"{self.new_task_record_selection_summary}" ) return selected def _voice_for_record(self, record: dict[str, Any]) -> AudioSource: voices = self._record_voices(record) session_voice_name = record.get("_session_voice_name") if session_voice_name is None: return self._select_voice(voices, row_id=int(record["row_id"])) voice_name = str(session_voice_name) indexed_catalog = getattr( self, "_audio_catalog_by_voice_name", None, ) selected = ( indexed_catalog.get(record["audio_dir"], {}).get(voice_name) if indexed_catalog is not None else next( ( voice for voice in voices if self._voice_name(voice) == voice_name ), None, ) ) # Preserve the per-record language filter even though the production # index contains every rendition in the directory. if selected is None or selected not in voices: raise RuntimeError( "Session voice is unavailable or ambiguous for a turn: " f"row={record.get('row_id')} voice={session_voice_name} " f"matches={int(selected is not None and selected in voices)}" ) return selected def _disfluency_listen_window_count( self, session: tuple[dict[str, Any], ...], turn_index: int, voice: AudioSource, *, conservative: bool, ) -> int: event_turn = session[0].get("_session_disfluency_turn_index") if event_turn is None or int(event_turn) != int(turn_index): return int(voice.window_count) subtype = str(session[0].get("_session_disfluency_subtype", "")) voice_name = self._voice_name(voice) final_roles = self.disfluency_instruction_segments.get( (str(session[turn_index]["audio_dir"]), voice_name) ) if final_roles is None: raise RuntimeError("Missing disfluency length-audit instruction segments") stamped_gap = int( session[0].get("_session_disfluency_gap_steps", 0) ) if not 1 <= stamped_gap <= 7: raise RuntimeError( f"Missing/invalid stamped disfluency gap: {stamped_gap}" ) # The exact gap is known at composition time. Segment and complete # command lengths also come from the audited manifest, so conservative # and runtime estimates are intentionally identical. gap_steps = stamped_gap if subtype in {"silent-pause", "filled-pause"}: samples = ( final_roles["prefix"].speech_samples + gap_steps * self.audio_stride_samples + final_roles["suffix"].speech_samples ) elif subtype == "self-repair": provisional_roles = self.disfluency_instruction_segments.get( ( str(session[0].get("_session_provisional_audio_dir", "")), voice_name, ) ) assignment = self.background_assignments.get(voice.source_key) if provisional_roles is None or assignment is None: raise RuntimeError("Incomplete self-repair length audit") samples = ( provisional_roles["prefix"].speech_samples + gap_steps * self.audio_stride_samples + assignment.speech_samples ) else: raise RuntimeError(f"Unknown disfluency subtype: {subtype!r}") return self._runtime_listen_window_count(int(samples)) def _session_max_stream_steps( self, session: tuple[dict[str, Any], ...], ) -> int: """Conservative Qwen-window count for one complete session.""" total_frames = self.initial_idle_max_steps * self.action_exec_horizon interaction_mode = str( session[0].get("_session_interaction_mode", "turn-taking") ) control_turn = session[0].get("_session_control_turn_index") control_turn = None if control_turn is None else int(control_turn) for turn_index, record in enumerate(session): voices = self._record_voices(record) if record.get("_session_voice_name") is not None: selected_voice = self._voice_for_record(record) listen_steps = self._disfluency_listen_window_count( session, turn_index, selected_voice, conservative=True, ) elif self.fixed_voice_per_record: listen_steps = voices[ int(record["row_id"]) % len(voices) ].window_count else: listen_steps = max(voice.window_count for voice in voices) if bool(getattr(self, "causal_listen_timing", False)): # A clean onset cannot occur after the complete speech # waveform. Use the selected waveform's window count as the # conservative bound instead of treating the configured # expected-onset budget as a fatal data constraint. onset_bound_steps = max( int(listen_steps), int(self.max_speech_onset_steps), ) total_frames += ( ( onset_bound_steps + self.listen_reaction_steps ) * self.action_exec_horizon ) entry_frames = ( self.turn_transition_frames if bool(getattr(self, "causal_listen_timing", False)) else self.stationary_blend_frames if turn_index == 0 else self.turn_transition_frames ) total_frames += listen_steps * self.action_exec_horizon total_frames += self._estimated_runtime_motion_frames(record) if not bool(getattr(self, "dynamic_task_entry_blend", False)): total_frames += max(entry_frames - 1, 1) # Every completed task settles into a new upright anchor. Up to # 19 extra frames align the next control transition to Qwen's # 20-frame cadence. total_frames += self.turn_transition_frames - 1 total_frames += self.action_exec_horizon - 1 if turn_index + 1 < len(session): # Stop-persistence v22 deliberately exposes a longer silent # background horizon than ordinary turn-taking gaps. gap_steps = ( self.control_post_idle_max_steps if interaction_mode == "emergency-stop" and turn_index == control_turn and self.control_post_idle_max_steps_explicit else 10 ) total_frames += gap_steps * self.action_exec_horizon else: terminal_steps = ( self.control_post_idle_max_steps if interaction_mode == "emergency-stop" and turn_index == control_turn and self.control_post_idle_max_steps_explicit else self.initial_idle_max_steps ) total_frames += ( terminal_steps * self.action_exec_horizon ) valid_starts = max(total_frames - self.prefix_frames - 1, 0) return int(math.ceil(valid_starts / self.action_exec_horizon)) def _cached_session_max_stream_steps( self, session: tuple[dict[str, Any], ...], ) -> int: """Reuse one conservative session-length calculation across filters.""" cache = getattr(self, "_session_length_cache", None) if cache is None: return self._session_max_stream_steps(session) key = id(session) value = cache.get(key) if value is None: value = int(self._session_max_stream_steps(session)) cache[key] = value return value def _record_max_stream_steps(self, record: dict[str, Any]) -> int: """Worst-case source length used to keep complete Qwen KV in context.""" voices = self._record_voices(record) if int(getattr(self, "session_voice_variants", 0)) > 0: # The record is filtered before static sessions choose a voice. # Guard against every valid rendition so no later variant can # exceed the complete-Qwen-context limit. listen_steps = max(voice.window_count for voice in voices) elif self.fixed_voice_per_record: listen_steps = voices[int(record["row_id"]) % len(voices)].window_count else: listen_steps = max(voice.window_count for voice in voices) task_frames = self._estimated_runtime_motion_frames(record) if not bool(getattr(self, "dynamic_task_entry_blend", False)): task_frames += max(self.stationary_blend_frames - 1, 1) # Match v26's absolute-action index: a sample at state ``t`` targets # frames ``t+1:t+31``. Keep enough starts for the largest RTC delay # (10 clean frames) to retain at least one real supervised target. dense_motion_starts = max( task_frames - self.prefix_frames - 1, 0, ) motion_steps = int( math.ceil(dense_motion_starts / self.action_exec_horizon) ) if self.max_motion_steps > 0: motion_steps = min(motion_steps, self.max_motion_steps) return int(self.initial_idle_max_steps + listen_steps + motion_steps) def _estimated_runtime_motion_frames(self, record: dict[str, Any]) -> int: """Convert 120 Hz metadata lengths to the 50 Hz exported CSV length.""" declared_frames = max(int(record.get("declared_frames", 0)), 1) metadata_fps = max(int(getattr(self, "motion_metadata_fps", 120)), 1) return max( int(round(declared_frames * self.motion_fps / metadata_fps)), 1, ) def _load_background_window( self, source_key: str, window_index: int, ) -> torch.Tensor: source = self.background_catalog[source_key] index = int(window_index) if index < 0 or index >= source.window_count: raise IndexError( f"Background window is outside one continuous session: " f"key={source_key} index={index} count={source.window_count}" ) start = source.offsets[index] end = source.offsets[index + 1] return load_hidden_states_slice( source.packed_path, start, end - start, ).contiguous() def _load_background_window_block( self, source_key: str, start_index: int, count: int, ) -> tuple[torch.Tensor, ...]: """Read adjacent windows with one safetensors slice and one CPFS open.""" source = self.background_catalog[source_key] start_index = int(start_index) count = int(count) if count <= 0: return () end_index = start_index + count if start_index < 0 or end_index > source.window_count: raise IndexError( "Background block is outside one continuous session: " f"key={source_key} start={start_index} count={count} " f"windows={source.window_count}" ) flat_start = source.offsets[start_index] flat_end = source.offsets[end_index] flat = load_hidden_states_slice( source.packed_path, flat_start, flat_end - flat_start, ) return tuple( flat[ source.offsets[index] - flat_start : source.offsets[index + 1] - flat_start ] for index in range(start_index, end_index) ) def _load_waveform_window_block( self, waveform_path: Path, *, start_index: int, count: int, ) -> tuple[np.ndarray, ...]: """Read adjacent 0.6 s windows using one seek/read operation.""" start_index = int(start_index) count = int(count) if start_index < 0 or count < 0: raise ValueError( f"Invalid waveform window block: start={start_index}, count={count}" ) if count == 0: return () flat = _read_pcm16_wave_segment( waveform_path, start_sample=start_index * self.audio_stride_samples, sample_count=( (count - 1) * self.audio_stride_samples + self.audio_window_samples ), expected_rate=self.audio_sampling_rate, ) return tuple( flat[ offset * self.audio_stride_samples : offset * self.audio_stride_samples + self.audio_window_samples ] for offset in range(count) ) def _load_background_waveform_window_block( self, source_key: str, start_index: int, count: int, ) -> tuple[np.ndarray, ...]: source = self.background_catalog[source_key] if not isinstance(source, BackgroundWaveformSource): raise TypeError( "Online HFDv2 expected a BackgroundWaveformSource, got " f"{type(source).__name__}" ) end_index = int(start_index) + int(count) if int(start_index) < 0 or end_index > source.window_count: raise IndexError( "Background waveform block is outside one continuous session: " f"key={source_key} start={start_index} count={count} " f"windows={source.window_count}" ) return self._load_waveform_window_block( source.waveform_path, start_index=int(start_index), count=int(count), ) def _load_mixed_waveform_window_block( self, voice: AudioSource, assignment: SourceBackgroundAssignment, ) -> tuple[np.ndarray, ...]: """Remix the assigned speech/background with the deployment RTC overlap.""" if voice.waveform_path is None or assignment.original_audio_path is None: raise RuntimeError( f"Online voice has no complete waveform assignment: {voice.source_key}" ) background = self.background_catalog[assignment.background_key] if not isinstance(background, BackgroundWaveformSource): raise TypeError( "Stride-1 mixed audio requires a waveform background source" ) count = int(assignment.listen_window_count) flat_count = ( (count - 1) * self.audio_stride_samples + self.audio_window_samples ) flat = _read_pcm16_wave_segment( background.waveform_path, start_sample=( int(assignment.listen_start_index) * self.audio_stride_samples ), sample_count=flat_count, expected_rate=self.audio_sampling_rate, ) speech = _read_resampled_speech_float( assignment.original_audio_path, self.audio_sampling_rate, ) speech_samples = int(speech.size) if speech_samples != int(assignment.speech_samples): raise RuntimeError( "Speech waveform changed after background assignment: " f"source={voice.source_key} actual={speech_samples} " f"assigned={assignment.speech_samples}" ) speech_start = self.audio_window_samples - self.audio_stride_samples speech_end = speech_start + speech_samples if speech_end > flat_count: raise RuntimeError( f"Stride-1 speech reservation is too short: {voice.source_key}" ) flat[speech_start:speech_end] += ( speech * np.float32(assignment.speech_scale) ) if float(np.max(np.abs(flat), initial=0.0)) >= 1.0: raise RuntimeError( f"Stride-1 speech/background mix clips: {voice.source_key}" ) flat = ( np.rint(flat * 32768.0) .clip(-32768, 32767) .astype("<i2") .astype(np.float32) / np.float32(32768.0) ) return tuple( flat[ offset * self.audio_stride_samples : offset * self.audio_stride_samples + self.audio_window_samples ] for offset in range(count) ) def _load_stationary_motion_pools( self, ) -> dict[str, tuple[StationaryMotionSource, ...]]: """Build audited standing-motion pools from exact SEED metadata labels. Keyword matching is intentionally forbidden here: many unrelated SEED descriptions merely say that an action *ends* idle, and several filenames containing ``idle`` are handstands, crouches or injured poses. Exact content labels plus semantic metadata keep those motions out of the training targets. """ columns = { "filename", "move_g1_path", "move_duration_frames", "content_name", "content_short_description", "content_natural_desc_1", "content_natural_desc_4", "content_body_position", "content_props", "is_mirror", } table = pd.read_csv( self.motion_metadata_csv, usecols=lambda name: name in columns, ) pools: dict[str, list[StationaryMotionSource]] = { "idle": [], "listen": [], } for row in table.itertuples(index=False): mirror_value = getattr(row, "is_mirror", False) if not pd.isna(mirror_value) and bool(mirror_value): continue content_name = str(getattr(row, "content_name", "")).strip() if content_name in IDLE_MOTION_CONTENT_NAMES: phase = "idle" elif content_name in LISTEN_MOTION_CONTENT_NAMES: phase = "listen" else: continue body_position = str( getattr(row, "content_body_position", "") ).strip().lower() props = str(getattr(row, "content_props", "0")).strip().lower() descriptions = " ".join( str(getattr(row, key, "")).strip().lower() for key in ( "content_short_description", "content_natural_desc_1", "content_natural_desc_4", ) ) if body_position != "standing" or props not in {"", "0", "0.0", "nan"}: continue if phase == "idle" and not ( "neutral" in descriptions and ("not moving" in descriptions or "standing idle" in descriptions) ): continue if phase == "listen" and not ( "listen" in descriptions and ("ear" in descriptions or "right hand" in descriptions) ): continue motion_dir = _map_seed_motion_dir( self.motion_root, str(getattr(row, "move_g1_path", "")), ) if not all( (motion_dir / filename).is_file() for filename in ("joint_pos.csv", "body_pos.csv", "body_quat.csv") ): continue frame_value = getattr(row, "move_duration_frames", 0) try: declared_frames = ( int(frame_value) if not pd.isna(frame_value) else 0 ) except (TypeError, ValueError): declared_frames = 0 pools[phase].append( StationaryMotionSource( filename=str(getattr(row, "filename", "")).strip(), motion_dir=motion_dir, declared_frames=declared_frames, content_name=content_name, ) ) result = { phase: tuple(sorted(values, key=lambda item: item.filename)) for phase, values in pools.items() } empty = [phase for phase, values in result.items() if not values] if empty: raise RuntimeError( "No audited natural standing motions remain for phases " f"{empty}; metadata={self.motion_metadata_csv}" ) return result def _load_records(self) -> list[dict[str, Any]]: motion_table = pd.read_csv( self.motion_metadata_csv, usecols=lambda name: name in { "filename", "move_g1_path", "move_duration_frames", "content_body_position", "content_name", "content_short_description", "content_natural_desc_1", "content_natural_desc_4", }, ) motion_index: dict[str, tuple[Path, int, bool, str]] = {} unsafe_end_markers = ( "handstand", "fall", "lying", "lie down", "crawl", "kneel", "sitting", "sit down", "crouch", "somersault", "backflip", "frontflip", "injured", ) for row in motion_table.itertuples(index=False): filename = str(getattr(row, "filename")).strip() move_path = str(getattr(row, "move_g1_path")).strip() if not filename or not move_path: continue duration_value = getattr(row, "move_duration_frames", 0) try: frames = int(duration_value) if not pd.isna(duration_value) else 0 except (TypeError, ValueError): frames = 0 body_position = str( getattr(row, "content_body_position", "") ).strip().lower() descriptions = " ".join( str(getattr(row, key, "")).strip().lower() for key in ( "content_name", "content_short_description", "content_natural_desc_1", "content_natural_desc_4", ) ) stable_turn_end = ( body_position == "standing" and not any( marker in descriptions for marker in unsafe_end_markers ) ) motion_index[filename] = ( _map_seed_motion_dir(self.motion_root, move_path), frames, stable_turn_end, str( getattr(row, "content_short_description", "") ).strip(), ) manifest_rows: list[dict[str, Any]] = [] audio_directory_texts: dict[str, tuple[str, str]] = {} unsafe_turn_rows_filtered = 0 known_translation_mismatch_rows = 0 known_translation_mismatch_audio_dirs: set[str] = set() with self.metadata_jsonl.open("r", encoding="utf-8") as handle: for row_id, line in enumerate(handle): line = line.strip() if not line: continue row = json.loads(line) filename = str(row.get("filename", "")).strip() audio_dir = _normalise_audio_dir(str(row.get("audio_path", ""))) instruction = str( row.get("content_short_description", "") ).strip() instruction_zh = str( row.get("content_short_description_zh", "") ).strip() if not instruction or not instruction_zh: raise RuntimeError( "Audio-command manifest needs exact English and Chinese " f"TTS text: filename={filename} audio_dir={audio_dir}" ) text_pair = (instruction, instruction_zh) previous_pair = audio_directory_texts.setdefault( audio_dir, text_pair, ) if previous_pair != text_pair: raise RuntimeError( "One TTS audio directory maps to conflicting text: " f"audio_dir={audio_dir} first={previous_pair!r} " f"current={text_pair!r}" ) motion_info = motion_index.get(filename) if motion_info is None or audio_dir not in self.audio_catalog: continue if instruction != motion_info[3]: raise RuntimeError( "English TTS source text differs from the paired motion " "metadata after whitespace normalization: " f"filename={filename} tts={instruction!r} " f"motion={motion_info[3]!r}" ) # The original motion label is "retro" (复古), but one # generated Chinese TTS batch mistranslated it as spatial # "后退" (backward). Keep the valid English renditions and # exclude only the contradictory Chinese waveforms. This is # intentionally a narrow, auditable rule rather than a fuzzy # semantic-cleaning heuristic. exclude_zh_voices = bool( getattr( self, "filter_known_tts_translation_mismatches", False, ) and has_known_tts_translation_mismatch( instruction, instruction_zh, ) ) if exclude_zh_voices: known_translation_mismatch_rows += 1 known_translation_mismatch_audio_dirs.add(audio_dir) if self.multi_turn_sessions and not motion_info[2]: unsafe_turn_rows_filtered += 1 continue manifest_rows.append( { "row_id": row_id, "filename": filename, "instruction": instruction, "instruction_zh": instruction_zh, "audio_dir": audio_dir, "_exclude_zh_voices": exclude_zh_voices, "motion_dir": motion_info[0], "declared_frames": motion_info[1], } ) self.unsafe_turn_rows_filtered = unsafe_turn_rows_filtered self.known_translation_mismatch_rows = ( known_translation_mismatch_rows ) self.known_translation_mismatch_audio_dirs = len( known_translation_mismatch_audio_dirs ) self.known_translation_voice_variants_excluded = sum( sum( self._voice_name(voice).endswith("_zh") for voice in self.audio_catalog.get(audio_dir, ()) ) for audio_dir in known_translation_mismatch_audio_dirs ) self.canonical_motion_summary = { "enabled": False, "before": len(manifest_rows), "after": len(manifest_rows), "removed": 0, } if self.canonical_motion_manifest is not None: selection, ambiguous_chinese = _load_canonical_motion_manifest( self.canonical_motion_manifest ) candidates: dict[tuple[str, str], set[str]] = {} for row in manifest_rows: key = (str(row["instruction"]), str(row["instruction_zh"])) candidates.setdefault(key, set()).add(str(row["filename"])) missing_keys = sorted(set(candidates).difference(selection)) invalid = { key: selection[key] for key in candidates.keys() & selection.keys() if selection[key] not in candidates[key] } unused_keys = sorted(set(selection).difference(candidates)) if missing_keys or invalid or unused_keys: raise RuntimeError( "Canonical-motion manifest does not exactly match the " "current eligible corpus: " f"missing={missing_keys[:3]} invalid=" f"{list(invalid.items())[:3]} unused={unused_keys[:3]}" ) before = len(manifest_rows) manifest_rows = [ row for row in manifest_rows if selection[ (str(row["instruction"]), str(row["instruction_zh"])) ] == str(row["filename"]) ] for row in manifest_rows: if str(row["instruction_zh"]) in ambiguous_chinese: row["_exclude_zh_voices"] = True if len(manifest_rows) != len(selection): raise RuntimeError( "Canonical-motion filtering must leave exactly one row " "per exact instruction pair: " f"rows={len(manifest_rows)} selections={len(selection)}" ) self.canonical_motion_summary = { "enabled": True, "before": before, "after": len(manifest_rows), "removed": before - len(manifest_rows), "ambiguous_chinese_phrases_excluded": len( ambiguous_chinese ), "manifest": str(self.canonical_motion_manifest), } source_ids = [ motion_source_id(self.split_source_namespace, row["filename"]) for row in manifest_rows ] selected = select_source_row_ids( source_ids, split=self.split, val_ratio=self.val_ratio, split_seed=self.split_seed, ) return [manifest_rows[int(index)] for index in selected] def _load_motion(self, path: Path) -> np.ndarray: key = str(path) # Task motions are unique in this corpus, so caching them only evicts # the small idle/listen set that every source reuses. Reserve the LRU # for those audited stationary motions. cacheable = key in getattr(self, "_stationary_motion_paths", ()) if cacheable: cached = self._motion_cache.get(key) if cached is not None: self._motion_cache.move_to_end(key) return cached joint = pd.read_csv(path / "joint_pos.csv").to_numpy(dtype=np.float32) body_pos = pd.read_csv(path / "body_pos.csv").to_numpy(dtype=np.float32) body_quat = pd.read_csv(path / "body_quat.csv").to_numpy(dtype=np.float32) lengths = {joint.shape[0], body_pos.shape[0], body_quat.shape[0]} if len(lengths) != 1 or joint.shape[1] != 29 or body_pos.shape[1] != 3 or body_quat.shape[1] != 4: raise ValueError(f"Invalid 36D SEED motion at {path}") motion = np.concatenate([joint, body_pos, body_quat], axis=1).astype(np.float32) if cacheable and self.motion_cache_size > 0: self._motion_cache[key] = motion self._motion_cache.move_to_end(key) while len(self._motion_cache) > self.motion_cache_size: self._motion_cache.popitem(last=False) return motion @staticmethod def _load_slice(item: HiddenSlice) -> torch.Tensor: if not item.shard_path.is_file(): raise FileNotFoundError(f"Missing hidden-state shard: {item.shard_path}") return load_hidden_states_slice( item.shard_path, item.offset, item.length ).contiguous() def _load_voice_hidden( self, voice: AudioSource, ) -> tuple[torch.Tensor, ...]: if voice.waveform_path is not None: raise RuntimeError( "HFDv2 online waveform sources must not read cached hidden states" ) if voice.packed_path is None: return tuple(self._load_slice(clip) for clip in voice.clips) frame_ids, hidden_windows = load_hidden_states_source( voice.packed_path, expected_window_seconds=self.audio_window_seconds, expected_step_frames=self.audio_sample_step_frames, expected_motion_fps=self.motion_fps, expected_sampling_rate=self.audio_sampling_rate, ) expected_frame_ids = tuple( index * self.audio_sample_step_frames for index in range(voice.window_count) ) if frame_ids != expected_frame_ids: raise RuntimeError( f"HFD source/index frame mismatch for {voice.source_key}: " f"file={frame_ids} index={expected_frame_ids}" ) return hidden_windows def _raw_step( self, *, hidden_states: torch.Tensor | None = None, audio_waveform: np.ndarray | None = None, actions: np.ndarray, action_mask: np.ndarray, state: np.ndarray, action_reference_state: np.ndarray, smoothness_context_actions: np.ndarray, smoothness_context_reference_state: np.ndarray, action_control_labels: np.ndarray, future_progress: np.ndarray, control_progress: np.ndarray, previous_control_label: int, phase: str, source_id: str, step_index: int, frame_id: int, semantic_window_index: int, action_substep: int, semantic_frame_id: int, record: dict[str, Any], instruction_text: str, update_semantic_cache: bool, control_transition: bool, instruction_history_texts: tuple[str, ...] = (), ) -> dict[str, Any]: semantic_inputs = int(hidden_states is not None) + int( audio_waveform is not None ) if update_semantic_cache and semantic_inputs != 1: raise ValueError( "A Qwen-update step must contain exactly one semantic input" ) if not update_semantic_cache and semantic_inputs != 0: raise ValueError( "A dense action-only step must reuse the previous Qwen " "condition and contain no new semantic input" ) progress_start = ( 0.0 if phase in {"idle", "listen"} else float(min(max(frame_id, 0), int(record["actual_frames"]) - 1)) / float(max(int(record["actual_frames"]) - 1, 1)) ) if phase == "idle": control_label = IDLE_TOKEN_ID elif phase == "listen": control_label = LISTEN_TOKEN_ID elif phase == "motion": control_label = MOTION_TOKEN_ID else: raise ValueError(f"Unsupported full-duplex phase: {phase!r}") latest_instruction_target = bool( update_semantic_cache and control_transition and control_label == MOTION_TOKEN_ID ) exact_instruction_text = str(instruction_text).strip() if latest_instruction_target and not exact_instruction_text: raise RuntimeError( "A listen-to-motion boundary has no latest-instruction text " f"target: source={source_id} step={step_index}" ) exact_instruction_history = tuple( str(value).strip() for value in instruction_history_texts if str(value).strip() ) memory_target = bool( latest_instruction_target and exact_instruction_history ) if memory_target: if len(exact_instruction_history) == 1: memory_slot = "previous" memory_text = exact_instruction_history[-1] elif len(exact_instruction_history) % 2 == 0: # Third command: ask for the first turn. Fourth command: ask # for the immediately previous turn. Across 2--4 turn # sessions this balances long-range and local retrieval while # adding only one extra sparse side readout per boundary. memory_slot = "first" memory_text = exact_instruction_history[0] else: memory_slot = "previous" memory_text = exact_instruction_history[-1] else: memory_slot = "" memory_text = "" output = { "actions": actions.astype(np.float32), "action_mask": action_mask.astype(np.float32), "states": state.reshape(1, -1).astype(np.float32), "action_reference_states": action_reference_state.reshape( 1, -1 ).astype(np.float32), # The production temporal loss needs the two real action tokens # immediately before this horizon. Keeping them source-local is # what lets velocity/acceleration span a dense-window boundary # instead of restarting at every 30-frame target. "smoothness_context_actions": np.asarray( smoothness_context_actions, dtype=np.float32, ).reshape(2, -1), "smoothness_context_reference_states": np.asarray( smoothness_context_reference_state, dtype=np.float32, ).reshape(1, -1), # The 30-frame action horizon can cross idle/listen/motion # boundaries. Keep the exact phase of every future frame instead # of pretending that the scalar current-window token owns all 30. "action_control_labels": np.asarray( action_control_labels, dtype=np.int64, ).reshape(self.action_chunk_size), "progress": np.asarray( [ progress_start, progress_start, 1.0 - progress_start, math.sin(2.0 * math.pi * progress_start), math.cos(2.0 * math.pi * progress_start), ], dtype=np.float32, ), "future_progress": future_progress.astype(np.float32), # Causal context for the serial token. This describes the state # already executed immediately before the new 0.4 s decision; it # never reads the target phase or future GT progress. "control_progress": np.asarray( control_progress, dtype=np.float32, ).reshape(5), "previous_control_label": int(previous_control_label), "instruction": record["instruction"], "dataset_name": "hfd-audio-command-full-duplex", "source_id": source_id, "source_audio_key": source_id.split("::", 1)[-1], "phase": phase, "control_label": control_label, "control_token": CONTROL_TOKEN_NAMES[control_label], "control_transition": bool(control_transition), "update_semantic_cache": bool(update_semantic_cache), "step_index": int(step_index), "frame_id": int(frame_id), "semantic_frame_id": int(semantic_frame_id), "semantic_window_index": int(semantic_window_index), "action_substep": int(action_substep), "motion_name": record["filename"], "motion_dir": str(record["motion_dir"]), "total_frames": int(record["actual_frames"]), # One target per turn, exposed only at the causal Qwen boundary # where the current scalar control changes into motion. The raw # primitive duration excludes the known stationary join. "motion_duration_target_seconds": float(record["actual_frames"]) / float(getattr(self, "motion_fps", 50)), "motion_duration_target_mask": bool( latest_instruction_target ), # Version 18 applies teacher-forced text CE exactly once per turn, # at the same causal boundary as duration supervision. No VAD or # inference-time text decoder is introduced. "latest_instruction_text": ( exact_instruction_text if latest_instruction_target else "" ), "latest_instruction_target_mask": latest_instruction_target, "instruction_memory_text": memory_text, "instruction_memory_slot": memory_slot, "instruction_memory_target_mask": memory_target, } if hidden_states is not None: output["vlm_hidden_states"] = hidden_states elif audio_waveform is not None: waveform = np.asarray(audio_waveform, dtype=np.float32).reshape(-1) if waveform.shape != (self.audio_window_samples,): raise ValueError( "Online HFDv2 audio window must be exactly " f"{self.audio_window_samples} samples, got {waveform.shape}" ) output["audio_waveform"] = waveform output["audio_sampling_rate"] = int(self.audio_sampling_rate) return output def __len__(self) -> int: if bool(getattr(self, "multi_turn_sessions", False)): return len(self.sessions) return len(self.records) def estimated_stream_steps(self, index: int) -> int: """Estimate dense action optimizer steps for cross-rank load balance.""" lengths = getattr(self, "_estimated_stream_lengths", None) if lengths is not None: return lengths[int(index)] return self._estimate_stream_steps(index) def _estimate_stream_steps(self, index: int) -> int: if bool(getattr(self, "multi_turn_sessions", False)): session = self.sessions[int(index)] interaction_mode = str( session[0].get("_session_interaction_mode", "turn-taking") ) control_turn = session[0].get("_session_control_turn_index") control_turn = None if control_turn is None else int(control_turn) total_frames = ( int(round(self.initial_idle_max_steps * 0.5)) * self.action_exec_horizon ) for turn_index, record in enumerate(session): voices = self._record_voices(record) if record.get("_session_voice_name") is not None: selected_voice = self._voice_for_record(record) listen_steps = self._disfluency_listen_window_count( session, turn_index, selected_voice, conservative=False, ) elif self.fixed_voice_per_record: listen_steps = voices[ int(record["row_id"]) % len(voices) ].window_count else: listen_steps = int( round( sum(voice.window_count for voice in voices) / len(voices) ) ) if bool(getattr(self, "causal_listen_timing", False)): # Estimate from the selected waveform duration. The # context filter above still uses the full conservative # duration, so a long valid leading silence cannot escape # initialization and crash a worker hours later. onset_bound_steps = max( int(listen_steps), int(self.max_speech_onset_steps), ) total_frames += max( int(round(onset_bound_steps * 0.5)), 1, ) * self.action_exec_horizon total_frames += ( self.listen_reaction_steps * self.action_exec_horizon ) entry_frames = ( self.turn_transition_frames if bool(getattr(self, "causal_listen_timing", False)) else self.stationary_blend_frames if turn_index == 0 else self.turn_transition_frames ) total_frames += listen_steps * self.action_exec_horizon task_frames = self._estimated_runtime_motion_frames(record) if not bool( getattr(self, "dynamic_task_entry_blend", False) ): task_frames += max(entry_frames - 1, 1) if ( interaction_mode == "interruption" and turn_index + 1 < len(session) ): # Match the runtime truncation closely enough for the # length-grouped sampler. The conservative context filter # remains `_session_max_stream_steps`; this branch only # prevents interrupted sources from being scheduled as if # every discarded tail were still executed. minimum = int( math.ceil( self.interruption_min_start_seconds * self.motion_fps / self.action_exec_horizon ) ) * self.action_exec_horizon requested_maximum = int( math.floor( self.interruption_max_start_seconds * self.motion_fps / self.action_exec_horizon ) ) * self.action_exec_horizon next_record = session[turn_index + 1] next_listen_steps = self._voice_for_record( next_record ).window_count onset_estimate = max( int(round(max(next_listen_steps, self.max_speech_onset_steps) * 0.5)), 1, ) next_reaction_frames = ( onset_estimate + self.listen_reaction_steps ) * self.action_exec_horizon remaining = max( int( math.ceil( self.interruption_min_remaining_seconds * self.motion_fps ) ), next_reaction_frames, ) latest = ( (task_frames - remaining) // self.action_exec_horizon ) * self.action_exec_horizon maximum = min(requested_maximum, latest) estimated_cut = ( minimum if maximum <= minimum else ( (minimum + maximum) // (2 * self.action_exec_horizon) ) * self.action_exec_horizon ) total_frames += estimated_cut continue total_frames += task_frames recovery_frames = self.turn_transition_frames recovery_frames += ( -(total_frames + recovery_frames - 1) ) % self.action_exec_horizon total_frames += recovery_frames - 1 if ( interaction_mode == "emergency-stop" and turn_index == control_turn and self.control_post_idle_max_steps_explicit ): stop_idle_mean = int( round( ( self.control_post_idle_min_steps + self.control_post_idle_max_steps ) * 0.5 ) ) total_frames += ( stop_idle_mean * self.action_exec_horizon ) continue if turn_index + 1 < len(session): # Mean of 50% immediate, 35% 1..3 and 15% 4..10. total_frames += 2 * self.action_exec_horizon else: terminal_mean = max( int(round((self.initial_idle_max_steps + 1) * 0.5)), 1, ) total_frames += terminal_mean * self.action_exec_horizon semantic_estimate = int( math.ceil( max(total_frames - self.prefix_frames - 1, 0) / self.action_exec_horizon ) ) if ( self.max_stream_steps > 0 and semantic_estimate > self.max_stream_steps ): raise RuntimeError( "Context-filtered session still estimates " f"{semantic_estimate} semantic windows above " f"limit={self.max_stream_steps}: index={index}" ) return max(total_frames - self.prefix_frames - 1, 0) record = self.records[int(index)] voices = self._record_voices(record) if self.fixed_voice_per_record: listen_steps = voices[ int(record["row_id"]) % len(voices) ].window_count else: listen_steps = int( round( sum(voice.window_count for voice in voices) / len(voices) ) ) idle_steps = int(round(self.initial_idle_max_steps * 0.5)) task_frames = self._estimated_runtime_motion_frames(record) if not bool(getattr(self, "dynamic_task_entry_blend", False)): task_frames += max(self.stationary_blend_frames - 1, 1) # ``estimated_stream_steps`` returns dense action starts, not the # number of semantic/Qwen windows. The final ``prefix+1`` states are # excluded exactly as in v26's valid_sample_count(): after a maximal # 10-frame RTC prefix there must still be one real target frame. motion_action_steps = max( task_frames - self.prefix_frames - 1, 0, ) if self.max_motion_steps > 0: motion_action_steps = min( motion_action_steps, self.max_motion_steps * self.action_exec_horizon, ) motion_semantic_steps = int( math.ceil(motion_action_steps / self.action_exec_horizon) ) semantic_estimate = idle_steps + listen_steps + motion_semantic_steps if self.max_stream_steps > 0 and semantic_estimate > self.max_stream_steps: raise RuntimeError( "Context-filtered record still estimates " f"{semantic_estimate} semantic windows " f"above limit={self.max_stream_steps}: index={index}" ) return ( (idle_steps + listen_steps) * self.action_exec_horizon + motion_action_steps ) def _select_voice( self, voices: tuple[AudioSource, ...], *, row_id: int, ) -> AudioSource: if not voices: raise RuntimeError("An audio-command record has no valid TTS voice") if self.fixed_voice_per_record: return voices[int(row_id) % len(voices)] return random.choice(voices) if len(voices) > 1 else voices[0] @staticmethod def _stationary_motion_is_safe(motion: np.ndarray) -> bool: """Reject malformed or locomoting candidates even after metadata audit.""" if ( motion.ndim != 2 or motion.shape[1] != 36 or motion.shape[0] < 30 or not np.isfinite(motion).all() ): return False root = motion[:, 29:32] horizontal_range = float( np.linalg.norm(root[:, :2] - root[:1, :2], axis=-1).max() ) height_range = float(np.ptp(root[:, 2])) return horizontal_range <= 0.15 and height_range <= 0.08 @classmethod def _idle_motion_is_quiet_and_continuous(cls, motion: np.ndarray) -> bool: """Reject jump-cut or gesture-like clips from the neutral idle pool. Listen clips are intentionally expressive, so this stricter gate is applied only to idle. At 50 Hz a neutral standing residual should not jump by more than 0.08 rad in one frame, spend its upper tail above 0.03 rad/frame, or sweep a joint through more than 0.45 rad. These limits retain subtle breathing/postural variation while rejecting the audited A120 discontinuities and large unrelated arm gestures. """ if not cls._stationary_motion_is_safe(motion): return False joint_step = np.max(np.abs(np.diff(motion[:, :29], axis=0)), axis=1) joint_range = np.ptp(motion[:, :29], axis=0) return ( float(np.max(joint_step, initial=0.0)) <= 0.08 and float(np.percentile(joint_step, 95)) <= 0.03 and float(np.max(joint_range, initial=0.0)) <= 0.45 ) def _initialise_safe_stationary_anchors(self) -> None: """Audit neutral clips and cache quiet, upright two-support poses. Metadata semantics alone cannot catch a motion that is perfectly static but contains a corrupted whole-body pose. We therefore compare every idle clip's median 29-D joint pose against the complete pool with a median/MAD robust score. This is deliberately data-driven: no filename, action class or manually maintained joint-angle blacklist is used. Accepted clips then contribute only upright, low-velocity frames whose two leg chains satisfy the same contact proxy used by TextOp. """ loaded: list[tuple[StationaryMotionSource, np.ndarray]] = [] representatives: list[np.ndarray] = [] continuity_rejected_names: list[str] = [] for source in self.stationary_motion_pools["idle"]: motion = self._load_motion(source.motion_dir) if not self._stationary_motion_is_safe(motion): continue if bool( getattr(self, "stationary_idle_residual_retargeting", False) ) and not self._idle_motion_is_quiet_and_continuous(motion): continuity_rejected_names.append(source.filename) continue loaded.append((source, motion)) representatives.append(np.median(motion[:, :29], axis=0)) if not loaded: raise RuntimeError("No kinematically valid neutral idle clips remain") representative_rows = np.stack(representatives).astype(np.float32) center = np.median(representative_rows, axis=0) mad = np.median(np.abs(representative_rows - center), axis=0) scale = np.maximum(1.4826 * mad, 0.05).astype(np.float32) clip_scores = np.sqrt( np.mean(np.square((representative_rows - center) / scale), axis=1) ) score_center = float(np.median(clip_scores)) score_mad = float(np.median(np.abs(clip_scores - score_center))) # Six robust standard deviations is intentionally conservative. It # rejects isolated structural corruption while retaining legitimate # variations in neutral arm placement. clip_score_limit = score_center + max( 6.0 * 1.4826 * score_mad, 1.0, ) accepted_paths: set[str] = set() rejected_names: list[str] = [] anchor_rows: list[np.ndarray] = [] sampling_stride = max(int(self.motion_fps) // 5, 1) for (source, motion), clip_score in zip(loaded, clip_scores): if float(clip_score) > clip_score_limit: rejected_names.append(source.filename) continue joint_delta = np.diff( motion[:, :29], axis=0, prepend=motion[:1, :29] ) root_delta = np.diff( motion[:, 29:32], axis=0, prepend=motion[:1, 29:32] ) contacts = estimate_two_foot_contacts(joint_delta, root_delta) quat = _normalise_motion_quaternions(motion)[:, 32:36] upright = 1.0 - 2.0 * ( np.square(quat[:, 1]) + np.square(quat[:, 2]) ) frame_scores = np.sqrt( np.mean(np.square((motion[:, :29] - center) / scale), axis=1) ) low_velocity = ( np.linalg.norm(joint_delta, axis=-1) <= 0.05 ) & ( np.linalg.norm(root_delta, axis=-1) <= 0.025 ) valid = ( np.all(contacts > 0.5, axis=-1) & (upright >= 0.80) & low_velocity & (frame_scores <= clip_score_limit) ) valid[0] = False # prepend-diff would make frame zero trivially static. indices = np.flatnonzero(valid) if indices.size == 0: rejected_names.append(source.filename) continue accepted_paths.add(str(source.motion_dir)) selected = indices[::sampling_stride] if selected[-1] != indices[-1]: selected = np.concatenate([selected, indices[-1:]]) anchor_rows.extend(motion[selected].astype(np.float32, copy=True)) if not anchor_rows: raise RuntimeError("Whole-body idle audit produced no safe anchor frames") self._safe_idle_motion_paths = frozenset(accepted_paths) self._stationary_anchor_poses = _normalise_motion_quaternions( np.stack(anchor_rows) ) self._stationary_joint_center = center.astype(np.float32) self._stationary_joint_scale = scale self.stationary_audit_summary = { "enabled": True, "accepted_clips": len(accepted_paths), "rejected_clips": len(rejected_names), "rejected_names": tuple(sorted(rejected_names)), "continuity_rejected_clips": len(continuity_rejected_names), "continuity_rejected_names": tuple( sorted(continuity_rejected_names) ), "anchor_frames": int(len(self._stationary_anchor_poses)), "clip_score_limit": float(clip_score_limit), "max_accepted_clip_score": float( max( score for (_, _), score in zip(loaded, clip_scores) if float(score) <= clip_score_limit ) ), } def _nearest_safe_stationary_anchor( self, current_pose: np.ndarray, ) -> tuple[np.ndarray, float]: """Return the audited standing pose closest in normalized joint space.""" pose = np.asarray(current_pose, dtype=np.float32).reshape(36) anchors = np.asarray(self._stationary_anchor_poses, dtype=np.float32) if anchors.ndim != 2 or anchors.shape[0] == 0: raise RuntimeError("Safe stationary anchor catalog is empty") scale = np.asarray(self._stationary_joint_scale, dtype=np.float32) distance = np.mean( np.square((anchors[:, :29] - pose[:29]) / scale[None, :]), axis=1, ) # Root height is physically coupled to the selected leg configuration; # include it in nearest-neighbour selection, but never overwrite it. height_scale = max(float(np.std(anchors[:, 31])), 0.05) distance += 0.25 * np.square((anchors[:, 31] - pose[31]) / height_scale) selected = int(np.argmin(distance)) anchor = anchors[selected].astype(np.float32, copy=True) anchor[29:31] = pose[29:31] return anchor, float(distance[selected]) @staticmethod def _crop_or_resample_motion( motion: np.ndarray, frame_count: int, ) -> np.ndarray: if frame_count <= 0: return np.empty((0, 36), dtype=np.float32) if motion.shape[0] >= frame_count: max_start = int(motion.shape[0] - frame_count) start = random.randint(0, max_start) if max_start > 0 else 0 return np.asarray( motion[start : start + frame_count], dtype=np.float32, ).copy() source_x = np.linspace(0.0, 1.0, int(motion.shape[0]), dtype=np.float32) target_x = np.linspace(0.0, 1.0, int(frame_count), dtype=np.float32) values = np.stack( [np.interp(target_x, source_x, motion[:, dim]) for dim in range(36)], axis=-1, ).astype(np.float32) return _normalise_motion_quaternions(values) @staticmethod def _listen_hold_target(motion: np.ndarray, anchor: np.ndarray) -> np.ndarray: """Pick an actual, settled expressive pose from an audited listen clip. Use both arms, not a filename or a hard-coded joint angle. Retain a recorded whole-body frame (including its matching height/legs), never assemble a hand pose with unrelated root height. The upper excursion quartile excludes the clip's neutral entry/exit; local velocity picks a settled part rather than the fastest raising/lowering frame. """ arm_distance = np.linalg.norm(motion[:, 15:29] - anchor[None, 15:29], axis=1) speed = np.linalg.norm(np.gradient(motion[:, :29], axis=0), axis=1) radius = min(5, (len(motion) - 1) // 2) smooth_speed = np.convolve( np.pad(speed, (radius, radius), mode="edge"), np.ones(2 * radius + 1) / (2 * radius + 1), mode="valid", ) candidates = np.flatnonzero(arm_distance >= np.quantile(arm_distance, 0.75)) selected = candidates[np.argmin(smooth_speed[candidates])] return motion[int(selected)].astype(np.float32, copy=True) def _sample_stationary_sequence( self, phase: str, *, frame_count: int, anchor_pose: np.ndarray, blend_frames: int | None = None, entry_pose: np.ndarray | None = None, ) -> tuple[np.ndarray, str | None]: if frame_count <= 0: return np.empty((0, 36), dtype=np.float32), None pool = self.stationary_motion_pools[str(phase)] candidates = list(pool) if ( str(phase) == "idle" and bool(getattr(self, "safe_stationary_anchors", False)) ): candidates = [ source for source in candidates if str(source.motion_dir) in self._safe_idle_motion_paths ] random.shuffle(candidates) selected: np.ndarray | None = None selected_name = "" hold_listen = str(phase) == "listen" and bool( getattr(self, "stationary_listen_hold", False) ) for candidate in candidates: motion = self._load_motion(candidate.motion_dir) if self._stationary_motion_is_safe(motion): selected = self._crop_or_resample_motion(motion, frame_count) if hold_listen: # Keep the legacy crop RNG draw: enabling this target fix # must not reshuffle later voices, backgrounds or turns. pose = self._listen_hold_target(motion, np.asarray(anchor_pose)) selected = np.repeat(pose[None, :], frame_count, axis=0) selected_name = candidate.filename break if selected is None: raise RuntimeError( f"All audited {phase} motions failed the kinematic safety gate" ) anchor = np.asarray(anchor_pose, dtype=np.float32).reshape(36) if bool(getattr(self, "stationary_heading_alignment", False)): # Express every stationary candidate in the current robot heading. # The old path aligned XY only, so a perfectly valid listen clip # recorded facing another world direction taught an arbitrary turn # (up to ~180 degrees in real validation sessions). This rigid yaw # transform preserves local motion and the candidate's own height. selected = _align_task_motion_to_anchor_heading(selected, anchor) else: # Historical v14 data contract: align only ground-plane position. selected[:, 29:31] += anchor[29:31] - selected[0, 29:31] if ( str(phase) == "idle" and bool( getattr( self, "stationary_idle_residual_retargeting", False, ) ) ): # An idle clip is a source of tiny *changes*, not a destination # pose. Blending an arbitrary recorded standing pose against the # current anchor can otherwise manufacture a large arm excursion: # both endpoints are individually static, but their wrists may be # far apart. Retarget joint/root residuals around the current # anchor and hold its heading. This preserves breathing and small # posture changes while removing the unrelated absolute pose. selected_first = selected[0].copy() selected[:, :29] = ( anchor[None, :29] + selected[:, :29] - selected_first[None, :29] ) selected[:, 29:32] = ( anchor[None, 29:32] + selected[:, 29:32] - selected_first[None, 29:32] ) selected[:, 32:36] = anchor[None, 32:36] # Legacy: both ends return exactly to the target motion's first pose. The # smooth envelope lets idle/listen move naturally in the middle while # guaranteeing continuous idle->listen and listen->task boundaries. if frame_count == 1: motion_weight = np.zeros(1, dtype=np.float32) else: ramp = float( max( ( self.stationary_blend_frames if blend_frames is None else int(blend_frames) ) - 1, 1, ) ) offsets = np.arange(frame_count, dtype=np.float32) motion_weight = np.minimum( np.minimum(offsets / ramp, offsets[::-1] / ramp), 1.0, ) if hold_listen: motion_weight = np.minimum(offsets / ramp, 1.0) motion_weight = np.clip(motion_weight, 0.0, 1.0) if hold_listen: motion_weight = motion_weight**3 * ( motion_weight * (motion_weight * 6.0 - 15.0) + 10.0 ) else: motion_weight = motion_weight * motion_weight * ( 3.0 - 2.0 * motion_weight ) start_pose = entry_pose if hold_listen and entry_pose is not None else anchor result = _blend_pose_rows(start_pose, selected, motion_weight) if entry_pose is not None and not hold_listen: entry = np.asarray(entry_pose, dtype=np.float32).reshape(36) entry_count = min( int( self.turn_transition_frames if blend_frames is None else blend_frames ), frame_count, ) entry_weight = np.ones(frame_count, dtype=np.float32) if entry_count > 1: phase_weight = np.linspace( 0.0, 1.0, entry_count, dtype=np.float32 ) phase_weight = phase_weight * phase_weight * phase_weight * ( phase_weight * (phase_weight * 6.0 - 15.0) + 10.0 ) entry_weight[:entry_count] = phase_weight else: entry_weight[0] = 1.0 result = _blend_pose_rows(entry, result, entry_weight) if not np.isfinite(result).all(): raise RuntimeError( f"Non-finite blended {phase} motion from {selected_name}" ) return result, selected_name def _sample_stationary_anchor(self, task_start_pose: np.ndarray) -> np.ndarray: """Choose a verified upright neutral pose at the task's root position.""" if bool(getattr(self, "safe_stationary_anchors", False)): anchor, _ = self._nearest_safe_stationary_anchor(task_start_pose) return _normalise_motion_quaternions(anchor[None, :])[0] candidates = list(self.stationary_motion_pools["idle"]) random.shuffle(candidates) for candidate in candidates: motion = self._load_motion(candidate.motion_dir) if not self._stationary_motion_is_safe(motion): continue frame = random.randint(0, int(motion.shape[0]) - 1) anchor = motion[frame].astype(np.float32, copy=True) # Align the neutral pose to the command's ground-plane position, # but keep its own physically matched root height. The existing # smooth transition handles the later stationary-to-task Z change. anchor[29:31] = np.asarray(task_start_pose, dtype=np.float32)[29:31] return _normalise_motion_quaternions(anchor[None, :])[0] raise RuntimeError("No safe neutral idle pose is available as RTC anchor") def _build_task_sequence( self, motion: np.ndarray, *, stationary_anchor: np.ndarray, transition_frames: int | None = None, ) -> tuple[np.ndarray, int]: """Align task heading, then transition smoothly out of standing.""" aligned_motion = _align_task_motion_to_anchor_heading( motion, stationary_anchor, ) transition_frames = max( int( self.stationary_blend_frames if transition_frames is None else transition_frames ), 2, ) if bool(getattr(self, "dynamic_task_entry_blend", False)): blend_count = min(transition_frames, len(aligned_motion)) phase = np.linspace(0.0, 1.0, blend_count, dtype=np.float32) phase = phase * phase * phase * ( phase * (phase * 6.0 - 15.0) + 10.0 ) task_sequence = aligned_motion.astype(np.float32, copy=True) task_sequence[:blend_count] = _blend_pose_rows( stationary_anchor, aligned_motion[:blend_count], phase, ) return task_sequence, blend_count target = np.repeat( aligned_motion[0:1], transition_frames, axis=0, ) phase = np.linspace(0.0, 1.0, transition_frames, dtype=np.float32) if bool(getattr(self, "multi_turn_sessions", False)): # Multi-turn joins are deliberately more relaxed: the quintic # curve reaches both poses with zero velocity and acceleration. # Historical single-turn versions retain their exact cubic data. phase = phase * phase * phase * ( phase * (phase * 6.0 - 15.0) + 10.0 ) else: phase = phase * phase * (3.0 - 2.0 * phase) transition = _blend_pose_rows(stationary_anchor, target, phase) # Avoid duplicating the target's first frame after the transition. task_sequence = np.concatenate( [transition, aligned_motion[1:]], axis=0, ) return task_sequence.astype(np.float32, copy=False), transition_frames def _task_original_frame_indices( self, *, sequence_frames: int, motion_frames: int, transition_frames: int, ) -> np.ndarray: """Map constructed task rows back to progress in the raw primitive.""" if bool(getattr(self, "dynamic_task_entry_blend", False)): return np.arange(sequence_frames, dtype=np.int64).clip( 0, motion_frames - 1 ) return np.clip( np.arange(sequence_frames, dtype=np.int64) - (int(transition_frames) - 1), 0, motion_frames - 1, ) @staticmethod def _build_pose_transition( start_pose: np.ndarray, end_pose: np.ndarray, frame_count: int, ) -> np.ndarray: """Join two absolute poses with a zero-slope quintic transition.""" count = max(int(frame_count), 2) phase = np.linspace(0.0, 1.0, count, dtype=np.float32) # Smootherstep has zero first and second derivatives at both ends, # avoiding the abrupt velocity change of the previous 0.4 s cubic join. weight = phase * phase * phase * ( phase * (phase * 6.0 - 15.0) + 10.0 ) target = np.repeat( np.asarray(end_pose, dtype=np.float32).reshape(1, 36), count, axis=0, ) return _blend_pose_rows( np.asarray(start_pose, dtype=np.float32).reshape(36), target, weight, ) @staticmethod def _stationary_progress_rows(frame_count: int) -> np.ndarray: row = np.asarray([0.0, 1.0, 0.0, 0.0, 1.0], dtype=np.float32) return np.repeat(row[None, :], max(int(frame_count), 0), axis=0) @staticmethod def _completed_motion_progress_rows(frame_count: int) -> np.ndarray: row = np.asarray([1.0, 0.0, 1.0, 0.0, 1.0], dtype=np.float32) return np.repeat(row[None, :], max(int(frame_count), 0), axis=0) @staticmethod def _sample_interturn_idle_steps() -> int: """Sample the approved immediate/short/long gap mixture on a 0.4 s grid.""" draw = random.random() if draw < 0.50: return 0 if draw < 0.85: return random.randint(1, 3) # 0.4--1.2 s return random.randint(4, 10) # 1.6--4.0 s def _sample_control_post_idle_steps( self, interaction_mode: str, *, has_followup: bool, ) -> int: """Sample post-control background without changing legacy runs. An explicit maximum enables the longer Stop-persistence curriculum. Zero keeps the historical follow-up/terminal distributions bit for bit, which is important for clean ablations and old run contracts. """ if ( str(interaction_mode) == "emergency-stop" and bool(self.control_post_idle_max_steps_explicit) ): if self.control_post_idle_sampling == "short_heavy_long_tail": minimum = int(self.control_post_idle_min_steps) maximum = int(self.control_post_idle_max_steps) span = maximum - minimum short_end = min( maximum, minimum + max(int(round(span * 4.0 / 17.0)), 0), ) middle_start = min(short_end + 1, maximum) middle_end = min( maximum, minimum + max(int(round(span * 11.0 / 17.0)), 0), ) long_start = min(middle_end + 1, maximum) draw = random.random() if draw < 0.70 or middle_start > middle_end: return random.randint(minimum, short_end) if draw < 0.95 or long_start > maximum: return random.randint(middle_start, middle_end) return random.randint(long_start, maximum) return random.randint( int(self.control_post_idle_min_steps), int(self.control_post_idle_max_steps), ) if bool(has_followup): return max( self._sample_interturn_idle_steps(), int(self.control_post_idle_min_steps), ) return random.randint( int(self.control_post_idle_min_steps), max(self.initial_idle_max_steps, 1), ) def _sample_interruption_prefix_frames( self, task_sequence: np.ndarray, *, next_reaction_frames: int, ) -> int: """Choose a Qwen-aligned speech onset inside an unfinished motion.""" sequence_frames = int(np.asarray(task_sequence).shape[0]) stride = int(self.action_exec_horizon) minimum = int( math.ceil( self.interruption_min_start_seconds * self.motion_fps / stride ) ) * stride requested_maximum = int( math.floor( self.interruption_max_start_seconds * self.motion_fps / stride ) ) * stride remaining = max( int( math.ceil( self.interruption_min_remaining_seconds * self.motion_fps ) ), int(next_reaction_frames), ) latest = ((sequence_frames - remaining) // stride) * stride maximum = min(requested_maximum, latest) if maximum < minimum: raise RuntimeError( "Motion cannot satisfy the interruption timing contract: " f"frames={sequence_frames} start>={minimum} " f"remaining>={remaining}" ) candidates = tuple(range(minimum, maximum + 1, stride)) return int(random.choice(candidates)) def _sample_control_command( self, intent: str, voice_name: str, ) -> tuple[ControlCommandSource, np.ndarray, int]: """Pick a same-speaker command and return waveform/reaction frames.""" candidates = self.control_command_catalog.get(str(intent), {}).get( str(voice_name), (), ) if not candidates: raise RuntimeError( f"No {intent} TTS command for session voice {voice_name}" ) source = random.choice(candidates) speech = _read_resampled_speech_float( source.waveform_path, self.audio_sampling_rate, ) if int(speech.size) != int(source.speech_samples): raise RuntimeError( "Control-command waveform changed after manifest creation: " f"{source.waveform_path}" ) # The state changes only after the final 0.6 s window needed to hear # the complete command. There is no synthetic </listen> interval. reaction_frames = ( self._runtime_listen_window_count(int(speech.size)) * self.action_exec_horizon ) return source, speech, int(reaction_frames) def _backchannel_candidates( self, voice_name: str, ) -> tuple[BackchannelSource, ...]: candidates = tuple( self.backchannel_catalog.get(str(voice_name), ()) ) if not candidates: raise RuntimeError( f"No {self.split} backchannel TTS for session voice " f"{voice_name}" ) return candidates def _backchannel_max_reaction_frames(self, voice_name: str) -> int: return max( self._runtime_listen_window_count(source.speech_samples) * self.action_exec_horizon for source in self._backchannel_candidates(voice_name) ) def _backchannel_motion_is_eligible( self, record: dict[str, Any], *, voice_name: str, ) -> bool: """Whether every held-out acknowledgement fits inside real motion.""" stride = int(self.action_exec_horizon) minimum_start = max( int( math.ceil( self.backchannel_min_start_seconds * self.motion_fps / stride ) ) * stride, int(math.ceil(self.turn_transition_frames / stride)) * stride, ) minimum_remaining = max( int( math.ceil( self.backchannel_min_remaining_seconds * self.motion_fps ) ), stride, ) required = ( minimum_start + self._backchannel_max_reaction_frames(voice_name) + minimum_remaining ) return self._estimated_runtime_motion_frames(record) >= required def _sample_backchannel( self, voice_name: str, ) -> tuple[BackchannelSource, np.ndarray, int]: source = random.choice(self._backchannel_candidates(voice_name)) speech = _read_resampled_speech_float( source.waveform_path, self.audio_sampling_rate, ) if int(speech.size) != int(source.speech_samples): raise RuntimeError( "Backchannel waveform changed after manifest creation: " f"{source.waveform_path}" ) reaction_frames = ( self._runtime_listen_window_count(int(speech.size)) * self.action_exec_horizon ) return source, speech, int(reaction_frames) @staticmethod def _sample_disfluency_gap_steps( rng: random.Random | None = None, ) -> int: """Sample 0.4--2.8 s with 60/30/10% short/medium/long mass.""" chooser = rng if rng is not None else random return int( chooser.choices( population=(1, 2, 3, 4, 5, 6, 7), weights=(18, 18, 9, 9, 2, 2, 2), k=1, )[0] ) def _read_disfluency_audio( self, source: DisfluencyAudioSource, ) -> np.ndarray: speech = _read_resampled_speech_float( source.waveform_path, self.audio_sampling_rate, ) if int(speech.size) != int(source.speech_samples): raise RuntimeError( "Hesitation/pause waveform changed after manifest creation: " f"{source.waveform_path}" ) return np.asarray( speech * np.float32(source.speech_scale), dtype=np.float32, ) def _build_disfluency_turn_speech( self, *, record: dict[str, Any], voice: AudioSource, assignment: SourceBackgroundAssignment, subtype: str, gap_steps: int, marker_id: str = "", repair_style: str = "", provisional_audio_dir: str = "", provisional_instruction: str = "", provisional_instruction_zh: str = "", ) -> tuple[AudioSource, SourceBackgroundAssignment, np.ndarray, dict[str, Any]]: """Compose one clean multi-segment utterance over later background mix.""" voice_name = self._voice_name(voice) key = (str(record["audio_dir"]), voice_name) by_role = self.disfluency_instruction_segments.get(key) if by_role is None or set(by_role) != {"prefix", "suffix"}: raise RuntimeError(f"Missing split instruction speech for {key}") prefix_source = by_role["prefix"] suffix_source = by_role["suffix"] prefix = self._read_disfluency_audio(prefix_source) suffix = self._read_disfluency_audio(suffix_source) gap_steps = int(gap_steps) if not 1 <= gap_steps <= 7: raise ValueError(f"Invalid disfluency gap steps: {gap_steps}") gap_samples = gap_steps * self.audio_stride_samples if subtype == "self-repair" and not repair_style: # Historical v23 callers always used an explicit repair cue and # predate the style field. Preserve that exact construction. repair_style = "explicit-cue" component_rows: list[dict[str, Any]] = [] def assemble(parts: list[tuple[str, np.ndarray, str]]) -> np.ndarray: offset = 0 arrays: list[np.ndarray] = [] for role, values, text in parts: array = np.asarray(values, dtype=np.float32).reshape(-1) arrays.append(array) component_rows.append( { "role": role, "text": text, "start_sample": int(offset), "end_sample": int(offset + array.size), } ) offset += int(array.size) return np.concatenate(arrays).astype(np.float32, copy=False) if subtype == "silent-pause": speech = assemble( [ ("final-prefix", prefix, prefix_source.text), ("background-only-pause", np.zeros(gap_samples, np.float32), ""), ("final-suffix", suffix, suffix_source.text), ] ) marker_source = None elif subtype in {"filled-pause", "self-repair"}: explicit_marker = ( subtype == "filled-pause" or repair_style == "explicit-cue" ) marker_source = None marker: np.ndarray | None = None if explicit_marker: marker_category = ( "hesitation_filler" if subtype == "filled-pause" else "repair_cue" ) marker_candidates = self.disfluency_marker_catalog.get( voice_name, {}, ).get(marker_category, ()) if not marker_candidates: raise RuntimeError( f"No {marker_category} for split={self.split} " f"voice={voice_name}" ) marker_source = next( ( candidate for candidate in marker_candidates if candidate.segment_id == str(marker_id) ), None, ) if marker_source is None: raise RuntimeError( f"Unknown {marker_category} marker {marker_id!r} for " f"split={self.split} voice={voice_name}" ) marker = self._read_disfluency_audio(marker_source) minimum_span = int( marker.size + round(0.16 * self.audio_sampling_rate) ) required_gap_steps = int( math.ceil(minimum_span / self.audio_stride_samples) ) if gap_steps < required_gap_steps: raise RuntimeError( "Stamped hesitation gap is too short for its marker: " f"gap={gap_steps} required={required_gap_steps} " f"path={marker_source.waveform_path}" ) leading = (gap_samples - int(marker.size)) // 2 trailing = gap_samples - int(marker.size) - leading elif subtype != "self-repair" or repair_style != "implicit-restart": raise RuntimeError( f"Invalid marker-free disfluency: {subtype}/{repair_style}" ) if subtype == "filled-pause": assert marker is not None and marker_source is not None speech = assemble( [ ("final-prefix", prefix, prefix_source.text), ("hesitation-background-before", np.zeros(leading, np.float32), ""), ("hesitation-filler", marker, marker_source.text), ("hesitation-background-after", np.zeros(trailing, np.float32), ""), ("final-suffix", suffix, suffix_source.text), ] ) else: provisional_key = (str(provisional_audio_dir), voice_name) provisional_roles = self.disfluency_instruction_segments.get( provisional_key ) if provisional_roles is None: raise RuntimeError( "Self-repair provisional command lacks regenerated prefix: " f"{provisional_key}" ) provisional_prefix_source = provisional_roles["prefix"] provisional_prefix = self._read_disfluency_audio( provisional_prefix_source ) if assignment.original_audio_path is None: raise RuntimeError("Self-repair needs the complete final TTS") final_speech = _read_resampled_speech_float( assignment.original_audio_path, self.audio_sampling_rate, ) if final_speech.size != int(assignment.speech_samples): raise RuntimeError( f"Final self-repair TTS changed: {assignment.original_audio_path}" ) final_speech = np.asarray( final_speech * np.float32(assignment.speech_scale), dtype=np.float32, ) if repair_style == "implicit-restart": repair_parts = [ ( "provisional-prefix", provisional_prefix, provisional_prefix_source.text, ), ( "repair-implicit-background", np.zeros(gap_samples, np.float32), "", ), ( "complete-final-instruction", final_speech, instruction_text_for_voice(record, voice), ), ] else: assert marker is not None and marker_source is not None repair_parts = [ ( "provisional-prefix", provisional_prefix, provisional_prefix_source.text, ), ( "repair-background-before", np.zeros(leading, np.float32), "", ), ("repair-cue", marker, marker_source.text), ( "repair-background-after", np.zeros(trailing, np.float32), "", ), ( "complete-final-instruction", final_speech, instruction_text_for_voice(record, voice), ), ] speech = assemble(repair_parts) else: raise ValueError(f"Unsupported disfluency subtype: {subtype!r}") peak = float(np.max(np.abs(speech), initial=0.0)) if not np.isfinite(speech).all() or peak >= 1.0: raise RuntimeError( f"Invalid/clipping disfluency speech peak={peak:g} subtype={subtype}" ) window_count = self._runtime_listen_window_count(int(speech.size)) custom_voice = AudioSource( source_key=voice.source_key, clips=(), waveform_path=voice.waveform_path, waveform_count=window_count, ) custom_assignment = SourceBackgroundAssignment( source_key=assignment.source_key, background_key=assignment.background_key, listen_start_index=assignment.listen_start_index, listen_window_count=window_count, background_window_count=assignment.background_window_count, initial_idle_max_steps=assignment.initial_idle_max_steps, max_motion_steps=assignment.max_motion_steps, safety_steps=assignment.safety_steps, mixed_audio_path=assignment.mixed_audio_path, original_audio_path=assignment.original_audio_path, speech_samples=int(speech.size), speech_scale=1.0, ) metadata = { "subtype": subtype, "turn_index": -1, "gap_steps": int(gap_steps), "gap_seconds": gap_steps * self.audio_stride_samples / float(self.audio_sampling_rate), "voice_name": voice_name, "marker_id": ( marker_source.segment_id if marker_source is not None else None ), "marker_text": marker_source.text if marker_source is not None else "", "repair_style": str(repair_style), "provisional_instruction": str(provisional_instruction), "provisional_instruction_zh": str(provisional_instruction_zh), "final_instruction": str(record["instruction"]), "final_instruction_zh": str(record["instruction_zh"]), "speech_samples": int(speech.size), "listen_window_count": int(window_count), "components": component_rows, } return custom_voice, custom_assignment, speech, metadata def _sample_backchannel_speech_start( self, task_sequence: np.ndarray, *, reaction_frames: int, ) -> int: """Place a short acknowledgement wholly inside active task motion.""" frames = int(np.asarray(task_sequence).shape[0]) stride = int(self.action_exec_horizon) minimum = max( int( math.ceil( self.backchannel_min_start_seconds * self.motion_fps / stride ) ) * stride, int(math.ceil(self.turn_transition_frames / stride)) * stride, ) requested_maximum = int( math.floor( self.backchannel_max_start_seconds * self.motion_fps / stride ) ) * stride minimum_tail = max( int( math.ceil( self.backchannel_min_remaining_seconds * self.motion_fps ) ), stride, ) latest = ( (frames - int(reaction_frames) - minimum_tail) // stride ) * stride maximum = min(requested_maximum, latest) if maximum < minimum: raise RuntimeError( "Motion is too short for an in-motion backchannel: " f"frames={frames} reaction={reaction_frames}" ) return int(random.choice(tuple(range(minimum, maximum + 1, stride)))) def _control_command_min_start_frames(self) -> int: """First valid command onset after the task-entry transition.""" stride = int(self.action_exec_horizon) requested = int( math.ceil( float( getattr( self, "control_min_start_seconds", getattr(self, "interruption_min_start_seconds", 2.0), ) ) * self.motion_fps / stride ) ) * stride transition_complete = int( math.ceil(self.turn_transition_frames / stride) ) * stride return max(stride, requested, transition_complete) def _control_max_reaction_frames( self, intent: str, voice_name: str, ) -> int: candidates = self.control_command_catalog.get(str(intent), {}).get( str(voice_name), (), ) if not candidates: return 0 return max( self._runtime_listen_window_count(int(source.speech_samples)) * self.action_exec_horizon for source in candidates ) def _control_motion_is_eligible( self, record: dict[str, Any], *, intent: str, voice_name: str, ) -> bool: """Whether every phrase for this voice fits inside real task motion. The normal instruction-to-motion transition is part of the generated task sequence. A control command may start only after that transition, must be heard completely, and must leave the configured real-motion tail before the original end. Filtering is task-local: it never removes a primitive from the preserved turn-taking/interruption pools. """ reaction_frames = self._control_max_reaction_frames( intent, voice_name, ) if reaction_frames <= 0: return False task_frames = self._estimated_runtime_motion_frames(record) max_motion_steps = int(getattr(self, "max_motion_steps", 0)) if max_motion_steps > 0: # The smoke/debug path truncates the loaded motion before session # assembly. Eligibility must use that same runtime length; # otherwise an originally long primitive can pass the index-time # filter but fail later when the complete Stop utterance no longer # fits inside its truncated task sequence. debug_frames = max( self.prefix_frames + 2, max_motion_steps * self.action_exec_horizon, ) task_frames = min(task_frames, debug_frames) if not bool(getattr(self, "dynamic_task_entry_blend", False)): task_frames += max(self.turn_transition_frames - 1, 1) minimum_tail = max( int( math.ceil( float( getattr( self, "control_min_remaining_seconds", self.action_exec_horizon / float(self.motion_fps), ) ) * self.motion_fps ) ), self.action_exec_horizon, ) required = ( self._control_command_min_start_frames() + reaction_frames + minimum_tail ) return task_frames >= required @staticmethod def _control_motion_activity( task_sequence: np.ndarray, ) -> np.ndarray: """Generic per-frame whole-body motion magnitude. The score has no action-name/category assumptions: joint RMS change, root translation and root rotation all contribute. It is used only to rank otherwise valid stop-command times within the same source. """ sequence = np.asarray(task_sequence, dtype=np.float32) if sequence.ndim != 2 or sequence.shape[1] != 36: raise ValueError( f"Control timing expects a (T,36) motion, got {sequence.shape}" ) count = int(sequence.shape[0]) if count <= 1: return np.zeros(count, dtype=np.float32) joint_delta = np.diff(sequence[:, :29], axis=0) root_delta = np.diff(sequence[:, 29:32], axis=0) quaternions = _normalise_motion_quaternions(sequence)[:, 32:36] quaternion_dot = np.abs( np.sum(quaternions[1:] * quaternions[:-1], axis=-1) ) angular_delta = 2.0 * np.arccos( np.clip(quaternion_dot, 0.0, 1.0) ) delta_score = ( np.sqrt(np.mean(np.square(joint_delta), axis=-1)) + 2.0 * np.linalg.norm(root_delta, axis=-1) + 0.25 * angular_delta ).astype(np.float32) return np.concatenate([delta_score[:1], delta_score], axis=0) def _sample_control_speech_start( self, task_sequence: np.ndarray, *, reaction_frames: int, ) -> tuple[int, dict[str, float | int | bool]]: """Choose an aligned, preferably active mid-motion command start.""" frames = int(np.asarray(task_sequence).shape[0]) stride = int(self.action_exec_horizon) minimum = self._control_command_min_start_frames() requested_maximum = int( math.floor( float( getattr( self, "control_max_start_seconds", self.interruption_max_start_seconds, ) ) * self.motion_fps / stride ) ) * stride minimum_tail = max( int( math.ceil( float( getattr( self, "control_min_remaining_seconds", stride / float(self.motion_fps), ) ) * self.motion_fps ) ), stride, ) # The configured tail is measured after the full command is audible, # not merely after its onset. The activity selector below then keeps # otherwise legal Stop examples away from already-stationary tails. latest = ( (frames - int(reaction_frames) - minimum_tail) // stride ) * stride maximum = min(requested_maximum, latest) if maximum < minimum: raise RuntimeError( "Motion is too short for a complete control command: " f"frames={frames} reaction={reaction_frames}" ) candidates = tuple(range(minimum, maximum + 1, stride)) diagnostics: dict[str, float | int | bool] = { "candidate_count": len(candidates), "active_candidate_count": len(candidates), "activity_score": 0.0, "activity_reference": 0.0, "activity_fallback": False, "minimum_remaining_frames": int(minimum_tail), } if not bool( getattr(self, "control_activity_aware_timing", False) ): return int(random.choice(candidates)), diagnostics activity = self._control_motion_activity(task_sequence) reference = float(np.quantile(activity, 0.90)) active_floor = max(reference * 0.15, 1e-6) ranked: list[tuple[float, int]] = [] active: list[tuple[float, int]] = [] for start in candidates: response = start + int(reaction_frames) onset_score = float(np.mean(activity[start : start + stride])) speech_score = float(np.mean(activity[start:response])) response_score = float( np.mean( activity[ max(response - stride, start) : min( response + stride, frames, ) ] ) ) # Both the command onset and the point at which the complete # command becomes actionable must be moving. The middle term # breaks ties without allowing one active instant to hide a # stationary boundary. score = min(onset_score, response_score) + 0.25 * speech_score ranked.append((score, start)) if onset_score >= active_floor and response_score >= active_floor: active.append((score, start)) pool = active if active else ranked best_score = max(score for score, _ in pool) near_best = [ (score, start) for score, start in pool if score >= 0.75 * best_score ] selected_score, selected_start = random.choice(near_best) diagnostics.update( { "active_candidate_count": len(active), "activity_score": float(selected_score), "activity_reference": reference, "activity_fallback": not bool(active), } ) return int(selected_start), diagnostics def _safe_hold_boundary( self, task_sequence: np.ndarray, *, earliest_frame: int, ) -> int: """Find the first supported, upright, low-velocity Qwen boundary. This uses the same joint/root velocity proxy that creates TextOp foot contacts. It therefore lets a backflip finish and land, while a pose that is already balanced can stop earlier. If no earlier supported window exists, the complete source motion is retained and its real end pose is held; no neutral-idle pose is substituted. """ sequence = np.asarray(task_sequence, dtype=np.float32) count = int(sequence.shape[0]) stride = int(self.action_exec_horizon) window = int(self.safe_hold_stability_window_frames) first = max( int(math.ceil(int(earliest_frame) / stride) * stride), window, ) if count <= first: return count joint_delta = np.diff(sequence[:, :29], axis=0, prepend=sequence[:1, :29]) root_delta = np.diff(sequence[:, 29:32], axis=0, prepend=sequence[:1, 29:32]) contacts = estimate_two_foot_contacts(joint_delta, root_delta) quat = _normalise_motion_quaternions(sequence)[:, 32:36] upright = 1.0 - 2.0 * ( np.square(quat[:, 1]) + np.square(quat[:, 2]) ) for boundary in range(first, count, stride): start = max(boundary - window, 1) stop = boundary if stop - start < max(window // 2, 2): continue supported = np.any(contacts[start:stop] > 0.5, axis=-1) joint_speed = np.linalg.norm(joint_delta[start:stop], axis=-1) horizontal_speed = np.linalg.norm(root_delta[start:stop, :2], axis=-1) vertical_speed = np.abs(root_delta[start:stop, 2]) if ( float(np.mean(supported)) >= 0.8 and float(np.min(upright[start:stop])) >= 0.55 and float(np.quantile(joint_speed, 0.8)) <= 0.08 and float(np.quantile(horizontal_speed, 0.8)) <= 0.025 and float(np.quantile(vertical_speed, 0.8)) <= 0.025 ): return int(boundary) return count def _nearest_safe_recovery_boundary( self, task_sequence: np.ndarray, *, earliest_frame: int, ) -> tuple[int, np.ndarray]: """Choose a nearby support phase and its closest audited stand pose. The search is action-agnostic. It considers every frame in the next one-second transition budget, preferring upright double-support frames and then minimizing whole-body distance and velocity to the audited neutral-pose manifold. Looking at every frame matters for cyclic gaits: a 0.4-second sampling grid can skip the brief double-support phase entirely. This avoids both a hard-coded locomotion/jump taxonomy and an unsafe instant morph from a flight or single-support frame. """ sequence = np.asarray(task_sequence, dtype=np.float32) count = int(len(sequence)) if count <= 0: raise ValueError("Safe recovery requires a non-empty task sequence") first = min(max(int(earliest_frame), 1), count) if not bool(getattr(self, "safe_stationary_anchors", False)): pose = sequence[first - 1] return first, self._sample_stationary_anchor(pose) last = min(count, first + int(self.turn_transition_frames)) boundaries = range(first, last + 1) joint_delta = np.diff( sequence[:, :29], axis=0, prepend=sequence[:1, :29] ) root_delta = np.diff( sequence[:, 29:32], axis=0, prepend=sequence[:1, 29:32] ) contacts = estimate_two_foot_contacts(joint_delta, root_delta) quat = _normalise_motion_quaternions(sequence)[:, 32:36] upright = 1.0 - 2.0 * ( np.square(quat[:, 1]) + np.square(quat[:, 2]) ) ranked: list[tuple[tuple[float, ...], int, np.ndarray]] = [] for boundary in boundaries: frame = boundary - 1 pose = sequence[frame] safe_anchor, pose_distance = self._nearest_safe_stationary_anchor( pose ) support_count = int(np.count_nonzero(contacts[frame] > 0.5)) support_rank = 0 if support_count == 2 else 1 if support_count == 1 else 2 upright_penalty = max(0.80 - float(upright[frame]), 0.0) velocity_penalty = ( 2.0 * float(np.linalg.norm(joint_delta[frame, :12])) + 4.0 * float(np.linalg.norm(root_delta[frame])) ) wait_fraction = float(boundary - first) / max(last - first, 1) rank = ( float(support_rank), float(upright_penalty > 0.0), float(pose_distance + velocity_penalty + 0.05 * wait_fraction), ) ranked.append((rank, boundary, safe_anchor)) _, boundary, anchor = min(ranked, key=lambda item: item[0]) return int(boundary), anchor.astype(np.float32, copy=False) def _stream_action_rows( self, sequence: np.ndarray, *, start: int, ) -> tuple[ np.ndarray, np.ndarray, np.ndarray, np.ndarray, np.ndarray, np.ndarray, ]: """Return v26's 30 future frames after the current state. ``start`` indexes the observed/current frame. Action token zero is therefore ``sequence[start + 1]``; the tail is repeated only for padding and is masked out. Keeping this distinction is important for TextOp root deltas and for strict train/inference RTC alignment. """ if sequence.ndim != 2 or sequence.shape[1] != 36 or not len(sequence): raise ValueError(f"Invalid streaming sequence shape: {sequence.shape}") start = int(start) if start < 0 or start >= len(sequence): raise IndexError( "Dense RTC state index is outside the source timeline: " f"start={start} sequence={len(sequence)}" ) future_indices = np.arange( start + 1, start + 1 + self.action_chunk_size, dtype=np.int64, ) valid = future_indices < len(sequence) clipped = np.clip(future_indices, 0, len(sequence) - 1) actions = sequence[clipped].astype(np.float32, copy=True) state = sequence[start].astype(np.float32, copy=True) action_reference_state = state.copy() # Two prior action tokens need three absolute poses: # x[t-2] -> x[t-1], x[t-1] -> x[t]. # At the source boundary, repeat x[0] so the initial velocity is zero # without ever borrowing history from another source. context_indices = np.clip( np.arange(start - 2, start + 1, dtype=np.int64), 0, len(sequence) - 1, ) smoothness_context_reference_state = sequence[ context_indices[0] ].astype(np.float32, copy=True) smoothness_context_actions = sequence[ context_indices[1:] ].astype(np.float32, copy=True) action_mask = valid.astype(np.float32, copy=False) if not np.any(action_mask[self.prefix_frames:] > 0.0): raise IndexError( "Dense RTC action target has no supervised frame after the " f"maximum prefix: start={start} sequence={len(sequence)}" ) return ( actions, action_mask, state, action_reference_state, smoothness_context_actions, smoothness_context_reference_state, ) def _build_session_audio_windows( self, *, turn_audio: list[dict[str, Any]], semantic_window_count: int, ) -> tuple[tuple[np.ndarray, ...], str, int]: """Mix every turn into one uninterrupted background waveform. Existing assignment rows are reused only for the original clean TTS path and its audited gain. Their old single-turn background offsets are intentionally not concatenated. One new contiguous background slice covers the complete session, so every overlapping 0.6 s window shares bit-identical PCM with its neighbours. """ count = int(semantic_window_count) if count <= 0: raise ValueError("A session needs at least one semantic window") choices: list[tuple[BackgroundWaveformSource, int]] = [] total_slots = 0 for source in sorted( self.background_catalog.values(), key=lambda item: item.source_key, ): if not isinstance(source, BackgroundWaveformSource): continue slots = int(source.window_count) - count + 1 if slots > 0: choices.append((source, slots)) total_slots += slots if total_slots <= 0: raise RuntimeError( "No one-piece office background can cover the complete " f"multi-turn session: windows={count}" ) slot = random.randrange(total_slots) background: BackgroundWaveformSource | None = None background_start = 0 for candidate, slots in choices: if slot < slots: background = candidate background_start = int(slot) break slot -= slots if background is None: raise AssertionError("Background slot selection fell through") flat_count = ( (count - 1) * self.audio_stride_samples + self.audio_window_samples ) flat = _read_pcm16_wave_segment( background.waveform_path, start_sample=background_start * self.audio_stride_samples, sample_count=flat_count, expected_rate=self.audio_sampling_rate, ) prefix_samples = self.audio_window_samples - self.audio_stride_samples previous_speech_end = -1 for payload in turn_audio: assignment = payload["assignment"] if not isinstance(assignment, SourceBackgroundAssignment): raise TypeError("Invalid multi-turn speech assignment") speech_start_frame = int( payload.get("speech_start_frame", payload["listen_start_frame"]) ) if speech_start_frame % self.audio_sample_step_frames != 0: raise RuntimeError( "A speech boundary must align with one Qwen update: " f"frame={speech_start_frame}" ) if assignment.original_audio_path is None: raise RuntimeError( f"Missing clean TTS waveform for {assignment.source_key}" ) payload_speech = payload.get("speech_waveform") speech = ( np.asarray(payload_speech, dtype=np.float32).reshape(-1) if payload_speech is not None else _read_resampled_speech_float( assignment.original_audio_path, self.audio_sampling_rate, ) ) if speech.size != int(assignment.speech_samples): raise RuntimeError( "Speech waveform changed after production assignment: " f"source={assignment.source_key} actual={speech.size} " f"expected={assignment.speech_samples}" ) start = ( prefix_samples + speech_start_frame // self.audio_sample_step_frames * self.audio_stride_samples ) end = start + int(speech.size) if start < previous_speech_end: raise RuntimeError( "Two command waveforms overlap inside one session; " "speech may overlap motion, but commands may not overlap " "each other" ) if end > flat_count: raise RuntimeError( "Multi-turn speech exceeds its continuous background: " f"source={assignment.source_key} end={end} flat={flat_count}" ) flat[start:end] += speech * np.float32(assignment.speech_scale) previous_speech_end = end if float(np.max(np.abs(flat), initial=0.0)) >= 1.0: raise RuntimeError( "Multi-turn speech/background mix clips; refusing to hide it " "with output clipping" ) flat = ( np.rint(flat * 32768.0) .clip(-32768, 32767) .astype("<i2") .astype(np.float32) / np.float32(32768.0) ) windows = tuple( flat[ offset * self.audio_stride_samples : offset * self.audio_stride_samples + self.audio_window_samples ] for offset in range(count) ) if any(window.shape != (self.audio_window_samples,) for window in windows): raise AssertionError("Incomplete multi-turn audio window") return windows, background.source_key, background_start def __getitem__(self, index: int) -> dict[str, Any]: """Build one source; validation augmentation is repeatable per row.""" deterministic = bool( getattr(self, "deterministic_eval", True) and getattr(self, "split", "train") == "val" ) if not deterministic: return self._build_source(index) sessions = getattr(self, "sessions", None) session = ( sessions[int(index)] if sessions is not None else (self.records[int(index)],) ) row_fingerprint = sum( (turn_index + 1) * int(record["row_id"]) for turn_index, record in enumerate(session) ) deterministic_index = int( session[0].get("_session_pool_index", index) ) seed = ( int(getattr(self, "split_seed", 42)) * 1_000_003 + row_fingerprint * 97_409 + deterministic_index ) & ((1 << 64) - 1) random_state = random.getstate() random.seed(seed) try: return self._build_source(index) finally: random.setstate(random_state) def _build_source(self, index: int) -> dict[str, Any]: if bool(getattr(self, "multi_turn_sessions", False)): return self._build_session_source(index) return self._build_single_source(index) def _build_session_source(self, index: int) -> dict[str, Any]: """Build one 1--4 turn session with continuous action, audio and KV.""" session = tuple(dict(record) for record in self.sessions[int(index)]) if not self.session_min_turns <= len(session) <= self.session_max_turns: raise RuntimeError( f"Invalid runtime session size: {len(session)}" ) interaction_modes = { str(record.get("_session_interaction_mode", "turn-taking")) for record in session } if len(interaction_modes) != 1: raise RuntimeError( f"One session mixes interaction modes: {interaction_modes}" ) interaction_mode = interaction_modes.pop() if interaction_mode == "interruption" and len(session) < 2: raise RuntimeError("Command interruption cannot be a single-source session") if interaction_mode not in { "turn-taking", "interruption", "emergency-stop", "safe-hold", "backchannel", "hesitation", "pause", }: raise RuntimeError( f"Unsupported session interaction mode: {interaction_mode!r}" ) if interaction_mode == "interruption" and not bool( getattr(self, "interruption_sessions", False) ): raise RuntimeError( "An interruption session reached a dataset with the feature disabled" ) session_control_turn_index = session[0].get( "_session_control_turn_index" ) session_control_has_followup = bool( session[0].get("_session_control_has_followup", False) ) session_backchannel_turn_index = session[0].get( "_session_backchannel_turn_index" ) session_disfluency_turn_index = session[0].get( "_session_disfluency_turn_index" ) session_disfluency_subtype = session[0].get( "_session_disfluency_subtype" ) session_disfluency_gap_steps = session[0].get( "_session_disfluency_gap_steps" ) session_disfluency_marker_id = session[0].get( "_session_disfluency_marker_id" ) session_disfluency_repair_style = session[0].get( "_session_disfluency_repair_style", "", ) if session_disfluency_turn_index is not None: if any( record.get("_session_disfluency_turn_index") != session_disfluency_turn_index or record.get("_session_disfluency_subtype") != session_disfluency_subtype or record.get("_session_disfluency_gap_steps") != session_disfluency_gap_steps or record.get("_session_disfluency_marker_id") != session_disfluency_marker_id or record.get("_session_disfluency_repair_style", "") != session_disfluency_repair_style for record in session ): raise RuntimeError("Disfluency-event placement drifted within session") session_disfluency_turn_index = int(session_disfluency_turn_index) if ( interaction_mode not in {"hesitation", "pause"} or not 0 <= session_disfluency_turn_index < len(session) or session_disfluency_subtype not in {"silent-pause", "filled-pause", "self-repair"} or not 1 <= int(session_disfluency_gap_steps or 0) <= 7 ): raise RuntimeError( "Invalid hesitation/pause event: " f"mode={interaction_mode} turn={session_disfluency_turn_index} " f"subtype={session_disfluency_subtype}" ) if session_disfluency_subtype == "self-repair": if session_disfluency_repair_style not in { "explicit-cue", "implicit-restart", }: raise RuntimeError( "Invalid self-repair style: " f"{session_disfluency_repair_style!r}" ) if ( session_disfluency_repair_style == "explicit-cue" ) != bool(session_disfluency_marker_id): raise RuntimeError( "Self-repair cue/style mismatch: " f"style={session_disfluency_repair_style!r} " f"marker={session_disfluency_marker_id!r}" ) elif session_disfluency_repair_style: raise RuntimeError( "Only self-repair may carry a repair style: " f"subtype={session_disfluency_subtype!r} " f"style={session_disfluency_repair_style!r}" ) elif interaction_mode in {"hesitation", "pause"}: raise RuntimeError("Hesitation/pause session has no event placement") if session_backchannel_turn_index is not None: if any( record.get("_session_backchannel_turn_index") != session_backchannel_turn_index for record in session ): raise RuntimeError( "Backchannel-event placement drifted within session" ) session_backchannel_turn_index = int( session_backchannel_turn_index ) if ( interaction_mode != "backchannel" or not 0 <= session_backchannel_turn_index < len(session) ): raise RuntimeError( "Invalid backchannel-event placement: " f"mode={interaction_mode} " f"turn={session_backchannel_turn_index}" ) elif interaction_mode == "backchannel": raise RuntimeError("Backchannel session has no event placement") if session_control_turn_index is not None: if any( record.get("_session_control_turn_index") != session_control_turn_index or bool( record.get("_session_control_has_followup", False) ) != session_control_has_followup for record in session ): raise RuntimeError("Control-event placement drifted within session") session_control_turn_index = int(session_control_turn_index) if not 0 <= session_control_turn_index < len(session): raise RuntimeError( "Control-event turn is outside its session: " f"turn={session_control_turn_index} turns={len(session)}" ) if session_control_has_followup != ( session_control_turn_index + 1 < len(session) ): raise RuntimeError( "Control follow-up flag disagrees with event placement" ) sequence_parts: list[np.ndarray] = [] control_parts: list[np.ndarray] = [] progress_parts: list[np.ndarray] = [] original_frame_parts: list[np.ndarray] = [] turn_index_parts: list[np.ndarray] = [] turn_audio: list[dict[str, Any]] = [] idle_motion_names: list[str | None] = [] listen_motion_names: list[str | None] = [] task_transition_frames: list[int] = [] recovery_transition_frames: list[int] = [] inter_turn_idle_steps: list[int] = [] listen_reaction_frames: list[int] = [] listen_reaction_idle_motion_names: list[str | None] = [] speech_onset_samples: list[int] = [] interruption_start_frames: list[int] = [] interruption_original_frames: list[int] = [] interruption_remaining_frames: list[int] = [] interruption_terminal_hold_frames: list[int] = [] control_command_events: list[dict[str, Any]] = [] backchannel_events: list[dict[str, Any]] = [] disfluency_events: list[dict[str, Any]] = [] def append_part( values: np.ndarray, *, control_label: int, progress: np.ndarray, original_frames: np.ndarray, turn_index: int, ) -> None: array = np.asarray(values, dtype=np.float32) count = int(array.shape[0]) if array.shape != (count, 36): raise ValueError(f"Invalid session motion part: {array.shape}") if progress.shape != (count, 5): raise ValueError( f"Invalid session progress part: {progress.shape}" ) if original_frames.shape != (count,): raise ValueError( "Invalid session original-frame part: " f"{original_frames.shape}" ) if count <= 0: return sequence_parts.append(array) control_parts.append( np.full(count, int(control_label), dtype=np.int64) ) progress_parts.append(np.asarray(progress, dtype=np.float32)) original_frame_parts.append( np.asarray(original_frames, dtype=np.int64) ) turn_index_parts.append( np.full(count, int(turn_index), dtype=np.int64) ) motions: list[np.ndarray] = [] voices: list[AudioSource] = [] assignments: list[SourceBackgroundAssignment] = [] for record in session: motion = self._load_motion(record["motion_dir"]) if motion.shape[0] <= 0: raise RuntimeError(f"Empty motion: {record['motion_dir']}") if self.max_motion_steps > 0: debug_frames = max( self.prefix_frames + 2, self.max_motion_steps * self.action_exec_horizon, ) motion = motion[: min(len(motion), debug_frames)] record["actual_frames"] = int(motion.shape[0]) voice = self._voice_for_record(record) assignment = self.background_assignments.get(voice.source_key) if assignment is None: raise RuntimeError( f"No raw-audio assignment for {voice.source_key}" ) motions.append(motion) voices.append(voice) assignments.append(assignment) session_voice_name = session[0].get("_session_voice_name") session_voice_variant = session[0].get("_session_voice_variant") if session_voice_name is not None and any( record.get("_session_voice_name") != session_voice_name for record in session ): raise RuntimeError("One session unexpectedly contains multiple voices") # Load and measure every utterance before assembling motion. An # interruption cut must leave enough real old-motion frames to cover # the next utterance's causal reaction period. turn_speeches: list[np.ndarray | None] = [] turn_onset_samples: list[int] = [] turn_reaction_frames: list[int] = [] turn_disfluency_metadata: list[dict[str, Any] | None] = [ None for _ in session ] for turn_index, (record, voice, assignment) in enumerate( zip(session, voices, assignments) ): speech: np.ndarray | None = None onset_sample = 0 reaction_frames = 0 if bool(getattr(self, "causal_listen_timing", False)): if turn_index == session_disfluency_turn_index: ( voice, assignment, speech, event_metadata, ) = self._build_disfluency_turn_speech( record=record, voice=voice, assignment=assignment, subtype=str(session_disfluency_subtype), gap_steps=int(session_disfluency_gap_steps), marker_id=str(session_disfluency_marker_id or ""), repair_style=str( session_disfluency_repair_style or "" ), provisional_audio_dir=str( record.get("_session_provisional_audio_dir", "") ), provisional_instruction=str( record.get("_session_provisional_instruction", "") ), provisional_instruction_zh=str( record.get( "_session_provisional_instruction_zh", "", ) ), ) voices[turn_index] = voice assignments[turn_index] = assignment turn_disfluency_metadata[turn_index] = event_metadata else: if assignment.original_audio_path is None: raise RuntimeError( "Causal listen timing needs the clean TTS waveform: " f"{assignment.source_key}" ) speech = _read_resampled_speech_float( assignment.original_audio_path, self.audio_sampling_rate, ) if speech.size != int(assignment.speech_samples): raise RuntimeError( "Speech waveform changed after production assignment: " f"source={assignment.source_key} actual={speech.size} " f"expected={assignment.speech_samples}" ) onset_sample = detect_clean_speech_onset_sample( speech, self.audio_sampling_rate, ) prefix_samples = ( self.audio_window_samples - self.audio_stride_samples ) onset_steps = max( 1, int( math.ceil( (prefix_samples + onset_sample) / self.audio_stride_samples ) ), ) reaction_frames = ( onset_steps + self.listen_reaction_steps ) * self.action_exec_horizon turn_speeches.append(speech) turn_onset_samples.append(int(onset_sample)) turn_reaction_frames.append(int(reaction_frames)) anchor = self._sample_stationary_anchor(motions[0][0]) initial_idle_steps = random.randint(0, self.initial_idle_max_steps) initial_idle_frames = initial_idle_steps * self.action_exec_horizon idle_sequence, idle_name = self._sample_stationary_sequence( "idle", frame_count=initial_idle_frames, anchor_pose=anchor, ) idle_motion_names.append(idle_name) append_part( idle_sequence, control_label=IDLE_TOKEN_ID, progress=self._stationary_progress_rows(initial_idle_frames), original_frames=np.zeros(initial_idle_frames, dtype=np.int64), turn_index=0, ) pending_task_sequence: np.ndarray | None = None pending_task_original_frames: np.ndarray | None = None pending_task_cursor = 0 for turn_index, (record, motion, voice, assignment) in enumerate( zip(session, motions, voices, assignments) ): if sum(len(part) for part in sequence_parts) % self.action_exec_horizon: raise RuntimeError( "A listen phase would start inside one Qwen window" ) speech_start_frame = sum(len(part) for part in sequence_parts) speech = turn_speeches[turn_index] onset_sample = turn_onset_samples[turn_index] reaction_frames = turn_reaction_frames[turn_index] listen_entry_pose: np.ndarray | None = None if bool(getattr(self, "causal_listen_timing", False)): if interaction_mode == "interruption" and turn_index > 0: if ( pending_task_sequence is None or pending_task_original_frames is None ): raise AssertionError( "Interrupted speech has no preceding motion tail" ) reaction_stop = min( pending_task_cursor + reaction_frames, len(pending_task_sequence), ) continued_motion = pending_task_sequence[ pending_task_cursor:reaction_stop ] continued_original = pending_task_original_frames[ pending_task_cursor:reaction_stop ] previous_motion_frames = int( session[turn_index - 1]["actual_frames"] ) append_part( continued_motion, control_label=MOTION_TOKEN_ID, progress=_progress_rows( continued_original, previous_motion_frames, ), original_frames=continued_original, turn_index=turn_index - 1, ) reaction_hold_frames = max( pending_task_cursor + reaction_frames - len(pending_task_sequence), 0, ) anchor = ( continued_motion[-1].copy() if len(continued_motion) else pending_task_sequence[-1].copy() ) if reaction_hold_frames: # The speech onset was still at least the configured # 1.5 s before the original task end. If unusually # long leading silence delays the causal listen point # beyond that tail, keep the completed old pose under # </motion> rather than crashing a worker or looking # into future speech. The following quintic listen # entry remains the only control-state transition. append_part( np.repeat( anchor[None, :], reaction_hold_frames, axis=0, ), control_label=MOTION_TOKEN_ID, progress=self._completed_motion_progress_rows( reaction_hold_frames ), original_frames=np.full( reaction_hold_frames, previous_motion_frames - 1, dtype=np.int64, ), turn_index=turn_index - 1, ) interruption_terminal_hold_frames.append( int(reaction_hold_frames) ) listen_entry_pose = anchor.copy() if bool( getattr(self, "safe_stationary_anchors", False) ): safe_anchor = self._sample_stationary_anchor(anchor) anchor = _align_task_motion_to_anchor_heading( safe_anchor[None, :], anchor, )[0] pending_task_sequence = None pending_task_original_frames = None pending_task_cursor = 0 listen_reaction_idle_motion_names.append(None) else: reaction_idle, reaction_name = ( self._sample_stationary_sequence( "idle", frame_count=reaction_frames, anchor_pose=anchor, ) ) append_part( reaction_idle, control_label=IDLE_TOKEN_ID, progress=self._stationary_progress_rows( reaction_frames ), original_frames=np.zeros( reaction_frames, dtype=np.int64, ), turn_index=turn_index, ) listen_reaction_idle_motion_names.append(reaction_name) listen_start_frame = sum(len(part) for part in sequence_parts) listen_frames = voice.window_count * self.action_exec_horizon listen_kwargs: dict[str, Any] = {} if bool(getattr(self, "causal_listen_timing", False)): listen_kwargs["blend_frames"] = self.turn_transition_frames if listen_entry_pose is not None: # The first listen row is exactly the actually executed old # motion pose; the following rows converge into the nearest # audited two-support standing/listening trajectory. listen_kwargs["entry_pose"] = listen_entry_pose listen_sequence, listen_name = self._sample_stationary_sequence( "listen", frame_count=listen_frames, anchor_pose=anchor, **listen_kwargs, ) listen_motion_names.append(listen_name) append_part( listen_sequence, control_label=LISTEN_TOKEN_ID, progress=self._stationary_progress_rows(listen_frames), original_frames=np.zeros(listen_frames, dtype=np.int64), turn_index=turn_index, ) turn_audio.append( { "voice": voice, "assignment": assignment, "speech_start_frame": speech_start_frame, "listen_start_frame": listen_start_frame, "speech_waveform": speech, "speech_onset_sample": onset_sample, "listen_reaction_frames": reaction_frames, } ) speech_onset_samples.append(int(onset_sample)) listen_reaction_frames.append(int(reaction_frames)) if turn_disfluency_metadata[turn_index] is not None: event = dict(turn_disfluency_metadata[turn_index] or {}) uncertainty_components = [ component for component in event["components"] if component["role"] in DISFLUENCY_UNCERTAINTY_ROLES ] final_components = [ component for component in event["components"] if component["role"] in {"final-suffix", "complete-final-instruction"} ] if not uncertainty_components or len(final_components) != 1: raise RuntimeError( "Disfluency timeline lacks uncertainty/final components: " f"{event['components']}" ) def sample_to_frame(sample: int, *, ceil: bool) -> int: value = int(sample) * self.motion_fps frames = ( (value + self.audio_sampling_rate - 1) // self.audio_sampling_rate if ceil else value // self.audio_sampling_rate ) return int(speech_start_frame + frames) uncertainty_start_sample = min( int(component["start_sample"]) for component in uncertainty_components ) final_component = final_components[0] event.update( { "turn_index": int(turn_index), "speech_start_frame": int(speech_start_frame), "listen_start_frame": int(listen_start_frame), "motion_start_frame": int( listen_start_frame + listen_frames ), "uncertainty_start_frame": sample_to_frame( uncertainty_start_sample, ceil=False, ), "final_intent_start_frame": sample_to_frame( int(final_component["start_sample"]), ceil=False, ), "final_intent_complete_frame": sample_to_frame( int(final_component["end_sample"]), ceil=True, ), "expected_control_during_gap": LISTEN_TOKEN_ID, } ) disfluency_events.append(event) entry_frames = ( self.turn_transition_frames if bool(getattr(self, "causal_listen_timing", False)) else self.stationary_blend_frames if turn_index == 0 else self.turn_transition_frames ) task_sequence, entry_frames = self._build_task_sequence( motion, stationary_anchor=( listen_sequence[-1] if bool(getattr(self, "stationary_listen_hold", False)) and len(listen_sequence) else anchor ), transition_frames=entry_frames, ) task_transition_frames.append(entry_frames) task_original_frames = self._task_original_frame_indices( sequence_frames=len(task_sequence), motion_frames=len(motion), transition_frames=entry_frames, ) if ( interaction_mode == "backchannel" and turn_index == session_backchannel_turn_index ): voice_name = str(session_voice_name or self._voice_name(voice)) backchannel, backchannel_speech, backchannel_reaction_frames = ( self._sample_backchannel(voice_name) ) backchannel_start = self._sample_backchannel_speech_start( task_sequence, reaction_frames=backchannel_reaction_frames, ) backchannel_global_start = ( sum(len(part) for part in sequence_parts) + backchannel_start ) backchannel_assignment = SourceBackgroundAssignment( source_key=( f"backchannel/{backchannel.split}/" f"{backchannel.phrase_id}/{backchannel.voice_name}" ), background_key="session-selected", listen_start_index=0, listen_window_count=self._runtime_listen_window_count( backchannel.speech_samples ), background_window_count=0, initial_idle_max_steps=0, max_motion_steps=0, safety_steps=0, original_audio_path=backchannel.waveform_path, speech_samples=backchannel.speech_samples, speech_scale=backchannel.speech_scale, ) turn_audio.append( { "voice": None, "assignment": backchannel_assignment, "speech_start_frame": backchannel_global_start, "listen_start_frame": backchannel_global_start, "speech_waveform": backchannel_speech, "speech_onset_sample": ( detect_clean_speech_onset_sample( backchannel_speech, self.audio_sampling_rate, ) ), "listen_reaction_frames": ( backchannel_reaction_frames ), } ) backchannel_events.append( { "turn_index": int(turn_index), "category": backchannel.category, "phrase_id": backchannel.phrase_id, "split": backchannel.split, "voice_name": backchannel.voice_name, "text": backchannel.text, "speech_start_frame": int( backchannel_global_start ), "reaction_frames": int( backchannel_reaction_frames ), "source_motion_speech_start_frame": int( backchannel_start ), "source_motion_remaining_after_heard_frames": int( len(task_sequence) - backchannel_start - backchannel_reaction_frames ), "expected_control_id": MOTION_TOKEN_ID, } ) if ( interaction_mode in {"emergency-stop", "safe-hold"} and ( session_control_turn_index is None or turn_index == session_control_turn_index ) ): voice_name = str(session_voice_name or self._voice_name(voice)) command, command_speech, command_reaction_frames = ( self._sample_control_command( interaction_mode, voice_name, ) ) command_start, command_timing = ( self._sample_control_speech_start( task_sequence, reaction_frames=command_reaction_frames, ) ) append_part( task_sequence[:command_start], control_label=MOTION_TOKEN_ID, progress=_progress_rows( task_original_frames[:command_start], len(motion), ), original_frames=task_original_frames[:command_start], turn_index=turn_index, ) command_global_start = sum( len(part) for part in sequence_parts ) command_assignment = SourceBackgroundAssignment( source_key=( f"control/{command.intent}/{command.phrase_id}/" f"{command.voice_name}" ), background_key="session-selected", listen_start_index=0, listen_window_count=self._runtime_listen_window_count( command.speech_samples ), background_window_count=0, initial_idle_max_steps=0, max_motion_steps=0, safety_steps=0, original_audio_path=command.waveform_path, speech_samples=command.speech_samples, speech_scale=command.speech_scale, ) turn_audio.append( { "voice": None, "assignment": command_assignment, "speech_start_frame": command_global_start, "listen_start_frame": command_global_start, "speech_waveform": command_speech, "speech_onset_sample": detect_clean_speech_onset_sample( command_speech, self.audio_sampling_rate, ), "listen_reaction_frames": command_reaction_frames, } ) reaction_end = min( command_start + command_reaction_frames, len(task_sequence), ) append_part( task_sequence[command_start:reaction_end], control_label=MOTION_TOKEN_ID, progress=_progress_rows( task_original_frames[command_start:reaction_end], len(motion), ), original_frames=task_original_frames[ command_start:reaction_end ], turn_index=turn_index, ) if reaction_end < command_start + command_reaction_frames: missing = command_start + command_reaction_frames - reaction_end append_part( np.repeat(task_sequence[-1:], missing, axis=0), control_label=MOTION_TOKEN_ID, progress=self._completed_motion_progress_rows(missing), original_frames=np.full( missing, len(motion) - 1, dtype=np.int64, ), turn_index=turn_index, ) reaction_end = len(task_sequence) if interaction_mode == "safe-hold": hold_boundary = self._safe_hold_boundary( task_sequence, earliest_frame=reaction_end, ) if hold_boundary > reaction_end: append_part( task_sequence[reaction_end:hold_boundary], control_label=MOTION_TOKEN_ID, progress=_progress_rows( task_original_frames[ reaction_end:hold_boundary ], len(motion), ), original_frames=task_original_frames[ reaction_end:hold_boundary ], turn_index=turn_index, ) anchor = task_sequence[ min(max(hold_boundary - 1, 0), len(task_sequence) - 1) ].copy() # If the natural stable point is not exactly on the 0.4 s # grid, keep its real pose under </motion> until the next # decision boundary; the first </idle> frame is identical. alignment = ( -sum(len(part) for part in sequence_parts) ) % self.action_exec_horizon if alignment: append_part( np.repeat(anchor[None, :], alignment, axis=0), control_label=MOTION_TOKEN_ID, progress=self._completed_motion_progress_rows(alignment), original_frames=np.full( alignment, len(motion) - 1, dtype=np.int64, ), turn_index=turn_index, ) next_anchor = anchor.copy() recovery_transition_frames.append(0) else: recovery_boundary, next_anchor = ( self._nearest_safe_recovery_boundary( task_sequence, earliest_frame=reaction_end, ) ) if recovery_boundary > reaction_end: safe_tail = task_sequence[ reaction_end:recovery_boundary ] append_part( safe_tail, control_label=IDLE_TOKEN_ID, progress=self._stationary_progress_rows( len(safe_tail) ), original_frames=task_original_frames[ reaction_end:recovery_boundary ], turn_index=turn_index, ) stop_pose = task_sequence[recovery_boundary - 1].copy() next_anchor = _align_task_motion_to_anchor_heading( next_anchor[None, :], stop_pose, )[0] recovery_frames = int(self.turn_transition_frames) recovery_frames += ( -( sum(len(part) for part in sequence_parts) + recovery_frames ) ) % self.action_exec_horizon recovery = self._build_pose_transition( stop_pose, next_anchor, recovery_frames + 1, )[1:] append_part( recovery, control_label=IDLE_TOKEN_ID, progress=self._stationary_progress_rows(len(recovery)), original_frames=np.full( len(recovery), int(task_original_frames[recovery_boundary - 1]), dtype=np.int64, ), turn_index=turn_index, ) recovery_transition_frames.append( int(recovery_boundary - reaction_end + len(recovery)) ) has_followup = turn_index + 1 < len(session) idle_steps = self._sample_control_post_idle_steps( interaction_mode, has_followup=has_followup, ) if has_followup: inter_turn_idle_steps.append(idle_steps) idle_turn_index = turn_index + 1 else: terminal_idle_steps = idle_steps idle_turn_index = turn_index idle_frames = idle_steps * self.action_exec_horizon if interaction_mode == "safe-hold": post_control_idle = np.repeat( next_anchor[None, :], idle_frames, axis=0, ) idle_name = "held_actual_stable_pose" else: post_control_idle, idle_name = ( self._sample_stationary_sequence( "idle", frame_count=idle_frames, anchor_pose=next_anchor, ) ) idle_motion_names.append(idle_name) append_part( post_control_idle, control_label=IDLE_TOKEN_ID, progress=self._stationary_progress_rows(idle_frames), original_frames=np.zeros(idle_frames, dtype=np.int64), turn_index=idle_turn_index, ) control_command_events.append( { "turn_index": int(turn_index), "intent": command.intent, "phrase_id": command.phrase_id, "voice_name": command.voice_name, "text": command.text, "speech_start_frame": int(command_global_start), "reaction_frames": int(command_reaction_frames), "source_motion_speech_start_frame": int(command_start), "source_motion_response_frame": int( command_start + command_reaction_frames ), "source_motion_remaining_at_speech_start_frames": int( len(task_sequence) - command_start ), "source_motion_remaining_at_response_frames": int( len(task_sequence) - command_start - command_reaction_frames ), **command_timing, "held_original_pose": interaction_mode == "safe-hold", } ) anchor = next_anchor continue if ( interaction_mode == "interruption" and turn_index + 1 < len(session) ): prefix_frames = self._sample_interruption_prefix_frames( task_sequence, next_reaction_frames=turn_reaction_frames[turn_index + 1], ) append_part( task_sequence[:prefix_frames], control_label=MOTION_TOKEN_ID, progress=_progress_rows( task_original_frames[:prefix_frames], len(motion), ), original_frames=task_original_frames[:prefix_frames], turn_index=turn_index, ) interruption_start_frames.append( sum(len(part) for part in sequence_parts) ) interruption_original_frames.append( int(task_original_frames[prefix_frames - 1]) ) interruption_remaining_frames.append( int(len(task_sequence) - prefix_frames) ) pending_task_sequence = task_sequence pending_task_original_frames = task_original_frames pending_task_cursor = prefix_frames anchor = task_sequence[prefix_frames - 1].copy() continue append_part( task_sequence, control_label=MOTION_TOKEN_ID, progress=_progress_rows(task_original_frames, len(motion)), original_frames=task_original_frames, turn_index=turn_index, ) # Complete the task, then settle into a new neutral stance at the # task's actual final XY/yaw. The transition length is extended by # at most 19 frames so the next token boundary lands exactly on a # Qwen update instead of becoming impossible scalar supervision. task_end = task_sequence[-1] next_anchor = self._sample_stationary_anchor(task_end) next_anchor = _align_task_motion_to_anchor_heading( next_anchor[None, :], task_end, )[0] timeline_length = sum(len(part) for part in sequence_parts) recovery_frames = self.turn_transition_frames recovery_frames += ( -( timeline_length + recovery_frames - 1 ) ) % self.action_exec_horizon recovery = self._build_pose_transition( task_end, next_anchor, recovery_frames, )[1:] recovery_transition_frames.append(int(recovery_frames)) append_part( recovery, control_label=MOTION_TOKEN_ID, progress=self._completed_motion_progress_rows(len(recovery)), original_frames=np.full( len(recovery), len(motion) - 1, dtype=np.int64, ), turn_index=turn_index, ) if sum(len(part) for part in sequence_parts) % self.action_exec_horizon: raise AssertionError("Recovery did not align the Qwen boundary") if turn_index + 1 < len(session): idle_steps = self._sample_interturn_idle_steps() inter_turn_idle_steps.append(idle_steps) idle_turn_index = turn_index + 1 else: idle_steps = random.randint( 1, max(self.initial_idle_max_steps, 1), ) terminal_idle_steps = idle_steps idle_turn_index = turn_index idle_frames = idle_steps * self.action_exec_horizon turn_idle, turn_idle_name = self._sample_stationary_sequence( "idle", frame_count=idle_frames, anchor_pose=next_anchor, ) idle_motion_names.append(turn_idle_name) append_part( turn_idle, control_label=IDLE_TOKEN_ID, progress=self._stationary_progress_rows(idle_frames), original_frames=np.zeros(idle_frames, dtype=np.int64), turn_index=idle_turn_index, ) anchor = next_anchor if pending_task_sequence is not None: raise AssertionError("Final interruption motion was left pending") full_sequence = np.concatenate(sequence_parts, axis=0) timeline_control_labels = np.concatenate(control_parts, axis=0) timeline_progress = np.concatenate(progress_parts, axis=0) timeline_original_frames = np.concatenate( original_frame_parts, axis=0, ) timeline_turn_indices = np.concatenate(turn_index_parts, axis=0) if not ( len(full_sequence) == len(timeline_control_labels) == len(timeline_progress) == len(timeline_original_frames) == len(timeline_turn_indices) ): raise AssertionError("Multi-turn frame timelines have different lengths") valid_dense_starts = max( len(full_sequence) - self.prefix_frames - 1, 0, ) semantic_window_count = int( math.ceil(valid_dense_starts / self.action_exec_horizon) ) if semantic_window_count <= 0: raise RuntimeError("Multi-turn session is too short for RTC") if ( self.max_stream_steps > 0 and semantic_window_count > self.max_stream_steps ): raise RuntimeError( "Session exceeds the complete-KV context guard: " f"windows={semantic_window_count} limit={self.max_stream_steps}" ) turn_audio.sort(key=lambda payload: int(payload["speech_start_frame"])) normal_turn_audio = [ payload for payload in turn_audio if payload["voice"] is not None ] if len(normal_turn_audio) != len(session): raise AssertionError( "Normal instruction audio metadata drifted from session turns: " f"audio={len(normal_turn_audio)} turns={len(session)}" ) audio_windows, background_key, background_start = ( self._build_session_audio_windows( turn_audio=turn_audio, semantic_window_count=semantic_window_count, ) ) row_ids = [int(record["row_id"]) for record in session] source_id = ( f"session{int(index):06d}::rows" + "-".join(f"{row_id:06d}" for row_id in row_ids) ) source_id += f"::mode={interaction_mode}" if session_voice_name is not None: source_id += ( f"::voice={session_voice_name}" f"::variant={int(session_voice_variant)}" ) steps: list[dict[str, Any]] = [] action_phase_counts = {"idle": 0, "listen": 0, "motion": 0} phase_names = { IDLE_TOKEN_ID: "idle", LISTEN_TOKEN_ID: "listen", MOTION_TOKEN_ID: "motion", } final_start = len(full_sequence) - self.prefix_frames - 2 for semantic_index, semantic_input in enumerate(audio_windows): target_base = semantic_index * self.action_exec_horizon stop = min( target_base + self.action_exec_horizon, final_start + 1, ) if stop <= target_base: break for target_start in range( target_base, stop, self.action_sample_step_frames, ): action_substep = target_start - target_base update_semantic_cache = action_substep == 0 ( actions, action_mask, state, reference, smoothness_context_actions, smoothness_context_reference_state, ) = self._stream_action_rows( full_sequence, start=target_start, ) action_indices = np.arange( target_start + 1, target_start + 1 + self.action_chunk_size, dtype=np.int64, ) clipped = np.clip( action_indices, 0, len(full_sequence) - 1, ) control_id = int(timeline_control_labels[target_start]) phase = phase_names[control_id] active_turn = int(timeline_turn_indices[target_start]) semantic_kwargs: dict[str, Any] = {} if update_semantic_cache: semantic_kwargs["audio_waveform"] = semantic_input step = self._raw_step( **semantic_kwargs, actions=actions, action_mask=action_mask, state=state, action_reference_state=reference, smoothness_context_actions=smoothness_context_actions, smoothness_context_reference_state=( smoothness_context_reference_state ), action_control_labels=timeline_control_labels[clipped], future_progress=timeline_progress[clipped], control_progress=( timeline_progress[target_start - 1] if target_start > 0 else self._stationary_progress_rows(1)[0] ), previous_control_label=( int(timeline_control_labels[target_start - 1]) if target_start > 0 else 3 ), phase=phase, source_id=source_id, step_index=len(steps), frame_id=int(timeline_original_frames[target_start]), semantic_frame_id=target_base, semantic_window_index=semantic_index, action_substep=action_substep, record=session[active_turn], instruction_text=instruction_text_for_voice( session[active_turn], voices[active_turn], ), instruction_history_texts=tuple( instruction_text_for_voice( session[history_turn], voices[history_turn], ) for history_turn in range(active_turn) ), update_semantic_cache=update_semantic_cache, control_transition=bool( update_semantic_cache and target_start > 0 and timeline_control_labels[target_start] != timeline_control_labels[target_start - 1] ), ) step["turn_index"] = active_turn step["interaction_mode"] = interaction_mode step["background_window_index"] = ( background_start + semantic_index ) steps.append(step) action_phase_counts[phase] += 1 listen_steps = sum(voice.window_count for voice in voices) semantic_phase_labels = timeline_control_labels[ np.arange(semantic_window_count) * self.action_exec_horizon ] duration_targets = [ step for step in steps if bool(step["motion_duration_target_mask"]) ] duration_target_turns = [ int(step["turn_index"]) for step in duration_targets ] expected_duration_turns = list(range(len(session))) if duration_target_turns != expected_duration_turns: raise RuntimeError( "Each multi-turn primitive must expose exactly one causal " "motion-duration target before transforms: " f"source={source_id} expected_turns={expected_duration_turns} " f"actual_turns={duration_target_turns}" ) text_targets = [ step for step in steps if bool(step["latest_instruction_target_mask"]) ] text_target_turns = [int(step["turn_index"]) for step in text_targets] if text_target_turns != expected_duration_turns: raise RuntimeError( "Each multi-turn primitive must expose exactly one causal " "latest-instruction text target: " f"source={source_id} expected_turns={expected_duration_turns} " f"actual_turns={text_target_turns}" ) if any( step["latest_instruction_text"] != instruction_text_for_voice(session[turn], voices[turn]) for step, turn in zip(text_targets, expected_duration_turns) ): raise RuntimeError( f"Voice-specific instruction text drifted in {source_id}" ) memory_targets = [ step for step in steps if bool(step["instruction_memory_target_mask"]) ] expected_memory_turns = list(range(1, len(session))) memory_target_turns = [ int(step["turn_index"]) for step in memory_targets ] if memory_target_turns != expected_memory_turns: raise RuntimeError( "Each non-initial multi-turn primitive must expose exactly " "one historical-instruction target: " f"source={source_id} expected_turns={expected_memory_turns} " f"actual_turns={memory_target_turns}" ) expected_memory_slots = [ "previous" if turn == 1 or turn % 2 == 1 else "first" for turn in expected_memory_turns ] expected_memory_texts = [ instruction_text_for_voice( session[turn - 1 if slot == "previous" else 0], voices[turn - 1 if slot == "previous" else 0], ) for turn, slot in zip( expected_memory_turns, expected_memory_slots, ) ] if any( step["instruction_memory_slot"] != expected_slot or step["instruction_memory_text"] != expected_text for step, expected_slot, expected_text in zip( memory_targets, expected_memory_slots, expected_memory_texts, ) ): raise RuntimeError( f"Historical-instruction target drifted in {source_id}" ) return { "source_id": source_id, "source_audio_key": "+".join(voice.source_key for voice in voices), "manifest_row_id": row_ids[0], "turn_count": len(session), "turn_row_ids": row_ids, "turn_motion_names": [record["filename"] for record in session], "turn_audio_keys": [voice.source_key for voice in voices], "session_voice_name": session_voice_name, "session_voice_variant": session_voice_variant, "interaction_mode": interaction_mode, # Exact microphone ledger for independent VQA overlays. The # microphone has a 0.2 s prefix before the action frame clock. "microphone_speech_intervals": [ { "start_seconds": 0.2 + int(payload["speech_start_frame"]) / self.motion_fps, "end_seconds": 0.2 + int(payload["speech_start_frame"]) / self.motion_fps + int(payload["assignment"].speech_samples) / self.audio_sampling_rate, } for payload in turn_audio ], "turn_listen_start_frames": [ int(payload["listen_start_frame"]) for payload in normal_turn_audio ], "turn_speech_start_frames": [ int(payload["speech_start_frame"]) for payload in normal_turn_audio ], "turn_speech_onset_samples": speech_onset_samples, "turn_listen_reaction_frames": listen_reaction_frames, "turn_listen_reaction_idle_motion_names": ( listen_reaction_idle_motion_names ), "listen_transition_frames": ( self.turn_transition_frames if bool(getattr(self, "causal_listen_timing", False)) else self.stationary_blend_frames ), "motion_name": "+".join(record["filename"] for record in session), "idle_motion_name": idle_motion_names, "listen_motion_name": listen_motion_names, "task_transition_frames": task_transition_frames, "recovery_transition_frames": recovery_transition_frames, "inter_turn_idle_steps": inter_turn_idle_steps, "interruption_start_frames": interruption_start_frames, "interruption_original_frames": interruption_original_frames, "interruption_remaining_frames": interruption_remaining_frames, "interruption_terminal_hold_frames": ( interruption_terminal_hold_frames ), "control_command_events": control_command_events, "backchannel_events": backchannel_events, "backchannel_turn_index": session_backchannel_turn_index, "disfluency_events": disfluency_events, "disfluency_turn_index": session_disfluency_turn_index, "disfluency_subtype": session_disfluency_subtype, "control_turn_index": session_control_turn_index, "control_has_followup": session_control_has_followup, "terminal_idle_steps": int(terminal_idle_steps), "white_noise_start_index": background_start, "background_key": background_key, "background_listen_start_index": ( background_start + int(normal_turn_audio[0]["listen_start_frame"]) // self.action_exec_horizon ), "idle_steps": int( np.count_nonzero(semantic_phase_labels == IDLE_TOKEN_ID) ), "listen_steps": int(listen_steps), "motion_steps": int( np.count_nonzero(semantic_phase_labels == MOTION_TOKEN_ID) ), "semantic_steps": semantic_window_count, "action_idle_steps": action_phase_counts["idle"], "action_listen_steps": action_phase_counts["listen"], "action_motion_steps": action_phase_counts["motion"], "action_steps": len(steps), "steps": steps, } def _build_single_source(self, index: int) -> dict[str, Any]: record = dict(self.records[int(index)]) # ``getattr`` keeps lightweight unit fixtures and the retained legacy # cached-hidden compatibility path valid; formal HFDv2 instances set # this explicitly in ``__init__``. online_audio = bool(getattr(self, "online_audio", False)) motion = self._load_motion(record["motion_dir"]) if motion.shape[0] <= 0: raise RuntimeError(f"Empty motion: {record['motion_dir']}") record["actual_frames"] = int(motion.shape[0]) task_start_pose = motion[0].copy() stationary_anchor = self._sample_stationary_anchor(task_start_pose) task_sequence, task_transition_frames = self._build_task_sequence( motion, stationary_anchor=stationary_anchor, ) voices = self._record_voices(record) voice = self._select_voice(voices, row_id=int(record["row_id"])) source_id = f"row{record['row_id']:06d}::{voice.source_key}" steps: list[dict[str, Any]] = [] idle_step_count = random.randint(0, self.initial_idle_max_steps) listen_step_count = voice.window_count continuous_background = bool( getattr(self, "continuous_background", False) ) assignment = ( self.background_assignments[voice.source_key] if continuous_background else None ) noise_start_index = ( int(assignment.listen_start_index) if assignment is not None else random.randrange(len(self.white_noise_hidden)) ) # One dense row per current state, with the v26 validity rule above. motion_action_step_count = max( int(task_sequence.shape[0]) - self.prefix_frames - 1, 0, ) if self.max_motion_steps > 0: motion_action_step_count = min( motion_action_step_count, self.max_motion_steps * self.action_exec_horizon, ) motion_step_count = int( math.ceil(motion_action_step_count / self.action_exec_horizon) ) if motion_step_count <= 0: raise RuntimeError( "Task motion is too short for one complete 30-frame RTC target: " f"source={source_id} frames={task_sequence.shape[0]}" ) if ( assignment is not None and motion_step_count > assignment.max_motion_steps ): raise RuntimeError( "Motion is longer than its continuous-background reservation: " f"source={voice.source_key} actual_steps={motion_step_count} " f"reserved={assignment.max_motion_steps}" ) if assignment is not None: if online_audio: idle_background_inputs = ( self._load_background_waveform_window_block( assignment.background_key, assignment.listen_start_index - idle_step_count, idle_step_count, ) ) motion_background_inputs = ( self._load_background_waveform_window_block( assignment.background_key, assignment.listen_start_index + listen_step_count, motion_step_count, ) ) else: idle_background_inputs = self._load_background_window_block( assignment.background_key, assignment.listen_start_index - idle_step_count, idle_step_count, ) motion_background_inputs = self._load_background_window_block( assignment.background_key, assignment.listen_start_index + listen_step_count, motion_step_count, ) else: idle_background_inputs = () motion_background_inputs = () if online_audio: if assignment is None: raise RuntimeError( "Dense online audio requires a continuous background " f"assignment: {voice.source_key}" ) voice_inputs: tuple[torch.Tensor | np.ndarray, ...] = ( self._load_mixed_waveform_window_block( voice, assignment, ) ) else: voice_inputs = self._load_voice_hidden(voice) if len(voice_inputs) != listen_step_count: raise RuntimeError( f"Semantic window count mismatch for {voice.source_key}: " f"{len(voice_inputs)} != {listen_step_count}" ) total_stream_steps = ( idle_step_count + listen_step_count + motion_step_count ) max_stream_steps = int(getattr(self, "max_stream_steps", 0)) if max_stream_steps > 0 and total_stream_steps > max_stream_steps: raise RuntimeError( "Source exceeds the configured complete-KV context guard: " f"source={source_id} windows={total_stream_steps} " f"limit={max_stream_steps}. The source was expected to " "be removed by the worst-case construction filter." ) idle_sequence, idle_motion_name = self._sample_stationary_sequence( "idle", frame_count=idle_step_count * self.action_exec_horizon, anchor_pose=stationary_anchor, ) listen_sequence, listen_motion_name = self._sample_stationary_sequence( "listen", frame_count=listen_step_count * self.action_exec_horizon, anchor_pose=stationary_anchor, ) if bool(getattr(self, "stationary_listen_hold", False)) and len(listen_sequence): task_sequence, task_transition_frames = self._build_task_sequence( motion, stationary_anchor=listen_sequence[-1], ) pretask_sequence = np.concatenate( [idle_sequence, listen_sequence], axis=0, ) full_sequence = np.concatenate( [pretask_sequence, task_sequence], axis=0, ) timeline_control_labels = np.concatenate( [ np.full( len(idle_sequence), IDLE_TOKEN_ID, dtype=np.int64, ), np.full( len(listen_sequence), LISTEN_TOKEN_ID, dtype=np.int64, ), np.full( len(task_sequence), MOTION_TOKEN_ID, dtype=np.int64, ), ] ) if len(timeline_control_labels) != len(full_sequence): raise RuntimeError( "Full-duplex frame/token timeline length mismatch: " f"tokens={len(timeline_control_labels)} " f"motion={len(full_sequence)}" ) task_original_frames = self._task_original_frame_indices( sequence_frames=int(task_sequence.shape[0]), motion_frames=int(motion.shape[0]), transition_frames=task_transition_frames, ) timeline_original_frames = np.concatenate( [ np.zeros(len(pretask_sequence), dtype=np.int64), task_original_frames, ] ) action_phase_counts = {"idle": 0, "listen": 0, "motion": 0} semantic_window_index = 0 def append_dense_group( *, phase: str, semantic_input: torch.Tensor | np.ndarray, target_base: int, semantic_frame_id: int, background_index: int, ) -> None: nonlocal semantic_window_index # ``target_base`` indexes the current state. The last allowed # state leaves one real target after a maximal RTC prefix. final_start = len(full_sequence) - self.prefix_frames - 2 stop = min( int(target_base) + self.action_exec_horizon, final_start + 1, ) if stop <= int(target_base): raise RuntimeError( "Semantic window has no complete dense RTC action target: " f"source={source_id} phase={phase} base={target_base} " f"timeline={len(full_sequence)}" ) for target_start in range( int(target_base), stop, int(getattr(self, "action_sample_step_frames", 1)), ): action_substep = target_start - int(target_base) update_semantic_cache = action_substep == 0 ( actions, action_mask, state, action_reference_state, smoothness_context_actions, smoothness_context_reference_state, ) = self._stream_action_rows(full_sequence, start=target_start) semantic_kwargs: dict[str, Any] = {} if update_semantic_cache: semantic_kwargs = ( {"audio_waveform": semantic_input} if online_audio else {"hidden_states": semantic_input} ) action_indices = np.arange( int(target_start) + 1, int(target_start) + 1 + self.action_chunk_size, dtype=np.int64, ) action_control_labels = timeline_control_labels[ np.clip(action_indices, 0, len(timeline_control_labels) - 1) ] progress_indices = timeline_original_frames[ np.clip(action_indices, 0, len(timeline_original_frames) - 1) ] step = self._raw_step( **semantic_kwargs, actions=actions, action_mask=action_mask, state=state, action_reference_state=action_reference_state, smoothness_context_actions=smoothness_context_actions, smoothness_context_reference_state=( smoothness_context_reference_state ), action_control_labels=action_control_labels, future_progress=_progress_rows( progress_indices, int(motion.shape[0]), ), control_progress=( _progress_rows( [int(timeline_original_frames[target_start - 1])], int(motion.shape[0]), )[0] if target_start > 0 and int(timeline_control_labels[target_start - 1]) == MOTION_TOKEN_ID else self._stationary_progress_rows(1)[0] ), previous_control_label=( int(timeline_control_labels[target_start - 1]) if target_start > 0 else 3 ), phase=phase, source_id=source_id, step_index=len(steps), frame_id=( int( timeline_original_frames[ min(int(target_start), len(timeline_original_frames) - 1) ] ) if phase == "motion" else int(target_start) ), semantic_frame_id=int(semantic_frame_id), semantic_window_index=semantic_window_index, action_substep=action_substep, record=record, instruction_text=instruction_text_for_voice(record, voice), update_semantic_cache=update_semantic_cache, control_transition=bool( update_semantic_cache and int(target_start) > 0 and timeline_control_labels[int(target_start)] != timeline_control_labels[int(target_start) - 1] ), ) step["background_window_index"] = int(background_index) steps.append(step) action_phase_counts[phase] += 1 semantic_window_index += 1 for idle_step in range(idle_step_count): target_base = idle_step * self.action_exec_horizon if assignment is not None: background_index = ( assignment.listen_start_index - idle_step_count + idle_step ) semantic_input = idle_background_inputs[idle_step] else: background_index = ( noise_start_index + idle_step ) % len(self.white_noise_hidden) semantic_input = self.white_noise_hidden[background_index] append_dense_group( phase="idle", semantic_input=semantic_input, target_base=target_base, semantic_frame_id=target_base, background_index=background_index, ) for clip_index, semantic_input in enumerate(voice_inputs): semantic_frame_id = ( voice.clips[clip_index].frame_id if ( voice.waveform_path is None and voice.packed_path is None ) else clip_index * self.audio_sample_step_frames ) target_base = ( idle_step_count + clip_index ) * self.action_exec_horizon background_index = noise_start_index + clip_index append_dense_group( phase="listen", semantic_input=semantic_input, target_base=target_base, semantic_frame_id=semantic_frame_id, background_index=background_index, ) for motion_step in range(motion_step_count): target_base = ( len(pretask_sequence) + motion_step * self.action_exec_horizon ) if assignment is not None: background_index = ( assignment.listen_start_index + listen_step_count + motion_step ) semantic_input = motion_background_inputs[motion_step] else: background_index = ( noise_start_index + idle_step_count + motion_step ) % len(self.white_noise_hidden) semantic_input = self.white_noise_hidden[background_index] append_dense_group( phase="motion", semantic_input=semantic_input, target_base=target_base, semantic_frame_id=motion_step * self.action_exec_horizon, background_index=background_index, ) duration_targets = [ step for step in steps if bool(step["motion_duration_target_mask"]) ] if len(duration_targets) != 1: raise RuntimeError( "Each single-turn source must expose exactly one causal " "motion-duration target before transforms: " f"source={source_id} actual={len(duration_targets)}" ) text_targets = [ step for step in steps if bool(step["latest_instruction_target_mask"]) ] if len(text_targets) != 1 or text_targets[0][ "latest_instruction_text" ] != instruction_text_for_voice(record, voice): raise RuntimeError( "Each single-turn source must expose exactly one exact, " f"voice-specific text target: source={source_id}" ) return { "source_id": source_id, "source_audio_key": voice.source_key, "manifest_row_id": int(record["row_id"]), "motion_name": record["filename"], "idle_motion_name": idle_motion_name, "listen_motion_name": listen_motion_name, "task_transition_frames": task_transition_frames, "white_noise_start_index": noise_start_index, "background_key": ( assignment.background_key if assignment is not None else "legacy-cycle" ), "background_listen_start_index": noise_start_index, "idle_steps": idle_step_count, "listen_steps": listen_step_count, "motion_steps": motion_step_count, "semantic_steps": ( idle_step_count + listen_step_count + motion_step_count ), "action_idle_steps": action_phase_counts["idle"], "action_listen_steps": action_phase_counts["listen"], "action_motion_steps": action_phase_counts["motion"], "action_steps": len(steps), "steps": steps, } class FullDuplexTransformedDataset(torch.utils.data.Dataset): """Apply the existing SEED TextOp/normalisation path step by step.""" _STREAM_KEYS = ( "source_id", "source_audio_key", "phase", "control_label", "control_token", "control_transition", "action_control_labels", "update_semantic_cache", "step_index", "frame_id", "semantic_frame_id", "semantic_window_index", "action_substep", "turn_index", "interaction_mode", # SeedPosttrainRepackTransform deliberately builds a compact new # dictionary. Duration supervision is stream metadata, so restore it # explicitly just like control labels instead of silently losing every # target before the trainer sees it. "motion_duration_target_seconds", "motion_duration_target_mask", "latest_instruction_text", "latest_instruction_target_mask", "instruction_memory_text", "instruction_memory_slot", "instruction_memory_target_mask", # Dimensionless causal progress and the discrete previous-token ID # bypass the floating-point action/state transforms and are restored # verbatim below. "control_progress", "previous_control_label", ) def __init__(self, raw_dataset: FullDuplexAudioCommandDataset, transform, transform_kwargs=None): self.raw_dataset = raw_dataset self.transform = transform self.transform_kwargs = dict(transform_kwargs or {}) def __len__(self) -> int: return len(self.raw_dataset) def estimated_stream_steps(self, index: int) -> int: return self.raw_dataset.estimated_stream_steps(index) def source_modality(self, index: int) -> str: method = getattr(self.raw_dataset, "source_modality", None) return method(index) if method is not None else "audio_action" def __getitem__(self, index: int) -> dict[str, Any]: source = self.raw_dataset[index] transformed_steps = [] for raw_step in source["steps"]: if bool(raw_step["update_semantic_cache"]): transformed = self.transform( dict(raw_step), **self.transform_kwargs, ) else: # Repack and normalize every dense action target, but do not # run the expensive Qwen audio processor for the 19 samples # that deliberately reuse the preceding semantic condition. transformed = self.transform.repack( dict(raw_step), **self.transform_kwargs, ) transformed = self.transform.field( transformed, **self.transform_kwargs, ) for key in self._STREAM_KEYS: if key == "control_transition": transformed[key] = bool( raw_step.get("control_transition", False) ) elif key == "action_control_labels": transformed[key] = raw_step.get( "action_control_labels", np.full( len(raw_step["actions"]), int(raw_step["control_label"]), dtype=np.int64, ), ) elif key == "turn_index": transformed[key] = int(raw_step.get("turn_index", 0)) elif key == "interaction_mode": transformed[key] = str( raw_step.get("interaction_mode", "turn-taking") ) elif key == "motion_duration_target_seconds": transformed[key] = float(raw_step[key]) elif key == "motion_duration_target_mask": transformed[key] = bool(raw_step[key]) elif key == "latest_instruction_text": transformed[key] = str( raw_step.get("latest_instruction_text", "") ) elif key == "latest_instruction_target_mask": transformed[key] = bool( raw_step.get("latest_instruction_target_mask", False) ) elif key == "instruction_memory_text": transformed[key] = str( raw_step.get("instruction_memory_text", "") ) elif key == "instruction_memory_slot": transformed[key] = str( raw_step.get("instruction_memory_slot", "") ) elif key == "instruction_memory_target_mask": transformed[key] = bool( raw_step.get( "instruction_memory_target_mask", False, ) ) elif key == "control_progress": transformed[key] = np.asarray( raw_step.get( "control_progress", raw_step.get( "progress", np.asarray( [0.0, 1.0, 0.0, 0.0, 1.0], dtype=np.float32, ), ), ), dtype=np.float32, ).reshape(5) elif key == "previous_control_label": transformed[key] = int( raw_step.get("previous_control_label", 3) ) else: transformed[key] = raw_step[key] transformed_steps.append(transformed) raw_duration_targets = sum( bool(step["motion_duration_target_mask"]) for step in source["steps"] ) transformed_duration_targets = sum( bool(step["motion_duration_target_mask"]) for step in transformed_steps ) if transformed_duration_targets != raw_duration_targets: raise RuntimeError( "Motion-duration supervision changed during data transforms: " f"source={source['source_id']} raw={raw_duration_targets} " f"transformed={transformed_duration_targets}" ) raw_text_targets = sum( bool(step.get("latest_instruction_target_mask", False)) for step in source["steps"] ) transformed_text_targets = sum( bool(step["latest_instruction_target_mask"]) for step in transformed_steps ) if transformed_text_targets != raw_text_targets: raise RuntimeError( "Latest-instruction text supervision changed during data " f"transforms: source={source['source_id']} " f"raw={raw_text_targets} transformed={transformed_text_targets}" ) raw_memory_targets = sum( bool(step.get("instruction_memory_target_mask", False)) for step in source["steps"] ) transformed_memory_targets = sum( bool(step["instruction_memory_target_mask"]) for step in transformed_steps ) if transformed_memory_targets != raw_memory_targets: raise RuntimeError( "Historical-instruction supervision changed during data " f"transforms: source={source['source_id']} " f"raw={raw_memory_targets} " f"transformed={transformed_memory_targets}" ) result = {**source, "steps": transformed_steps} if "multimodal_overlay" in result: from psi.data.seed.live_multimodal_overlay import attach_live_packets return attach_live_packets(result) return result
from __future__ import annotations
import hashlib
import json
import math
import random
import re
import sqlite3
import wave
from collections import OrderedDict
from dataclasses import dataclass
from pathlib import Path
from typing import Any, Iterable
import numpy as np
import pandas as pd
import torch
from safetensors import safe_open
from safetensors.torch import load_file
from scipy.signal import resample_poly
from psi.data.seed.omni_hidden_cache import (
load_hidden_states_slice,
load_hidden_states_source,
)
from psi.data.seed.motion_repr import (
estimate_two_foot_contacts,
quat_mul_wxyz,
quat_to_yaw_wxyz,
rpy_to_quat_wxyz,
wrap_angle,
)
from psi.data.seed.source_split import motion_source_id, select_source_row_ids
from psi.data.seed.storage_paths import relocate_hfd_storage_path
LISTEN_TOKEN_ID = 0
MOTION_TOKEN_ID = 1
IDLE_TOKEN_ID = 2
CONTROL_TOKEN_NAMES = ("</listen>", "</motion>", "</idle>")
CANONICAL_MOTION_MANIFEST_FORMAT = "hfd_canonical_motion_per_instruction_v1"
HESITATION_PAUSE_MANIFEST_FORMAT = "hfd_hesitation_pause_tts_v1"
DISFLUENCY_UNCERTAINTY_ROLES = frozenset(
{
"background-only-pause",
"hesitation-background-before",
"hesitation-filler",
"hesitation-background-after",
"repair-background-before",
"repair-cue",
"repair-background-after",
"repair-implicit-background",
}
)
# The continuous-audio corpus is exported at the deployment cadence: one
# 0.6-second window every 20 motion frames (0.4 seconds).
EXPORTED_AUDIO_STRIDE_FRAMES = 20
# These are deliberately exact metadata labels, not fuzzy keyword matches.
# SEED contains unsafe names such as idle_handstand, injured_*_idle and
# crouch_*_idle. The selected labels were manually audited against every
# natural-language description and kinematically checked for root drift.
LISTEN_MOTION_CONTENT_NAMES = frozenset({"listening_R"})
IDLE_MOTION_CONTENT_NAMES = frozenset({"neutral_idle"})
@dataclass(frozen=True)
class HiddenSlice:
frame_id: int
shard_path: Path
offset: int
length: int
@dataclass(frozen=True)
class AudioSource:
source_key: str
clips: tuple[HiddenSlice, ...]
packed_path: Path | None = None
packed_count: int = 0
# HFDv2 reads the exact continuous mixed waveform online. This field is
# mutually exclusive with the two cached-hidden layouts above.
waveform_path: Path | None = None
waveform_count: int = 0
@property
def window_count(self) -> int:
if self.waveform_path is not None:
return int(self.waveform_count)
return (
int(self.packed_count)
if self.packed_path is not None
else len(self.clips)
)
def instruction_text_for_voice(
record: dict[str, Any],
voice: AudioSource,