from __future__ import annotations from typing import Any, Literal from psi.config.config import DataConfig class FullDuplexAudioCommandDataConfig(DataConfig): # Audited microphones + latest motion builder, before the visual overlay. # Empty defaults preserve historical run-config reconstruction/resume. third_party_root: str = "" third_party_train_selection: str = "" third_party_val_selection: str = "" # Formal speaking-while-acting extension. The repeat factor controls the # added SWA share without deleting any of the established audio sources. swa_visual_qa_enabled: bool = False swa_source_plan: str = "" swa_train_repeat: int = 1 # Opt-in unified input overlay. Unlike the legacy concat, this preserves # all five original task pools and adds no independent SWA body task. multimodal_overlay_manifests: tuple[str, ...] = () multimodal_overlay_qa_ratio: float = 0.75 multimodal_overlay_visual_only_ratio: float = 0.10 multimodal_overlay_strict_voice: bool = True multimodal_visual_input_mode: Literal["cached", "raw"] = "cached" # HFDv2 formal training consumes these waveforms directly and never reads # the offline Qwen hidden-state cache below. online_audio: bool = True continuous_audio_root: str = ( "/cpfs01/thscc/sharestorage/AIGC-06-01/Hithink_VideoGen_Group/user_workspace/yetinghong/videogen/" "hfd_audio_command_continuous_office_v1" ) hidden_root: str = ( "/cpfs01/thscc/sharestorage/AIGC-06-01/Hithink_VideoGen_Group/user_workspace/yetinghong/videogen/" "hfd_audio_command_continuous_office_v1/mixed_hidden" ) metadata_jsonl: str = "/cpfs01/thscc/sharestorage/AIGC-06-01/Hithink_VideoGen_Group/user_workspace/buwendong/user/code/qwen3-tts/metadata.jsonl" motion_metadata_csv: str = ( "/cpfs01/thscc/sharestorage/AIGC-06-01/Hithink_VideoGen_Group/user_workspace/yetinghong/code/Psi0/dataset/seed_all/metadata/seed_metadata_v004.csv" ) motion_root: str = "/cpfs01/thscc/sharestorage/AIGC-06-01/Hithink_VideoGen_Group/user_workspace/yetinghong/code/Psi0/dataset/seed_all" # Semantics-v3 binds every spoken source to one uninterrupted background # timeline. The speech windows above contain waveform-level speech + # background, while idle/motion windows below read adjacent pure-background # positions from that exact same timeline. white_noise_hidden_dir: str = "" background_hidden_root: str = ( "/cpfs01/thscc/sharestorage/AIGC-06-01/Hithink_VideoGen_Group/user_workspace/yetinghong/videogen/" "hfd_audio_command_continuous_office_v1/background_hidden" ) background_assignment_path: str = ( "/cpfs01/thscc/sharestorage/AIGC-06-01/Hithink_VideoGen_Group/user_workspace/yetinghong/videogen/" "hfd_audio_command_continuous_office_v1/manifests/" "source_background_assignments.jsonl" ) background_source_name: str = "hfd_continuous_office_background" background_task_name: str = "audio_command_text" val_ratio: float = 0.1 split_seed: int = 42 split_source_namespace: str = "seed" source_name: str = "qwen3_tts_seed_audio_command" task_name: str = "audio_command_text" action_exec_horizon: int = 20 motion_fps: int = 50 # SEED metadata records the source motion duration at 120 Hz, while the # exported G1 CSVs consumed by HFD are 50 Hz. motion_metadata_fps: int = 120 audio_sampling_rate: int = 16000 audio_window_seconds: float = 0.6 # Qwen/KV follows deployment cadence, while action targets are sampled at # every 50 Hz state to reproduce v26's dense RTC supervision. audio_sample_step_frames: int = 20 action_sample_step_frames: int = 1 motion_cache_size: int = 256 # Bind every motion record to a stable rendition. This makes the sampler's # predicted length equal to the item that workers actually build and still # distributes speaker/language variants across the full corpus. fixed_voice_per_record: bool = True # Reject rare broken TTS files that are many times longer than the other # renditions of the same short command. Zero disables the absolute cap. max_voice_windows: int = 30 max_voice_median_ratio: float = 3.0 # Validation must rebuild the exact same source at every checkpoint. deterministic_eval: bool = True # Non-zero values are explicit smoke/debug limits. Production defaults # retain every source and every motion step. limit_sources: int = 0 max_motion_steps: int = 0 # The v13 entry point overrides this to seven background-only positions # (0..2.8 s on the 0.4 s Qwen grid), covering the requested 0..3 s range # without placing an unobservable control boundary inside one window. # Five remains the historical single-turn/raw-corpus-compatible default. initial_idle_max_steps: int = 5 # Natural idle/listen targets smoothly leave and return to the paired task # motion's initial pose over this many 50 Hz frames. stationary_blend_frames: int = 20 # Version 15 rigidly re-expresses idle/listen candidates in the current # robot heading. Historical checkpoints retain the previous XY-only path. stationary_heading_alignment: bool = False # Production v22 performs a data-driven whole-body audit of the neutral # idle pool, then chooses the accepted standing pose closest to the current # robot state. Historical configs keep this disabled so their targets can # still be reconstructed exactly. safe_stationary_anchors: bool = False # Semantics v24 keeps the sampled idle clip's *small motion residual* but # expresses it around the current safe anchor. This prevents two static # standing poses with different absolute arm configurations from creating # a fast synthetic arm swing when they are blended together. stationary_idle_residual_retargeting: bool = False # Opt-in target revision: enter a recorded listening pose and hold it; # the following task blends from the actual last listen frame. False # preserves old checkpoints and immutable ablation fixtures exactly. stationary_listen_hold: bool = False # Blend the current safe standing state into the *moving* task prefix. # This avoids holding a locomotion clip's arbitrary first (often # single-support) frame for one second before suddenly restoring velocity. dynamic_task_entry_blend: bool = False # Versions 13/14 group primitive commands into one uninterrupted interaction. # The complete item owns one audio background and one Qwen KV lifetime. multi_turn_sessions: bool = False session_min_turns: int = 2 session_max_turns: int = 4 # Version 17 creates static train-session variants. Every primitive is # used once per variant with one shared speaker/language identity across # all turns. Validation always keeps one deterministic variant. # Zero retains the historical per-record voice policy exactly. session_voice_variants: int = 0 # Version 20 uses the two existing train variants as orthogonal behavior # variants: one complete turn-taking session and one interruption session. # Every primitive therefore contributes once to each behavior (exact 1:1 # primitive supervision); interrupted timelines are naturally shorter. # Validation deterministically builds both modes. interruption_sessions: bool = False interruption_ratio: float = 0.5 interruption_min_start_seconds: float = 2.0 interruption_max_start_seconds: float = 6.0 interruption_min_remaining_seconds: float = 1.5 # Version 21 keeps the complete two-variant turn-taking-v1 pool intact and # constructs every additional behaviour as an independent pool. Integer # variant counts are direct, auditable data weights: 2:2 is exactly 1:1. independent_task_pools: bool = False turn_taking_session_variants: int = 2 command_interruption_session_variants: int = 2 emergency_stop_session_variants: int = 1 safe_hold_session_variants: int = 1 # Backchannel is an independent hard-negative pool: short same-speaker # acknowledgements overlap an active motion without changing its action, # progress, RTC prefix or target. backchannel_session_variants: int = 0 # Semantics v23 adds two independent listen-side hard cases without # changing the established turn-taking/interruption/Stop/backchannel pools. # ``pause`` inserts background-only time inside one unfinished command; # ``hesitation`` inserts a filled pause or an incomplete false start plus a # repair cue. Both keep until the final intent is complete. hesitation_session_variants: int = 0 pause_session_variants: int = 0 # Optional final-session target applied after all context/background and # task-eligibility filtering. Zero preserves v21 integer-pool behavior; # production v22 balances the actual sampled sessions, not merely raw # primitive uses. interruption_task_session_ratio: float = 0.0 # Legacy v21 runs leave these at zero/one and therefore reproduce their # original integer-variant weighting. The v22 launcher keeps hold at zero, # assigns Stop and backchannel their requested final-session shares, and # retains follow-up support for the selected Stop sessions. control_task_session_ratio: float = 0.0 backchannel_task_session_ratio: float = 0.0 # These are final *stream-window* shares. v23 first reconstructs the exact # v22 selected session set, then appends enough new sessions to target 12% # hesitation and 8% pause by estimated optimizer work. hesitation_task_window_ratio: float = 0.0 pause_task_window_ratio: float = 0.0 control_followup_ratio: float = 0.0 control_post_idle_min_steps: int = 1 # Zero preserves historical behavior (the maximum follows # ``initial_idle_max_steps``). Production Stop-persistence training sets # this to 20 Qwen decisions, i.e. 8 seconds at the 0.4 s cadence. control_post_idle_max_steps: int = 0 # Historical checkpoints sampled the explicit range uniformly. The # balanced policy keeps the same 1.2--8 s support but makes ordinary short # pauses common and long persistence examples deliberately rare, avoiding # a large one-sided shift toward . control_post_idle_sampling: str = "legacy_uniform" # Emergency stop has a stricter timing contract than ordinary command # interruption. The command must arrive while the counterfactual source # motion is still active and must leave a real moving tail after the full # utterance has become observable. Historical run configs keep the old # values and disabled activity selector for exact checkpoint replay. control_min_start_seconds: float = 2.0 control_max_start_seconds: float = 6.0 control_min_remaining_seconds: float = 0.4 control_activity_aware_timing: bool = False # A small audited subset of the Chinese TTS corpus mistranslates English # ``retro`` as spatial ``backward``. When enabled, only those contradictory # Chinese renditions are removed; the matching English voices and motions # remain available. filter_known_tts_translation_mismatches: bool = False # Optional data ablation: a generated, audited manifest selects exactly # one motion trajectory for each exact English/Chinese instruction pair. # Empty keeps the full many-to-one corpus and is the formal default. canonical_motion_manifest: str = "" control_command_manifest: str = ( "/cpfs01/thscc/sharestorage/AIGC-06-01/Hithink_VideoGen_Group/user_workspace/yetinghong/videogen/" "hfd_control_commands_v1/manifest.jsonl" ) backchannel_command_manifest: str = ( "/cpfs01/thscc/sharestorage/AIGC-06-01/Hithink_VideoGen_Group/user_workspace/yetinghong/videogen/" "hfd_backchannel_commands_v2_short/manifest.jsonl" ) hesitation_pause_manifest: str = ( "/cpfs01/thscc/sharestorage/AIGC-06-01/Hithink_VideoGen_Group/user_workspace/yetinghong/videogen/" "hfd_hesitation_pause_v1/manifest.jsonl" ) # New pools cap duplicate motion rows for one exact bilingual instruction. # Existing task pools remain byte-for-byte unchanged. Selection is # deterministic and duration-stratified rather than first-N. new_task_max_rows_per_instruction_train: int = 32 new_task_max_rows_per_instruction_val: int = 8 # Filled hesitation versus self-repair. Silent intra-command pauses are a # separate task and therefore do not consume this split. hesitation_filled_ratio: float = 0.60 # Among self-repair hesitation sessions, omit an explicit ``I mean`` / # ``改成`` cue for this fraction. The false-start prefix is followed by a # pause and the complete final command, matching natural cue-less restarts. self_repair_implicit_ratio: float = 0.0 backchannel_min_start_seconds: float = 1.2 backchannel_max_start_seconds: float = 6.0 backchannel_min_remaining_seconds: float = 1.2 safe_hold_stability_window_frames: int = 20 # Motion -> standing and standing -> the next task use a slower one-second # transition. Version 13 retains the original 20-frame first-task entry; # causal-timing version 14 uses this transition for every task entry and # the stationary/listen envelope as well. turn_transition_frames: int = 50 # Version 14 starts the clean speech waveform first, keeps the target in # idle through the first Qwen boundary containing real voice onset, keeps # one explicit response interval, and then starts . Historical # runs keep this false so their checkpoint contract stays reproducible. causal_listen_timing: bool = False # Expected leading-silence budget recorded in the train/infer contract. # Longer valid TTS silence is measured and retained; source-duration bounds # are used for conservative sampler/context estimates. Three steps are # 1.2 seconds. max_speech_onset_steps: int = 3 # After the Qwen boundary that first contains sustained voice energy, keep # one more 0.4 s idle response interval before emitting . listen_reaction_steps: int = 1 # Qwen2.5-Omni-3B has a 32768-token context. A first 0.6 s window uses 82 # tokens and every later one uses 17, so 1923 complete windows fit while # preserving every source-local KV entry. max_stream_steps: int = 1923 # Drop only the deterministic longest tail after the final task mixture is # formed. Zero preserves historical data; production v22 uses 0.001. long_session_drop_ratio: float = 0.0 def __call__(self, split: str = "train", transform_kwargs=None, **kwargs) -> Any: from psi.data.seed.full_duplex_audio_command_dataset import ( FullDuplexAudioCommandDataset, FullDuplexTransformedDataset, ) repack = self.transform.repack action_chunk_size = int(getattr(repack, "action_chunk_size", 30)) prefix_frames = int(getattr(repack, "rtc_prefix_frames", 0)) expected_prefix = action_chunk_size - int(self.action_exec_horizon) if prefix_frames != expected_prefix: raise ValueError( "Full-duplex data/model RTC mismatch: " f"repack_prefix={prefix_frames}, expected={expected_prefix}" ) raw_dataset = FullDuplexAudioCommandDataset( hidden_root=self.hidden_root, metadata_jsonl=self.metadata_jsonl, motion_metadata_csv=self.motion_metadata_csv, motion_root=self.motion_root, white_noise_hidden_dir=self.white_noise_hidden_dir, background_hidden_root=self.background_hidden_root, background_assignment_path=self.background_assignment_path, continuous_audio_root=self.continuous_audio_root, online_audio=self.online_audio, background_source_name=self.background_source_name, background_task_name=self.background_task_name, split=split, val_ratio=self.val_ratio, split_seed=self.split_seed, split_source_namespace=self.split_source_namespace, source_name=self.source_name, task_name=self.task_name, action_chunk_size=action_chunk_size, action_exec_horizon=self.action_exec_horizon, motion_fps=self.motion_fps, motion_metadata_fps=self.motion_metadata_fps, audio_sampling_rate=self.audio_sampling_rate, audio_window_seconds=self.audio_window_seconds, audio_sample_step_frames=self.audio_sample_step_frames, action_sample_step_frames=self.action_sample_step_frames, motion_cache_size=self.motion_cache_size, fixed_voice_per_record=self.fixed_voice_per_record, max_voice_windows=self.max_voice_windows, max_voice_median_ratio=self.max_voice_median_ratio, deterministic_eval=self.deterministic_eval, limit_sources=self.limit_sources, max_motion_steps=self.max_motion_steps, initial_idle_max_steps=self.initial_idle_max_steps, stationary_blend_frames=self.stationary_blend_frames, stationary_heading_alignment=self.stationary_heading_alignment, safe_stationary_anchors=self.safe_stationary_anchors, stationary_idle_residual_retargeting=( self.stationary_idle_residual_retargeting ), stationary_listen_hold=self.stationary_listen_hold, dynamic_task_entry_blend=self.dynamic_task_entry_blend, multi_turn_sessions=self.multi_turn_sessions, session_min_turns=self.session_min_turns, session_max_turns=self.session_max_turns, session_voice_variants=self.session_voice_variants, interruption_sessions=self.interruption_sessions, interruption_ratio=self.interruption_ratio, interruption_min_start_seconds=( self.interruption_min_start_seconds ), interruption_max_start_seconds=( self.interruption_max_start_seconds ), interruption_min_remaining_seconds=( self.interruption_min_remaining_seconds ), independent_task_pools=self.independent_task_pools, turn_taking_session_variants=( self.turn_taking_session_variants ), command_interruption_session_variants=( self.command_interruption_session_variants ), emergency_stop_session_variants=( self.emergency_stop_session_variants ), safe_hold_session_variants=self.safe_hold_session_variants, backchannel_session_variants=( self.backchannel_session_variants ), hesitation_session_variants=( self.hesitation_session_variants ), pause_session_variants=self.pause_session_variants, interruption_task_session_ratio=( self.interruption_task_session_ratio ), control_task_session_ratio=self.control_task_session_ratio, backchannel_task_session_ratio=( self.backchannel_task_session_ratio ), hesitation_task_window_ratio=( self.hesitation_task_window_ratio ), pause_task_window_ratio=self.pause_task_window_ratio, control_followup_ratio=self.control_followup_ratio, control_post_idle_min_steps=self.control_post_idle_min_steps, control_post_idle_max_steps=self.control_post_idle_max_steps, control_post_idle_sampling=self.control_post_idle_sampling, control_min_start_seconds=self.control_min_start_seconds, control_max_start_seconds=self.control_max_start_seconds, control_min_remaining_seconds=( self.control_min_remaining_seconds ), control_activity_aware_timing=( self.control_activity_aware_timing ), filter_known_tts_translation_mismatches=( self.filter_known_tts_translation_mismatches ), canonical_motion_manifest=self.canonical_motion_manifest, control_command_manifest=self.control_command_manifest, backchannel_command_manifest=( self.backchannel_command_manifest ), hesitation_pause_manifest=self.hesitation_pause_manifest, new_task_max_rows_per_instruction_train=( self.new_task_max_rows_per_instruction_train ), new_task_max_rows_per_instruction_val=( self.new_task_max_rows_per_instruction_val ), hesitation_filled_ratio=self.hesitation_filled_ratio, self_repair_implicit_ratio=self.self_repair_implicit_ratio, backchannel_min_start_seconds=( self.backchannel_min_start_seconds ), backchannel_max_start_seconds=( self.backchannel_max_start_seconds ), backchannel_min_remaining_seconds=( self.backchannel_min_remaining_seconds ), safe_hold_stability_window_frames=( self.safe_hold_stability_window_frames ), turn_transition_frames=self.turn_transition_frames, causal_listen_timing=self.causal_listen_timing, max_speech_onset_steps=self.max_speech_onset_steps, listen_reaction_steps=self.listen_reaction_steps, max_stream_steps=self.max_stream_steps, long_session_drop_ratio=self.long_session_drop_ratio, ) if self.third_party_root: from psi.data.seed.third_party_dataset import ThirdPartyAugmentedDataset selection = self.third_party_train_selection if split == "train" else self.third_party_val_selection if not selection: raise ValueError("Third-party data requires an explicit split selection") raw_dataset = ThirdPartyAugmentedDataset(raw_dataset, self.third_party_root, selection, split) if self.multimodal_overlay_manifests: if not self.swa_visual_qa_enabled: raise ValueError("Unified overlay requires the AV/independent QA model") from psi.data.seed.live_multimodal_overlay import LiveMultimodalOverlayDataset from psi.data.seed.hfd_multimodal_overlay import MultimodalOverlayPolicy raw_dataset = LiveMultimodalOverlayDataset( raw_dataset, self.multimodal_overlay_manifests, split, policy=MultimodalOverlayPolicy( qa_ratio=self.multimodal_overlay_qa_ratio, visual_only_ratio=self.multimodal_overlay_visual_only_ratio, audio_only_ratio=1. - self.multimodal_overlay_qa_ratio - self.multimodal_overlay_visual_only_ratio), seed=self.split_seed, strict_voice=self.multimodal_overlay_strict_voice, visual_input_mode=self.multimodal_visual_input_mode) audio_dataset = FullDuplexTransformedDataset( raw_dataset, self.transform, transform_kwargs=transform_kwargs, ) if self.multimodal_overlay_manifests or not self.swa_visual_qa_enabled: return audio_dataset if self.transform.repack.root40_actions: raise ValueError("Root40 requires raw-pose unified overlay data, not historical pretransformed 39D SWA bodies") from psi.data.seed.swa_visual_qa_dataset import ( MixedFullDuplexDataset, SwaVisualQaDataset, ) return MixedFullDuplexDataset( audio_dataset, SwaVisualQaDataset( self.swa_source_plan, split, repeat=self.swa_train_repeat, ), )