Download src/psi/config/data_full_duplex_audio_command.py from Tinghong-Ye22/code-HFD: direct link, hf CLI and curl.
- Browser
- Download file 24.3 kB
-
https://huggingface.co/Tinghong-Ye22/code-HFD/resolve/main/src/psi/config/data_full_duplex_audio_command.py
- Command line
-
hf download hf://Tinghong-Ye22/code-HFD/src/psi/config/data_full_duplex_audio_command.py
-
curl -L -o data_full_duplex_audio_command.py https://huggingface.co/Tinghong-Ye22/code-HFD/resolve/main/src/psi/config/data_full_duplex_audio_command.py
24.3 kB
| from __future__ import annotations | |
| from typing import Any, Literal | |
| from psi.config.config import DataConfig | |
| class FullDuplexAudioCommandDataConfig(DataConfig): | |
| # Audited microphones + latest motion builder, before the visual overlay. | |
| # Empty defaults preserve historical run-config reconstruction/resume. | |
| third_party_root: str = "" | |
| third_party_train_selection: str = "" | |
| third_party_val_selection: str = "" | |
| # Formal speaking-while-acting extension. The repeat factor controls the | |
| # added SWA share without deleting any of the established audio sources. | |
| swa_visual_qa_enabled: bool = False | |
| swa_source_plan: str = "" | |
| swa_train_repeat: int = 1 | |
| # Opt-in unified input overlay. Unlike the legacy concat, this preserves | |
| # all five original task pools and adds no independent SWA body task. | |
| multimodal_overlay_manifests: tuple[str, ...] = () | |
| multimodal_overlay_qa_ratio: float = 0.75 | |
| multimodal_overlay_visual_only_ratio: float = 0.10 | |
| multimodal_overlay_strict_voice: bool = True | |
| multimodal_visual_input_mode: Literal["cached", "raw"] = "cached" | |
| # HFDv2 formal training consumes these waveforms directly and never reads | |
| # the offline Qwen hidden-state cache below. | |
| online_audio: bool = True | |
| continuous_audio_root: str = ( | |
| "/cpfs01/thscc/sharestorage/AIGC-06-01/Hithink_VideoGen_Group/user_workspace/yetinghong/videogen/" | |
| "hfd_audio_command_continuous_office_v1" | |
| ) | |
| hidden_root: str = ( | |
| "/cpfs01/thscc/sharestorage/AIGC-06-01/Hithink_VideoGen_Group/user_workspace/yetinghong/videogen/" | |
| "hfd_audio_command_continuous_office_v1/mixed_hidden" | |
| ) | |
| metadata_jsonl: str = "/cpfs01/thscc/sharestorage/AIGC-06-01/Hithink_VideoGen_Group/user_workspace/buwendong/user/code/qwen3-tts/metadata.jsonl" | |
| motion_metadata_csv: str = ( | |
| "/cpfs01/thscc/sharestorage/AIGC-06-01/Hithink_VideoGen_Group/user_workspace/yetinghong/code/Psi0/dataset/seed_all/metadata/seed_metadata_v004.csv" | |
| ) | |
| motion_root: str = "/cpfs01/thscc/sharestorage/AIGC-06-01/Hithink_VideoGen_Group/user_workspace/yetinghong/code/Psi0/dataset/seed_all" | |
| # Semantics-v3 binds every spoken source to one uninterrupted background | |
| # timeline. The speech windows above contain waveform-level speech + | |
| # background, while idle/motion windows below read adjacent pure-background | |
| # positions from that exact same timeline. | |
| white_noise_hidden_dir: str = "" | |
| background_hidden_root: str = ( | |
| "/cpfs01/thscc/sharestorage/AIGC-06-01/Hithink_VideoGen_Group/user_workspace/yetinghong/videogen/" | |
| "hfd_audio_command_continuous_office_v1/background_hidden" | |
| ) | |
| background_assignment_path: str = ( | |
| "/cpfs01/thscc/sharestorage/AIGC-06-01/Hithink_VideoGen_Group/user_workspace/yetinghong/videogen/" | |
| "hfd_audio_command_continuous_office_v1/manifests/" | |
| "source_background_assignments.jsonl" | |
| ) | |
| background_source_name: str = "hfd_continuous_office_background" | |
| background_task_name: str = "audio_command_text" | |
| val_ratio: float = 0.1 | |
| split_seed: int = 42 | |
| split_source_namespace: str = "seed" | |
| source_name: str = "qwen3_tts_seed_audio_command" | |
| task_name: str = "audio_command_text" | |
| action_exec_horizon: int = 20 | |
| motion_fps: int = 50 | |
| # SEED metadata records the source motion duration at 120 Hz, while the | |
| # exported G1 CSVs consumed by HFD are 50 Hz. | |
| motion_metadata_fps: int = 120 | |
| audio_sampling_rate: int = 16000 | |
| audio_window_seconds: float = 0.6 | |
| # Qwen/KV follows deployment cadence, while action targets are sampled at | |
| # every 50 Hz state to reproduce v26's dense RTC supervision. | |
| audio_sample_step_frames: int = 20 | |
| action_sample_step_frames: int = 1 | |
| motion_cache_size: int = 256 | |
| # Bind every motion record to a stable rendition. This makes the sampler's | |
| # predicted length equal to the item that workers actually build and still | |
| # distributes speaker/language variants across the full corpus. | |
| fixed_voice_per_record: bool = True | |
| # Reject rare broken TTS files that are many times longer than the other | |
| # renditions of the same short command. Zero disables the absolute cap. | |
| max_voice_windows: int = 30 | |
| max_voice_median_ratio: float = 3.0 | |
| # Validation must rebuild the exact same source at every checkpoint. | |
| deterministic_eval: bool = True | |
| # Non-zero values are explicit smoke/debug limits. Production defaults | |
| # retain every source and every motion step. | |
| limit_sources: int = 0 | |
| max_motion_steps: int = 0 | |
| # The v13 entry point overrides this to seven background-only positions | |
| # (0..2.8 s on the 0.4 s Qwen grid), covering the requested 0..3 s range | |
| # without placing an unobservable control boundary inside one window. | |
| # Five remains the historical single-turn/raw-corpus-compatible default. | |
| initial_idle_max_steps: int = 5 | |
| # Natural idle/listen targets smoothly leave and return to the paired task | |
| # motion's initial pose over this many 50 Hz frames. | |
| stationary_blend_frames: int = 20 | |
| # Version 15 rigidly re-expresses idle/listen candidates in the current | |
| # robot heading. Historical checkpoints retain the previous XY-only path. | |
| stationary_heading_alignment: bool = False | |
| # Production v22 performs a data-driven whole-body audit of the neutral | |
| # idle pool, then chooses the accepted standing pose closest to the current | |
| # robot state. Historical configs keep this disabled so their targets can | |
| # still be reconstructed exactly. | |
| safe_stationary_anchors: bool = False | |
| # Semantics v24 keeps the sampled idle clip's *small motion residual* but | |
| # expresses it around the current safe anchor. This prevents two static | |
| # standing poses with different absolute arm configurations from creating | |
| # a fast synthetic arm swing when they are blended together. | |
| stationary_idle_residual_retargeting: bool = False | |
| # Opt-in target revision: enter a recorded listening pose and hold it; | |
| # the following task blends from the actual last listen frame. False | |
| # preserves old checkpoints and immutable ablation fixtures exactly. | |
| stationary_listen_hold: bool = False | |
| # Blend the current safe standing state into the *moving* task prefix. | |
| # This avoids holding a locomotion clip's arbitrary first (often | |
| # single-support) frame for one second before suddenly restoring velocity. | |
| dynamic_task_entry_blend: bool = False | |
| # Versions 13/14 group primitive commands into one uninterrupted interaction. | |
| # The complete item owns one audio background and one Qwen KV lifetime. | |
| multi_turn_sessions: bool = False | |
| session_min_turns: int = 2 | |
| session_max_turns: int = 4 | |
| # Version 17 creates static train-session variants. Every primitive is | |
| # used once per variant with one shared speaker/language identity across | |
| # all turns. Validation always keeps one deterministic variant. | |
| # Zero retains the historical per-record voice policy exactly. | |
| session_voice_variants: int = 0 | |
| # Version 20 uses the two existing train variants as orthogonal behavior | |
| # variants: one complete turn-taking session and one interruption session. | |
| # Every primitive therefore contributes once to each behavior (exact 1:1 | |
| # primitive supervision); interrupted timelines are naturally shorter. | |
| # Validation deterministically builds both modes. | |
| interruption_sessions: bool = False | |
| interruption_ratio: float = 0.5 | |
| interruption_min_start_seconds: float = 2.0 | |
| interruption_max_start_seconds: float = 6.0 | |
| interruption_min_remaining_seconds: float = 1.5 | |
| # Version 21 keeps the complete two-variant turn-taking-v1 pool intact and | |
| # constructs every additional behaviour as an independent pool. Integer | |
| # variant counts are direct, auditable data weights: 2:2 is exactly 1:1. | |
| independent_task_pools: bool = False | |
| turn_taking_session_variants: int = 2 | |
| command_interruption_session_variants: int = 2 | |
| emergency_stop_session_variants: int = 1 | |
| safe_hold_session_variants: int = 1 | |
| # Backchannel is an independent hard-negative pool: short same-speaker | |
| # acknowledgements overlap an active motion without changing its action, | |
| # progress, RTC prefix or </motion> target. | |
| backchannel_session_variants: int = 0 | |
| # Semantics v23 adds two independent listen-side hard cases without | |
| # changing the established turn-taking/interruption/Stop/backchannel pools. | |
| # ``pause`` inserts background-only time inside one unfinished command; | |
| # ``hesitation`` inserts a filled pause or an incomplete false start plus a | |
| # repair cue. Both keep </listen> until the final intent is complete. | |
| hesitation_session_variants: int = 0 | |
| pause_session_variants: int = 0 | |
| # Optional final-session target applied after all context/background and | |
| # task-eligibility filtering. Zero preserves v21 integer-pool behavior; | |
| # production v22 balances the actual sampled sessions, not merely raw | |
| # primitive uses. | |
| interruption_task_session_ratio: float = 0.0 | |
| # Legacy v21 runs leave these at zero/one and therefore reproduce their | |
| # original integer-variant weighting. The v22 launcher keeps hold at zero, | |
| # assigns Stop and backchannel their requested final-session shares, and | |
| # retains follow-up support for the selected Stop sessions. | |
| control_task_session_ratio: float = 0.0 | |
| backchannel_task_session_ratio: float = 0.0 | |
| # These are final *stream-window* shares. v23 first reconstructs the exact | |
| # v22 selected session set, then appends enough new sessions to target 12% | |
| # hesitation and 8% pause by estimated optimizer work. | |
| hesitation_task_window_ratio: float = 0.0 | |
| pause_task_window_ratio: float = 0.0 | |
| control_followup_ratio: float = 0.0 | |
| control_post_idle_min_steps: int = 1 | |
| # Zero preserves historical behavior (the maximum follows | |
| # ``initial_idle_max_steps``). Production Stop-persistence training sets | |
| # this to 20 Qwen decisions, i.e. 8 seconds at the 0.4 s cadence. | |
| control_post_idle_max_steps: int = 0 | |
| # Historical checkpoints sampled the explicit range uniformly. The | |
| # balanced policy keeps the same 1.2--8 s support but makes ordinary short | |
| # pauses common and long persistence examples deliberately rare, avoiding | |
| # a large one-sided shift toward </idle>. | |
| control_post_idle_sampling: str = "legacy_uniform" | |
| # Emergency stop has a stricter timing contract than ordinary command | |
| # interruption. The command must arrive while the counterfactual source | |
| # motion is still active and must leave a real moving tail after the full | |
| # utterance has become observable. Historical run configs keep the old | |
| # values and disabled activity selector for exact checkpoint replay. | |
| control_min_start_seconds: float = 2.0 | |
| control_max_start_seconds: float = 6.0 | |
| control_min_remaining_seconds: float = 0.4 | |
| control_activity_aware_timing: bool = False | |
| # A small audited subset of the Chinese TTS corpus mistranslates English | |
| # ``retro`` as spatial ``backward``. When enabled, only those contradictory | |
| # Chinese renditions are removed; the matching English voices and motions | |
| # remain available. | |
| filter_known_tts_translation_mismatches: bool = False | |
| # Optional data ablation: a generated, audited manifest selects exactly | |
| # one motion trajectory for each exact English/Chinese instruction pair. | |
| # Empty keeps the full many-to-one corpus and is the formal default. | |
| canonical_motion_manifest: str = "" | |
| control_command_manifest: str = ( | |
| "/cpfs01/thscc/sharestorage/AIGC-06-01/Hithink_VideoGen_Group/user_workspace/yetinghong/videogen/" | |
| "hfd_control_commands_v1/manifest.jsonl" | |
| ) | |
| backchannel_command_manifest: str = ( | |
| "/cpfs01/thscc/sharestorage/AIGC-06-01/Hithink_VideoGen_Group/user_workspace/yetinghong/videogen/" | |
| "hfd_backchannel_commands_v2_short/manifest.jsonl" | |
| ) | |
| hesitation_pause_manifest: str = ( | |
| "/cpfs01/thscc/sharestorage/AIGC-06-01/Hithink_VideoGen_Group/user_workspace/yetinghong/videogen/" | |
| "hfd_hesitation_pause_v1/manifest.jsonl" | |
| ) | |
| # New pools cap duplicate motion rows for one exact bilingual instruction. | |
| # Existing task pools remain byte-for-byte unchanged. Selection is | |
| # deterministic and duration-stratified rather than first-N. | |
| new_task_max_rows_per_instruction_train: int = 32 | |
| new_task_max_rows_per_instruction_val: int = 8 | |
| # Filled hesitation versus self-repair. Silent intra-command pauses are a | |
| # separate task and therefore do not consume this split. | |
| hesitation_filled_ratio: float = 0.60 | |
| # Among self-repair hesitation sessions, omit an explicit ``I mean`` / | |
| # ``改成`` cue for this fraction. The false-start prefix is followed by a | |
| # pause and the complete final command, matching natural cue-less restarts. | |
| self_repair_implicit_ratio: float = 0.0 | |
| backchannel_min_start_seconds: float = 1.2 | |
| backchannel_max_start_seconds: float = 6.0 | |
| backchannel_min_remaining_seconds: float = 1.2 | |
| safe_hold_stability_window_frames: int = 20 | |
| # Motion -> standing and standing -> the next task use a slower one-second | |
| # transition. Version 13 retains the original 20-frame first-task entry; | |
| # causal-timing version 14 uses this transition for every task entry and | |
| # the stationary/listen envelope as well. | |
| turn_transition_frames: int = 50 | |
| # Version 14 starts the clean speech waveform first, keeps the target in | |
| # idle through the first Qwen boundary containing real voice onset, keeps | |
| # one explicit response interval, and then starts </listen>. Historical | |
| # runs keep this false so their checkpoint contract stays reproducible. | |
| causal_listen_timing: bool = False | |
| # Expected leading-silence budget recorded in the train/infer contract. | |
| # Longer valid TTS silence is measured and retained; source-duration bounds | |
| # are used for conservative sampler/context estimates. Three steps are | |
| # 1.2 seconds. | |
| max_speech_onset_steps: int = 3 | |
| # After the Qwen boundary that first contains sustained voice energy, keep | |
| # one more 0.4 s idle response interval before emitting </listen>. | |
| listen_reaction_steps: int = 1 | |
| # Qwen2.5-Omni-3B has a 32768-token context. A first 0.6 s window uses 82 | |
| # tokens and every later one uses 17, so 1923 complete windows fit while | |
| # preserving every source-local KV entry. | |
| max_stream_steps: int = 1923 | |
| # Drop only the deterministic longest tail after the final task mixture is | |
| # formed. Zero preserves historical data; production v22 uses 0.001. | |
| long_session_drop_ratio: float = 0.0 | |
| def __call__(self, split: str = "train", transform_kwargs=None, **kwargs) -> Any: | |
| from psi.data.seed.full_duplex_audio_command_dataset import ( | |
| FullDuplexAudioCommandDataset, | |
| FullDuplexTransformedDataset, | |
| ) | |
| repack = self.transform.repack | |
| action_chunk_size = int(getattr(repack, "action_chunk_size", 30)) | |
| prefix_frames = int(getattr(repack, "rtc_prefix_frames", 0)) | |
| expected_prefix = action_chunk_size - int(self.action_exec_horizon) | |
| if prefix_frames != expected_prefix: | |
| raise ValueError( | |
| "Full-duplex data/model RTC mismatch: " | |
| f"repack_prefix={prefix_frames}, expected={expected_prefix}" | |
| ) | |
| raw_dataset = FullDuplexAudioCommandDataset( | |
| hidden_root=self.hidden_root, | |
| metadata_jsonl=self.metadata_jsonl, | |
| motion_metadata_csv=self.motion_metadata_csv, | |
| motion_root=self.motion_root, | |
| white_noise_hidden_dir=self.white_noise_hidden_dir, | |
| background_hidden_root=self.background_hidden_root, | |
| background_assignment_path=self.background_assignment_path, | |
| continuous_audio_root=self.continuous_audio_root, | |
| online_audio=self.online_audio, | |
| background_source_name=self.background_source_name, | |
| background_task_name=self.background_task_name, | |
| split=split, | |
| val_ratio=self.val_ratio, | |
| split_seed=self.split_seed, | |
| split_source_namespace=self.split_source_namespace, | |
| source_name=self.source_name, | |
| task_name=self.task_name, | |
| action_chunk_size=action_chunk_size, | |
| action_exec_horizon=self.action_exec_horizon, | |
| motion_fps=self.motion_fps, | |
| motion_metadata_fps=self.motion_metadata_fps, | |
| audio_sampling_rate=self.audio_sampling_rate, | |
| audio_window_seconds=self.audio_window_seconds, | |
| audio_sample_step_frames=self.audio_sample_step_frames, | |
| action_sample_step_frames=self.action_sample_step_frames, | |
| motion_cache_size=self.motion_cache_size, | |
| fixed_voice_per_record=self.fixed_voice_per_record, | |
| max_voice_windows=self.max_voice_windows, | |
| max_voice_median_ratio=self.max_voice_median_ratio, | |
| deterministic_eval=self.deterministic_eval, | |
| limit_sources=self.limit_sources, | |
| max_motion_steps=self.max_motion_steps, | |
| initial_idle_max_steps=self.initial_idle_max_steps, | |
| stationary_blend_frames=self.stationary_blend_frames, | |
| stationary_heading_alignment=self.stationary_heading_alignment, | |
| safe_stationary_anchors=self.safe_stationary_anchors, | |
| stationary_idle_residual_retargeting=( | |
| self.stationary_idle_residual_retargeting | |
| ), | |
| stationary_listen_hold=self.stationary_listen_hold, | |
| dynamic_task_entry_blend=self.dynamic_task_entry_blend, | |
| multi_turn_sessions=self.multi_turn_sessions, | |
| session_min_turns=self.session_min_turns, | |
| session_max_turns=self.session_max_turns, | |
| session_voice_variants=self.session_voice_variants, | |
| interruption_sessions=self.interruption_sessions, | |
| interruption_ratio=self.interruption_ratio, | |
| interruption_min_start_seconds=( | |
| self.interruption_min_start_seconds | |
| ), | |
| interruption_max_start_seconds=( | |
| self.interruption_max_start_seconds | |
| ), | |
| interruption_min_remaining_seconds=( | |
| self.interruption_min_remaining_seconds | |
| ), | |
| independent_task_pools=self.independent_task_pools, | |
| turn_taking_session_variants=( | |
| self.turn_taking_session_variants | |
| ), | |
| command_interruption_session_variants=( | |
| self.command_interruption_session_variants | |
| ), | |
| emergency_stop_session_variants=( | |
| self.emergency_stop_session_variants | |
| ), | |
| safe_hold_session_variants=self.safe_hold_session_variants, | |
| backchannel_session_variants=( | |
| self.backchannel_session_variants | |
| ), | |
| hesitation_session_variants=( | |
| self.hesitation_session_variants | |
| ), | |
| pause_session_variants=self.pause_session_variants, | |
| interruption_task_session_ratio=( | |
| self.interruption_task_session_ratio | |
| ), | |
| control_task_session_ratio=self.control_task_session_ratio, | |
| backchannel_task_session_ratio=( | |
| self.backchannel_task_session_ratio | |
| ), | |
| hesitation_task_window_ratio=( | |
| self.hesitation_task_window_ratio | |
| ), | |
| pause_task_window_ratio=self.pause_task_window_ratio, | |
| control_followup_ratio=self.control_followup_ratio, | |
| control_post_idle_min_steps=self.control_post_idle_min_steps, | |
| control_post_idle_max_steps=self.control_post_idle_max_steps, | |
| control_post_idle_sampling=self.control_post_idle_sampling, | |
| control_min_start_seconds=self.control_min_start_seconds, | |
| control_max_start_seconds=self.control_max_start_seconds, | |
| control_min_remaining_seconds=( | |
| self.control_min_remaining_seconds | |
| ), | |
| control_activity_aware_timing=( | |
| self.control_activity_aware_timing | |
| ), | |
| filter_known_tts_translation_mismatches=( | |
| self.filter_known_tts_translation_mismatches | |
| ), | |
| canonical_motion_manifest=self.canonical_motion_manifest, | |
| control_command_manifest=self.control_command_manifest, | |
| backchannel_command_manifest=( | |
| self.backchannel_command_manifest | |
| ), | |
| hesitation_pause_manifest=self.hesitation_pause_manifest, | |
| new_task_max_rows_per_instruction_train=( | |
| self.new_task_max_rows_per_instruction_train | |
| ), | |
| new_task_max_rows_per_instruction_val=( | |
| self.new_task_max_rows_per_instruction_val | |
| ), | |
| hesitation_filled_ratio=self.hesitation_filled_ratio, | |
| self_repair_implicit_ratio=self.self_repair_implicit_ratio, | |
| backchannel_min_start_seconds=( | |
| self.backchannel_min_start_seconds | |
| ), | |
| backchannel_max_start_seconds=( | |
| self.backchannel_max_start_seconds | |
| ), | |
| backchannel_min_remaining_seconds=( | |
| self.backchannel_min_remaining_seconds | |
| ), | |
| safe_hold_stability_window_frames=( | |
| self.safe_hold_stability_window_frames | |
| ), | |
| turn_transition_frames=self.turn_transition_frames, | |
| causal_listen_timing=self.causal_listen_timing, | |
| max_speech_onset_steps=self.max_speech_onset_steps, | |
| listen_reaction_steps=self.listen_reaction_steps, | |
| max_stream_steps=self.max_stream_steps, | |
| long_session_drop_ratio=self.long_session_drop_ratio, | |
| ) | |
| if self.third_party_root: | |
| from psi.data.seed.third_party_dataset import ThirdPartyAugmentedDataset | |
| selection = self.third_party_train_selection if split == "train" else self.third_party_val_selection | |
| if not selection: | |
| raise ValueError("Third-party data requires an explicit split selection") | |
| raw_dataset = ThirdPartyAugmentedDataset(raw_dataset, self.third_party_root, selection, split) | |
| if self.multimodal_overlay_manifests: | |
| if not self.swa_visual_qa_enabled: | |
| raise ValueError("Unified overlay requires the AV/independent QA model") | |
| from psi.data.seed.live_multimodal_overlay import LiveMultimodalOverlayDataset | |
| from psi.data.seed.hfd_multimodal_overlay import MultimodalOverlayPolicy | |
| raw_dataset = LiveMultimodalOverlayDataset( | |
| raw_dataset, self.multimodal_overlay_manifests, split, | |
| policy=MultimodalOverlayPolicy( | |
| qa_ratio=self.multimodal_overlay_qa_ratio, | |
| visual_only_ratio=self.multimodal_overlay_visual_only_ratio, | |
| audio_only_ratio=1. - self.multimodal_overlay_qa_ratio - self.multimodal_overlay_visual_only_ratio), | |
| seed=self.split_seed, strict_voice=self.multimodal_overlay_strict_voice, | |
| visual_input_mode=self.multimodal_visual_input_mode) | |
| audio_dataset = FullDuplexTransformedDataset( | |
| raw_dataset, | |
| self.transform, | |
| transform_kwargs=transform_kwargs, | |
| ) | |
| if self.multimodal_overlay_manifests or not self.swa_visual_qa_enabled: | |
| return audio_dataset | |
| if self.transform.repack.root40_actions: | |
| raise ValueError("Root40 requires raw-pose unified overlay data, not historical pretransformed 39D SWA bodies") | |
| from psi.data.seed.swa_visual_qa_dataset import ( | |
| MixedFullDuplexDataset, | |
| SwaVisualQaDataset, | |
| ) | |
| return MixedFullDuplexDataset( | |
| audio_dataset, | |
| SwaVisualQaDataset( | |
| self.swa_source_plan, | |
| split, | |
| repeat=self.swa_train_repeat, | |
| ), | |
| ) | |