code-HFD / src /psi /config /data_full_duplex_audio_command.py
Tinghong-Ye22's picture
Add files using upload-large-folder tool
06f9561 verified
Raw History Blame Contribute Delete
24.3 kB
from __future__ import annotations
from typing import Any, Literal
from psi.config.config import DataConfig
class FullDuplexAudioCommandDataConfig(DataConfig):
# Audited microphones + latest motion builder, before the visual overlay.
# Empty defaults preserve historical run-config reconstruction/resume.
third_party_root: str = ""
third_party_train_selection: str = ""
third_party_val_selection: str = ""
# Formal speaking-while-acting extension. The repeat factor controls the
# added SWA share without deleting any of the established audio sources.
swa_visual_qa_enabled: bool = False
swa_source_plan: str = ""
swa_train_repeat: int = 1
# Opt-in unified input overlay. Unlike the legacy concat, this preserves
# all five original task pools and adds no independent SWA body task.
multimodal_overlay_manifests: tuple[str, ...] = ()
multimodal_overlay_qa_ratio: float = 0.75
multimodal_overlay_visual_only_ratio: float = 0.10
multimodal_overlay_strict_voice: bool = True
multimodal_visual_input_mode: Literal["cached", "raw"] = "cached"
# HFDv2 formal training consumes these waveforms directly and never reads
# the offline Qwen hidden-state cache below.
online_audio: bool = True
continuous_audio_root: str = (
"/cpfs01/thscc/sharestorage/AIGC-06-01/Hithink_VideoGen_Group/user_workspace/yetinghong/videogen/"
"hfd_audio_command_continuous_office_v1"
)
hidden_root: str = (
"/cpfs01/thscc/sharestorage/AIGC-06-01/Hithink_VideoGen_Group/user_workspace/yetinghong/videogen/"
"hfd_audio_command_continuous_office_v1/mixed_hidden"
)
metadata_jsonl: str = "/cpfs01/thscc/sharestorage/AIGC-06-01/Hithink_VideoGen_Group/user_workspace/buwendong/user/code/qwen3-tts/metadata.jsonl"
motion_metadata_csv: str = (
"/cpfs01/thscc/sharestorage/AIGC-06-01/Hithink_VideoGen_Group/user_workspace/yetinghong/code/Psi0/dataset/seed_all/metadata/seed_metadata_v004.csv"
)
motion_root: str = "/cpfs01/thscc/sharestorage/AIGC-06-01/Hithink_VideoGen_Group/user_workspace/yetinghong/code/Psi0/dataset/seed_all"
# Semantics-v3 binds every spoken source to one uninterrupted background
# timeline. The speech windows above contain waveform-level speech +
# background, while idle/motion windows below read adjacent pure-background
# positions from that exact same timeline.
white_noise_hidden_dir: str = ""
background_hidden_root: str = (
"/cpfs01/thscc/sharestorage/AIGC-06-01/Hithink_VideoGen_Group/user_workspace/yetinghong/videogen/"
"hfd_audio_command_continuous_office_v1/background_hidden"
)
background_assignment_path: str = (
"/cpfs01/thscc/sharestorage/AIGC-06-01/Hithink_VideoGen_Group/user_workspace/yetinghong/videogen/"
"hfd_audio_command_continuous_office_v1/manifests/"
"source_background_assignments.jsonl"
)
background_source_name: str = "hfd_continuous_office_background"
background_task_name: str = "audio_command_text"
val_ratio: float = 0.1
split_seed: int = 42
split_source_namespace: str = "seed"
source_name: str = "qwen3_tts_seed_audio_command"
task_name: str = "audio_command_text"
action_exec_horizon: int = 20
motion_fps: int = 50
# SEED metadata records the source motion duration at 120 Hz, while the
# exported G1 CSVs consumed by HFD are 50 Hz.
motion_metadata_fps: int = 120
audio_sampling_rate: int = 16000
audio_window_seconds: float = 0.6
# Qwen/KV follows deployment cadence, while action targets are sampled at
# every 50 Hz state to reproduce v26's dense RTC supervision.
audio_sample_step_frames: int = 20
action_sample_step_frames: int = 1
motion_cache_size: int = 256
# Bind every motion record to a stable rendition. This makes the sampler's
# predicted length equal to the item that workers actually build and still
# distributes speaker/language variants across the full corpus.
fixed_voice_per_record: bool = True
# Reject rare broken TTS files that are many times longer than the other
# renditions of the same short command. Zero disables the absolute cap.
max_voice_windows: int = 30
max_voice_median_ratio: float = 3.0
# Validation must rebuild the exact same source at every checkpoint.
deterministic_eval: bool = True
# Non-zero values are explicit smoke/debug limits. Production defaults
# retain every source and every motion step.
limit_sources: int = 0
max_motion_steps: int = 0
# The v13 entry point overrides this to seven background-only positions
# (0..2.8 s on the 0.4 s Qwen grid), covering the requested 0..3 s range
# without placing an unobservable control boundary inside one window.
# Five remains the historical single-turn/raw-corpus-compatible default.
initial_idle_max_steps: int = 5
# Natural idle/listen targets smoothly leave and return to the paired task
# motion's initial pose over this many 50 Hz frames.
stationary_blend_frames: int = 20
# Version 15 rigidly re-expresses idle/listen candidates in the current
# robot heading. Historical checkpoints retain the previous XY-only path.
stationary_heading_alignment: bool = False
# Production v22 performs a data-driven whole-body audit of the neutral
# idle pool, then chooses the accepted standing pose closest to the current
# robot state. Historical configs keep this disabled so their targets can
# still be reconstructed exactly.
safe_stationary_anchors: bool = False
# Semantics v24 keeps the sampled idle clip's *small motion residual* but
# expresses it around the current safe anchor. This prevents two static
# standing poses with different absolute arm configurations from creating
# a fast synthetic arm swing when they are blended together.
stationary_idle_residual_retargeting: bool = False
# Opt-in target revision: enter a recorded listening pose and hold it;
# the following task blends from the actual last listen frame. False
# preserves old checkpoints and immutable ablation fixtures exactly.
stationary_listen_hold: bool = False
# Blend the current safe standing state into the *moving* task prefix.
# This avoids holding a locomotion clip's arbitrary first (often
# single-support) frame for one second before suddenly restoring velocity.
dynamic_task_entry_blend: bool = False
# Versions 13/14 group primitive commands into one uninterrupted interaction.
# The complete item owns one audio background and one Qwen KV lifetime.
multi_turn_sessions: bool = False
session_min_turns: int = 2
session_max_turns: int = 4
# Version 17 creates static train-session variants. Every primitive is
# used once per variant with one shared speaker/language identity across
# all turns. Validation always keeps one deterministic variant.
# Zero retains the historical per-record voice policy exactly.
session_voice_variants: int = 0
# Version 20 uses the two existing train variants as orthogonal behavior
# variants: one complete turn-taking session and one interruption session.
# Every primitive therefore contributes once to each behavior (exact 1:1
# primitive supervision); interrupted timelines are naturally shorter.
# Validation deterministically builds both modes.
interruption_sessions: bool = False
interruption_ratio: float = 0.5
interruption_min_start_seconds: float = 2.0
interruption_max_start_seconds: float = 6.0
interruption_min_remaining_seconds: float = 1.5
# Version 21 keeps the complete two-variant turn-taking-v1 pool intact and
# constructs every additional behaviour as an independent pool. Integer
# variant counts are direct, auditable data weights: 2:2 is exactly 1:1.
independent_task_pools: bool = False
turn_taking_session_variants: int = 2
command_interruption_session_variants: int = 2
emergency_stop_session_variants: int = 1
safe_hold_session_variants: int = 1
# Backchannel is an independent hard-negative pool: short same-speaker
# acknowledgements overlap an active motion without changing its action,
# progress, RTC prefix or </motion> target.
backchannel_session_variants: int = 0
# Semantics v23 adds two independent listen-side hard cases without
# changing the established turn-taking/interruption/Stop/backchannel pools.
# ``pause`` inserts background-only time inside one unfinished command;
# ``hesitation`` inserts a filled pause or an incomplete false start plus a
# repair cue. Both keep </listen> until the final intent is complete.
hesitation_session_variants: int = 0
pause_session_variants: int = 0
# Optional final-session target applied after all context/background and
# task-eligibility filtering. Zero preserves v21 integer-pool behavior;
# production v22 balances the actual sampled sessions, not merely raw
# primitive uses.
interruption_task_session_ratio: float = 0.0
# Legacy v21 runs leave these at zero/one and therefore reproduce their
# original integer-variant weighting. The v22 launcher keeps hold at zero,
# assigns Stop and backchannel their requested final-session shares, and
# retains follow-up support for the selected Stop sessions.
control_task_session_ratio: float = 0.0
backchannel_task_session_ratio: float = 0.0
# These are final *stream-window* shares. v23 first reconstructs the exact
# v22 selected session set, then appends enough new sessions to target 12%
# hesitation and 8% pause by estimated optimizer work.
hesitation_task_window_ratio: float = 0.0
pause_task_window_ratio: float = 0.0
control_followup_ratio: float = 0.0
control_post_idle_min_steps: int = 1
# Zero preserves historical behavior (the maximum follows
# ``initial_idle_max_steps``). Production Stop-persistence training sets
# this to 20 Qwen decisions, i.e. 8 seconds at the 0.4 s cadence.
control_post_idle_max_steps: int = 0
# Historical checkpoints sampled the explicit range uniformly. The
# balanced policy keeps the same 1.2--8 s support but makes ordinary short
# pauses common and long persistence examples deliberately rare, avoiding
# a large one-sided shift toward </idle>.
control_post_idle_sampling: str = "legacy_uniform"
# Emergency stop has a stricter timing contract than ordinary command
# interruption. The command must arrive while the counterfactual source
# motion is still active and must leave a real moving tail after the full
# utterance has become observable. Historical run configs keep the old
# values and disabled activity selector for exact checkpoint replay.
control_min_start_seconds: float = 2.0
control_max_start_seconds: float = 6.0
control_min_remaining_seconds: float = 0.4
control_activity_aware_timing: bool = False
# A small audited subset of the Chinese TTS corpus mistranslates English
# ``retro`` as spatial ``backward``. When enabled, only those contradictory
# Chinese renditions are removed; the matching English voices and motions
# remain available.
filter_known_tts_translation_mismatches: bool = False
# Optional data ablation: a generated, audited manifest selects exactly
# one motion trajectory for each exact English/Chinese instruction pair.
# Empty keeps the full many-to-one corpus and is the formal default.
canonical_motion_manifest: str = ""
control_command_manifest: str = (
"/cpfs01/thscc/sharestorage/AIGC-06-01/Hithink_VideoGen_Group/user_workspace/yetinghong/videogen/"
"hfd_control_commands_v1/manifest.jsonl"
)
backchannel_command_manifest: str = (
"/cpfs01/thscc/sharestorage/AIGC-06-01/Hithink_VideoGen_Group/user_workspace/yetinghong/videogen/"
"hfd_backchannel_commands_v2_short/manifest.jsonl"
)
hesitation_pause_manifest: str = (
"/cpfs01/thscc/sharestorage/AIGC-06-01/Hithink_VideoGen_Group/user_workspace/yetinghong/videogen/"
"hfd_hesitation_pause_v1/manifest.jsonl"
)
# New pools cap duplicate motion rows for one exact bilingual instruction.
# Existing task pools remain byte-for-byte unchanged. Selection is
# deterministic and duration-stratified rather than first-N.
new_task_max_rows_per_instruction_train: int = 32
new_task_max_rows_per_instruction_val: int = 8
# Filled hesitation versus self-repair. Silent intra-command pauses are a
# separate task and therefore do not consume this split.
hesitation_filled_ratio: float = 0.60
# Among self-repair hesitation sessions, omit an explicit ``I mean`` /
# ``改成`` cue for this fraction. The false-start prefix is followed by a
# pause and the complete final command, matching natural cue-less restarts.
self_repair_implicit_ratio: float = 0.0
backchannel_min_start_seconds: float = 1.2
backchannel_max_start_seconds: float = 6.0
backchannel_min_remaining_seconds: float = 1.2
safe_hold_stability_window_frames: int = 20
# Motion -> standing and standing -> the next task use a slower one-second
# transition. Version 13 retains the original 20-frame first-task entry;
# causal-timing version 14 uses this transition for every task entry and
# the stationary/listen envelope as well.
turn_transition_frames: int = 50
# Version 14 starts the clean speech waveform first, keeps the target in
# idle through the first Qwen boundary containing real voice onset, keeps
# one explicit response interval, and then starts </listen>. Historical
# runs keep this false so their checkpoint contract stays reproducible.
causal_listen_timing: bool = False
# Expected leading-silence budget recorded in the train/infer contract.
# Longer valid TTS silence is measured and retained; source-duration bounds
# are used for conservative sampler/context estimates. Three steps are
# 1.2 seconds.
max_speech_onset_steps: int = 3
# After the Qwen boundary that first contains sustained voice energy, keep
# one more 0.4 s idle response interval before emitting </listen>.
listen_reaction_steps: int = 1
# Qwen2.5-Omni-3B has a 32768-token context. A first 0.6 s window uses 82
# tokens and every later one uses 17, so 1923 complete windows fit while
# preserving every source-local KV entry.
max_stream_steps: int = 1923
# Drop only the deterministic longest tail after the final task mixture is
# formed. Zero preserves historical data; production v22 uses 0.001.
long_session_drop_ratio: float = 0.0
def __call__(self, split: str = "train", transform_kwargs=None, **kwargs) -> Any:
from psi.data.seed.full_duplex_audio_command_dataset import (
FullDuplexAudioCommandDataset,
FullDuplexTransformedDataset,
)
repack = self.transform.repack
action_chunk_size = int(getattr(repack, "action_chunk_size", 30))
prefix_frames = int(getattr(repack, "rtc_prefix_frames", 0))
expected_prefix = action_chunk_size - int(self.action_exec_horizon)
if prefix_frames != expected_prefix:
raise ValueError(
"Full-duplex data/model RTC mismatch: "
f"repack_prefix={prefix_frames}, expected={expected_prefix}"
)
raw_dataset = FullDuplexAudioCommandDataset(
hidden_root=self.hidden_root,
metadata_jsonl=self.metadata_jsonl,
motion_metadata_csv=self.motion_metadata_csv,
motion_root=self.motion_root,
white_noise_hidden_dir=self.white_noise_hidden_dir,
background_hidden_root=self.background_hidden_root,
background_assignment_path=self.background_assignment_path,
continuous_audio_root=self.continuous_audio_root,
online_audio=self.online_audio,
background_source_name=self.background_source_name,
background_task_name=self.background_task_name,
split=split,
val_ratio=self.val_ratio,
split_seed=self.split_seed,
split_source_namespace=self.split_source_namespace,
source_name=self.source_name,
task_name=self.task_name,
action_chunk_size=action_chunk_size,
action_exec_horizon=self.action_exec_horizon,
motion_fps=self.motion_fps,
motion_metadata_fps=self.motion_metadata_fps,
audio_sampling_rate=self.audio_sampling_rate,
audio_window_seconds=self.audio_window_seconds,
audio_sample_step_frames=self.audio_sample_step_frames,
action_sample_step_frames=self.action_sample_step_frames,
motion_cache_size=self.motion_cache_size,
fixed_voice_per_record=self.fixed_voice_per_record,
max_voice_windows=self.max_voice_windows,
max_voice_median_ratio=self.max_voice_median_ratio,
deterministic_eval=self.deterministic_eval,
limit_sources=self.limit_sources,
max_motion_steps=self.max_motion_steps,
initial_idle_max_steps=self.initial_idle_max_steps,
stationary_blend_frames=self.stationary_blend_frames,
stationary_heading_alignment=self.stationary_heading_alignment,
safe_stationary_anchors=self.safe_stationary_anchors,
stationary_idle_residual_retargeting=(
self.stationary_idle_residual_retargeting
),
stationary_listen_hold=self.stationary_listen_hold,
dynamic_task_entry_blend=self.dynamic_task_entry_blend,
multi_turn_sessions=self.multi_turn_sessions,
session_min_turns=self.session_min_turns,
session_max_turns=self.session_max_turns,
session_voice_variants=self.session_voice_variants,
interruption_sessions=self.interruption_sessions,
interruption_ratio=self.interruption_ratio,
interruption_min_start_seconds=(
self.interruption_min_start_seconds
),
interruption_max_start_seconds=(
self.interruption_max_start_seconds
),
interruption_min_remaining_seconds=(
self.interruption_min_remaining_seconds
),
independent_task_pools=self.independent_task_pools,
turn_taking_session_variants=(
self.turn_taking_session_variants
),
command_interruption_session_variants=(
self.command_interruption_session_variants
),
emergency_stop_session_variants=(
self.emergency_stop_session_variants
),
safe_hold_session_variants=self.safe_hold_session_variants,
backchannel_session_variants=(
self.backchannel_session_variants
),
hesitation_session_variants=(
self.hesitation_session_variants
),
pause_session_variants=self.pause_session_variants,
interruption_task_session_ratio=(
self.interruption_task_session_ratio
),
control_task_session_ratio=self.control_task_session_ratio,
backchannel_task_session_ratio=(
self.backchannel_task_session_ratio
),
hesitation_task_window_ratio=(
self.hesitation_task_window_ratio
),
pause_task_window_ratio=self.pause_task_window_ratio,
control_followup_ratio=self.control_followup_ratio,
control_post_idle_min_steps=self.control_post_idle_min_steps,
control_post_idle_max_steps=self.control_post_idle_max_steps,
control_post_idle_sampling=self.control_post_idle_sampling,
control_min_start_seconds=self.control_min_start_seconds,
control_max_start_seconds=self.control_max_start_seconds,
control_min_remaining_seconds=(
self.control_min_remaining_seconds
),
control_activity_aware_timing=(
self.control_activity_aware_timing
),
filter_known_tts_translation_mismatches=(
self.filter_known_tts_translation_mismatches
),
canonical_motion_manifest=self.canonical_motion_manifest,
control_command_manifest=self.control_command_manifest,
backchannel_command_manifest=(
self.backchannel_command_manifest
),
hesitation_pause_manifest=self.hesitation_pause_manifest,
new_task_max_rows_per_instruction_train=(
self.new_task_max_rows_per_instruction_train
),
new_task_max_rows_per_instruction_val=(
self.new_task_max_rows_per_instruction_val
),
hesitation_filled_ratio=self.hesitation_filled_ratio,
self_repair_implicit_ratio=self.self_repair_implicit_ratio,
backchannel_min_start_seconds=(
self.backchannel_min_start_seconds
),
backchannel_max_start_seconds=(
self.backchannel_max_start_seconds
),
backchannel_min_remaining_seconds=(
self.backchannel_min_remaining_seconds
),
safe_hold_stability_window_frames=(
self.safe_hold_stability_window_frames
),
turn_transition_frames=self.turn_transition_frames,
causal_listen_timing=self.causal_listen_timing,
max_speech_onset_steps=self.max_speech_onset_steps,
listen_reaction_steps=self.listen_reaction_steps,
max_stream_steps=self.max_stream_steps,
long_session_drop_ratio=self.long_session_drop_ratio,
)
if self.third_party_root:
from psi.data.seed.third_party_dataset import ThirdPartyAugmentedDataset
selection = self.third_party_train_selection if split == "train" else self.third_party_val_selection
if not selection:
raise ValueError("Third-party data requires an explicit split selection")
raw_dataset = ThirdPartyAugmentedDataset(raw_dataset, self.third_party_root, selection, split)
if self.multimodal_overlay_manifests:
if not self.swa_visual_qa_enabled:
raise ValueError("Unified overlay requires the AV/independent QA model")
from psi.data.seed.live_multimodal_overlay import LiveMultimodalOverlayDataset
from psi.data.seed.hfd_multimodal_overlay import MultimodalOverlayPolicy
raw_dataset = LiveMultimodalOverlayDataset(
raw_dataset, self.multimodal_overlay_manifests, split,
policy=MultimodalOverlayPolicy(
qa_ratio=self.multimodal_overlay_qa_ratio,
visual_only_ratio=self.multimodal_overlay_visual_only_ratio,
audio_only_ratio=1. - self.multimodal_overlay_qa_ratio - self.multimodal_overlay_visual_only_ratio),
seed=self.split_seed, strict_voice=self.multimodal_overlay_strict_voice,
visual_input_mode=self.multimodal_visual_input_mode)
audio_dataset = FullDuplexTransformedDataset(
raw_dataset,
self.transform,
transform_kwargs=transform_kwargs,
)
if self.multimodal_overlay_manifests or not self.swa_visual_qa_enabled:
return audio_dataset
if self.transform.repack.root40_actions:
raise ValueError("Root40 requires raw-pose unified overlay data, not historical pretransformed 39D SWA bodies")
from psi.data.seed.swa_visual_qa_dataset import (
MixedFullDuplexDataset,
SwaVisualQaDataset,
)
return MixedFullDuplexDataset(
audio_dataset,
SwaVisualQaDataset(
self.swa_source_plan,
split,
repeat=self.swa_train_repeat,
),
)