File size: 24,269 Bytes
06f9561 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 151 152 153 154 155 156 157 158 159 160 161 162 163 164 165 166 167 168 169 170 171 172 173 174 175 176 177 178 179 180 181 182 183 184 185 186 187 188 189 190 191 192 193 194 195 196 197 198 199 200 201 202 203 204 205 206 207 208 209 210 211 212 213 214 215 216 217 218 219 220 221 222 223 224 225 226 227 228 229 230 231 232 233 234 235 236 237 238 239 240 241 242 243 244 245 246 247 248 249 250 251 252 253 254 255 256 257 258 259 260 261 262 263 264 265 266 267 268 269 270 271 272 273 274 275 276 277 278 279 280 281 282 283 284 285 286 287 288 289 290 291 292 293 294 295 296 297 298 299 300 301 302 303 304 305 306 307 308 309 310 311 312 313 314 315 316 317 318 319 320 321 322 323 324 325 326 327 328 329 330 331 332 333 334 335 336 337 338 339 340 341 342 343 344 345 346 347 348 349 350 351 352 353 354 355 356 357 358 359 360 361 362 363 364 365 366 367 368 369 370 371 372 373 374 375 376 377 378 379 380 381 382 383 384 385 386 387 388 389 390 391 392 393 394 395 396 397 398 399 400 401 402 403 404 405 406 407 408 409 410 411 412 413 414 415 416 417 418 419 420 421 422 423 424 425 426 427 428 429 430 431 432 433 434 435 436 437 438 439 440 441 442 443 444 445 | from __future__ import annotations
from typing import Any, Literal
from psi.config.config import DataConfig
class FullDuplexAudioCommandDataConfig(DataConfig):
# Audited microphones + latest motion builder, before the visual overlay.
# Empty defaults preserve historical run-config reconstruction/resume.
third_party_root: str = ""
third_party_train_selection: str = ""
third_party_val_selection: str = ""
# Formal speaking-while-acting extension. The repeat factor controls the
# added SWA share without deleting any of the established audio sources.
swa_visual_qa_enabled: bool = False
swa_source_plan: str = ""
swa_train_repeat: int = 1
# Opt-in unified input overlay. Unlike the legacy concat, this preserves
# all five original task pools and adds no independent SWA body task.
multimodal_overlay_manifests: tuple[str, ...] = ()
multimodal_overlay_qa_ratio: float = 0.75
multimodal_overlay_visual_only_ratio: float = 0.10
multimodal_overlay_strict_voice: bool = True
multimodal_visual_input_mode: Literal["cached", "raw"] = "cached"
# HFDv2 formal training consumes these waveforms directly and never reads
# the offline Qwen hidden-state cache below.
online_audio: bool = True
continuous_audio_root: str = (
"/cpfs01/thscc/sharestorage/AIGC-06-01/Hithink_VideoGen_Group/user_workspace/yetinghong/videogen/"
"hfd_audio_command_continuous_office_v1"
)
hidden_root: str = (
"/cpfs01/thscc/sharestorage/AIGC-06-01/Hithink_VideoGen_Group/user_workspace/yetinghong/videogen/"
"hfd_audio_command_continuous_office_v1/mixed_hidden"
)
metadata_jsonl: str = "/cpfs01/thscc/sharestorage/AIGC-06-01/Hithink_VideoGen_Group/user_workspace/buwendong/user/code/qwen3-tts/metadata.jsonl"
motion_metadata_csv: str = (
"/cpfs01/thscc/sharestorage/AIGC-06-01/Hithink_VideoGen_Group/user_workspace/yetinghong/code/Psi0/dataset/seed_all/metadata/seed_metadata_v004.csv"
)
motion_root: str = "/cpfs01/thscc/sharestorage/AIGC-06-01/Hithink_VideoGen_Group/user_workspace/yetinghong/code/Psi0/dataset/seed_all"
# Semantics-v3 binds every spoken source to one uninterrupted background
# timeline. The speech windows above contain waveform-level speech +
# background, while idle/motion windows below read adjacent pure-background
# positions from that exact same timeline.
white_noise_hidden_dir: str = ""
background_hidden_root: str = (
"/cpfs01/thscc/sharestorage/AIGC-06-01/Hithink_VideoGen_Group/user_workspace/yetinghong/videogen/"
"hfd_audio_command_continuous_office_v1/background_hidden"
)
background_assignment_path: str = (
"/cpfs01/thscc/sharestorage/AIGC-06-01/Hithink_VideoGen_Group/user_workspace/yetinghong/videogen/"
"hfd_audio_command_continuous_office_v1/manifests/"
"source_background_assignments.jsonl"
)
background_source_name: str = "hfd_continuous_office_background"
background_task_name: str = "audio_command_text"
val_ratio: float = 0.1
split_seed: int = 42
split_source_namespace: str = "seed"
source_name: str = "qwen3_tts_seed_audio_command"
task_name: str = "audio_command_text"
action_exec_horizon: int = 20
motion_fps: int = 50
# SEED metadata records the source motion duration at 120 Hz, while the
# exported G1 CSVs consumed by HFD are 50 Hz.
motion_metadata_fps: int = 120
audio_sampling_rate: int = 16000
audio_window_seconds: float = 0.6
# Qwen/KV follows deployment cadence, while action targets are sampled at
# every 50 Hz state to reproduce v26's dense RTC supervision.
audio_sample_step_frames: int = 20
action_sample_step_frames: int = 1
motion_cache_size: int = 256
# Bind every motion record to a stable rendition. This makes the sampler's
# predicted length equal to the item that workers actually build and still
# distributes speaker/language variants across the full corpus.
fixed_voice_per_record: bool = True
# Reject rare broken TTS files that are many times longer than the other
# renditions of the same short command. Zero disables the absolute cap.
max_voice_windows: int = 30
max_voice_median_ratio: float = 3.0
# Validation must rebuild the exact same source at every checkpoint.
deterministic_eval: bool = True
# Non-zero values are explicit smoke/debug limits. Production defaults
# retain every source and every motion step.
limit_sources: int = 0
max_motion_steps: int = 0
# The v13 entry point overrides this to seven background-only positions
# (0..2.8 s on the 0.4 s Qwen grid), covering the requested 0..3 s range
# without placing an unobservable control boundary inside one window.
# Five remains the historical single-turn/raw-corpus-compatible default.
initial_idle_max_steps: int = 5
# Natural idle/listen targets smoothly leave and return to the paired task
# motion's initial pose over this many 50 Hz frames.
stationary_blend_frames: int = 20
# Version 15 rigidly re-expresses idle/listen candidates in the current
# robot heading. Historical checkpoints retain the previous XY-only path.
stationary_heading_alignment: bool = False
# Production v22 performs a data-driven whole-body audit of the neutral
# idle pool, then chooses the accepted standing pose closest to the current
# robot state. Historical configs keep this disabled so their targets can
# still be reconstructed exactly.
safe_stationary_anchors: bool = False
# Semantics v24 keeps the sampled idle clip's *small motion residual* but
# expresses it around the current safe anchor. This prevents two static
# standing poses with different absolute arm configurations from creating
# a fast synthetic arm swing when they are blended together.
stationary_idle_residual_retargeting: bool = False
# Opt-in target revision: enter a recorded listening pose and hold it;
# the following task blends from the actual last listen frame. False
# preserves old checkpoints and immutable ablation fixtures exactly.
stationary_listen_hold: bool = False
# Blend the current safe standing state into the *moving* task prefix.
# This avoids holding a locomotion clip's arbitrary first (often
# single-support) frame for one second before suddenly restoring velocity.
dynamic_task_entry_blend: bool = False
# Versions 13/14 group primitive commands into one uninterrupted interaction.
# The complete item owns one audio background and one Qwen KV lifetime.
multi_turn_sessions: bool = False
session_min_turns: int = 2
session_max_turns: int = 4
# Version 17 creates static train-session variants. Every primitive is
# used once per variant with one shared speaker/language identity across
# all turns. Validation always keeps one deterministic variant.
# Zero retains the historical per-record voice policy exactly.
session_voice_variants: int = 0
# Version 20 uses the two existing train variants as orthogonal behavior
# variants: one complete turn-taking session and one interruption session.
# Every primitive therefore contributes once to each behavior (exact 1:1
# primitive supervision); interrupted timelines are naturally shorter.
# Validation deterministically builds both modes.
interruption_sessions: bool = False
interruption_ratio: float = 0.5
interruption_min_start_seconds: float = 2.0
interruption_max_start_seconds: float = 6.0
interruption_min_remaining_seconds: float = 1.5
# Version 21 keeps the complete two-variant turn-taking-v1 pool intact and
# constructs every additional behaviour as an independent pool. Integer
# variant counts are direct, auditable data weights: 2:2 is exactly 1:1.
independent_task_pools: bool = False
turn_taking_session_variants: int = 2
command_interruption_session_variants: int = 2
emergency_stop_session_variants: int = 1
safe_hold_session_variants: int = 1
# Backchannel is an independent hard-negative pool: short same-speaker
# acknowledgements overlap an active motion without changing its action,
# progress, RTC prefix or </motion> target.
backchannel_session_variants: int = 0
# Semantics v23 adds two independent listen-side hard cases without
# changing the established turn-taking/interruption/Stop/backchannel pools.
# ``pause`` inserts background-only time inside one unfinished command;
# ``hesitation`` inserts a filled pause or an incomplete false start plus a
# repair cue. Both keep </listen> until the final intent is complete.
hesitation_session_variants: int = 0
pause_session_variants: int = 0
# Optional final-session target applied after all context/background and
# task-eligibility filtering. Zero preserves v21 integer-pool behavior;
# production v22 balances the actual sampled sessions, not merely raw
# primitive uses.
interruption_task_session_ratio: float = 0.0
# Legacy v21 runs leave these at zero/one and therefore reproduce their
# original integer-variant weighting. The v22 launcher keeps hold at zero,
# assigns Stop and backchannel their requested final-session shares, and
# retains follow-up support for the selected Stop sessions.
control_task_session_ratio: float = 0.0
backchannel_task_session_ratio: float = 0.0
# These are final *stream-window* shares. v23 first reconstructs the exact
# v22 selected session set, then appends enough new sessions to target 12%
# hesitation and 8% pause by estimated optimizer work.
hesitation_task_window_ratio: float = 0.0
pause_task_window_ratio: float = 0.0
control_followup_ratio: float = 0.0
control_post_idle_min_steps: int = 1
# Zero preserves historical behavior (the maximum follows
# ``initial_idle_max_steps``). Production Stop-persistence training sets
# this to 20 Qwen decisions, i.e. 8 seconds at the 0.4 s cadence.
control_post_idle_max_steps: int = 0
# Historical checkpoints sampled the explicit range uniformly. The
# balanced policy keeps the same 1.2--8 s support but makes ordinary short
# pauses common and long persistence examples deliberately rare, avoiding
# a large one-sided shift toward </idle>.
control_post_idle_sampling: str = "legacy_uniform"
# Emergency stop has a stricter timing contract than ordinary command
# interruption. The command must arrive while the counterfactual source
# motion is still active and must leave a real moving tail after the full
# utterance has become observable. Historical run configs keep the old
# values and disabled activity selector for exact checkpoint replay.
control_min_start_seconds: float = 2.0
control_max_start_seconds: float = 6.0
control_min_remaining_seconds: float = 0.4
control_activity_aware_timing: bool = False
# A small audited subset of the Chinese TTS corpus mistranslates English
# ``retro`` as spatial ``backward``. When enabled, only those contradictory
# Chinese renditions are removed; the matching English voices and motions
# remain available.
filter_known_tts_translation_mismatches: bool = False
# Optional data ablation: a generated, audited manifest selects exactly
# one motion trajectory for each exact English/Chinese instruction pair.
# Empty keeps the full many-to-one corpus and is the formal default.
canonical_motion_manifest: str = ""
control_command_manifest: str = (
"/cpfs01/thscc/sharestorage/AIGC-06-01/Hithink_VideoGen_Group/user_workspace/yetinghong/videogen/"
"hfd_control_commands_v1/manifest.jsonl"
)
backchannel_command_manifest: str = (
"/cpfs01/thscc/sharestorage/AIGC-06-01/Hithink_VideoGen_Group/user_workspace/yetinghong/videogen/"
"hfd_backchannel_commands_v2_short/manifest.jsonl"
)
hesitation_pause_manifest: str = (
"/cpfs01/thscc/sharestorage/AIGC-06-01/Hithink_VideoGen_Group/user_workspace/yetinghong/videogen/"
"hfd_hesitation_pause_v1/manifest.jsonl"
)
# New pools cap duplicate motion rows for one exact bilingual instruction.
# Existing task pools remain byte-for-byte unchanged. Selection is
# deterministic and duration-stratified rather than first-N.
new_task_max_rows_per_instruction_train: int = 32
new_task_max_rows_per_instruction_val: int = 8
# Filled hesitation versus self-repair. Silent intra-command pauses are a
# separate task and therefore do not consume this split.
hesitation_filled_ratio: float = 0.60
# Among self-repair hesitation sessions, omit an explicit ``I mean`` /
# ``改成`` cue for this fraction. The false-start prefix is followed by a
# pause and the complete final command, matching natural cue-less restarts.
self_repair_implicit_ratio: float = 0.0
backchannel_min_start_seconds: float = 1.2
backchannel_max_start_seconds: float = 6.0
backchannel_min_remaining_seconds: float = 1.2
safe_hold_stability_window_frames: int = 20
# Motion -> standing and standing -> the next task use a slower one-second
# transition. Version 13 retains the original 20-frame first-task entry;
# causal-timing version 14 uses this transition for every task entry and
# the stationary/listen envelope as well.
turn_transition_frames: int = 50
# Version 14 starts the clean speech waveform first, keeps the target in
# idle through the first Qwen boundary containing real voice onset, keeps
# one explicit response interval, and then starts </listen>. Historical
# runs keep this false so their checkpoint contract stays reproducible.
causal_listen_timing: bool = False
# Expected leading-silence budget recorded in the train/infer contract.
# Longer valid TTS silence is measured and retained; source-duration bounds
# are used for conservative sampler/context estimates. Three steps are
# 1.2 seconds.
max_speech_onset_steps: int = 3
# After the Qwen boundary that first contains sustained voice energy, keep
# one more 0.4 s idle response interval before emitting </listen>.
listen_reaction_steps: int = 1
# Qwen2.5-Omni-3B has a 32768-token context. A first 0.6 s window uses 82
# tokens and every later one uses 17, so 1923 complete windows fit while
# preserving every source-local KV entry.
max_stream_steps: int = 1923
# Drop only the deterministic longest tail after the final task mixture is
# formed. Zero preserves historical data; production v22 uses 0.001.
long_session_drop_ratio: float = 0.0
def __call__(self, split: str = "train", transform_kwargs=None, **kwargs) -> Any:
from psi.data.seed.full_duplex_audio_command_dataset import (
FullDuplexAudioCommandDataset,
FullDuplexTransformedDataset,
)
repack = self.transform.repack
action_chunk_size = int(getattr(repack, "action_chunk_size", 30))
prefix_frames = int(getattr(repack, "rtc_prefix_frames", 0))
expected_prefix = action_chunk_size - int(self.action_exec_horizon)
if prefix_frames != expected_prefix:
raise ValueError(
"Full-duplex data/model RTC mismatch: "
f"repack_prefix={prefix_frames}, expected={expected_prefix}"
)
raw_dataset = FullDuplexAudioCommandDataset(
hidden_root=self.hidden_root,
metadata_jsonl=self.metadata_jsonl,
motion_metadata_csv=self.motion_metadata_csv,
motion_root=self.motion_root,
white_noise_hidden_dir=self.white_noise_hidden_dir,
background_hidden_root=self.background_hidden_root,
background_assignment_path=self.background_assignment_path,
continuous_audio_root=self.continuous_audio_root,
online_audio=self.online_audio,
background_source_name=self.background_source_name,
background_task_name=self.background_task_name,
split=split,
val_ratio=self.val_ratio,
split_seed=self.split_seed,
split_source_namespace=self.split_source_namespace,
source_name=self.source_name,
task_name=self.task_name,
action_chunk_size=action_chunk_size,
action_exec_horizon=self.action_exec_horizon,
motion_fps=self.motion_fps,
motion_metadata_fps=self.motion_metadata_fps,
audio_sampling_rate=self.audio_sampling_rate,
audio_window_seconds=self.audio_window_seconds,
audio_sample_step_frames=self.audio_sample_step_frames,
action_sample_step_frames=self.action_sample_step_frames,
motion_cache_size=self.motion_cache_size,
fixed_voice_per_record=self.fixed_voice_per_record,
max_voice_windows=self.max_voice_windows,
max_voice_median_ratio=self.max_voice_median_ratio,
deterministic_eval=self.deterministic_eval,
limit_sources=self.limit_sources,
max_motion_steps=self.max_motion_steps,
initial_idle_max_steps=self.initial_idle_max_steps,
stationary_blend_frames=self.stationary_blend_frames,
stationary_heading_alignment=self.stationary_heading_alignment,
safe_stationary_anchors=self.safe_stationary_anchors,
stationary_idle_residual_retargeting=(
self.stationary_idle_residual_retargeting
),
stationary_listen_hold=self.stationary_listen_hold,
dynamic_task_entry_blend=self.dynamic_task_entry_blend,
multi_turn_sessions=self.multi_turn_sessions,
session_min_turns=self.session_min_turns,
session_max_turns=self.session_max_turns,
session_voice_variants=self.session_voice_variants,
interruption_sessions=self.interruption_sessions,
interruption_ratio=self.interruption_ratio,
interruption_min_start_seconds=(
self.interruption_min_start_seconds
),
interruption_max_start_seconds=(
self.interruption_max_start_seconds
),
interruption_min_remaining_seconds=(
self.interruption_min_remaining_seconds
),
independent_task_pools=self.independent_task_pools,
turn_taking_session_variants=(
self.turn_taking_session_variants
),
command_interruption_session_variants=(
self.command_interruption_session_variants
),
emergency_stop_session_variants=(
self.emergency_stop_session_variants
),
safe_hold_session_variants=self.safe_hold_session_variants,
backchannel_session_variants=(
self.backchannel_session_variants
),
hesitation_session_variants=(
self.hesitation_session_variants
),
pause_session_variants=self.pause_session_variants,
interruption_task_session_ratio=(
self.interruption_task_session_ratio
),
control_task_session_ratio=self.control_task_session_ratio,
backchannel_task_session_ratio=(
self.backchannel_task_session_ratio
),
hesitation_task_window_ratio=(
self.hesitation_task_window_ratio
),
pause_task_window_ratio=self.pause_task_window_ratio,
control_followup_ratio=self.control_followup_ratio,
control_post_idle_min_steps=self.control_post_idle_min_steps,
control_post_idle_max_steps=self.control_post_idle_max_steps,
control_post_idle_sampling=self.control_post_idle_sampling,
control_min_start_seconds=self.control_min_start_seconds,
control_max_start_seconds=self.control_max_start_seconds,
control_min_remaining_seconds=(
self.control_min_remaining_seconds
),
control_activity_aware_timing=(
self.control_activity_aware_timing
),
filter_known_tts_translation_mismatches=(
self.filter_known_tts_translation_mismatches
),
canonical_motion_manifest=self.canonical_motion_manifest,
control_command_manifest=self.control_command_manifest,
backchannel_command_manifest=(
self.backchannel_command_manifest
),
hesitation_pause_manifest=self.hesitation_pause_manifest,
new_task_max_rows_per_instruction_train=(
self.new_task_max_rows_per_instruction_train
),
new_task_max_rows_per_instruction_val=(
self.new_task_max_rows_per_instruction_val
),
hesitation_filled_ratio=self.hesitation_filled_ratio,
self_repair_implicit_ratio=self.self_repair_implicit_ratio,
backchannel_min_start_seconds=(
self.backchannel_min_start_seconds
),
backchannel_max_start_seconds=(
self.backchannel_max_start_seconds
),
backchannel_min_remaining_seconds=(
self.backchannel_min_remaining_seconds
),
safe_hold_stability_window_frames=(
self.safe_hold_stability_window_frames
),
turn_transition_frames=self.turn_transition_frames,
causal_listen_timing=self.causal_listen_timing,
max_speech_onset_steps=self.max_speech_onset_steps,
listen_reaction_steps=self.listen_reaction_steps,
max_stream_steps=self.max_stream_steps,
long_session_drop_ratio=self.long_session_drop_ratio,
)
if self.third_party_root:
from psi.data.seed.third_party_dataset import ThirdPartyAugmentedDataset
selection = self.third_party_train_selection if split == "train" else self.third_party_val_selection
if not selection:
raise ValueError("Third-party data requires an explicit split selection")
raw_dataset = ThirdPartyAugmentedDataset(raw_dataset, self.third_party_root, selection, split)
if self.multimodal_overlay_manifests:
if not self.swa_visual_qa_enabled:
raise ValueError("Unified overlay requires the AV/independent QA model")
from psi.data.seed.live_multimodal_overlay import LiveMultimodalOverlayDataset
from psi.data.seed.hfd_multimodal_overlay import MultimodalOverlayPolicy
raw_dataset = LiveMultimodalOverlayDataset(
raw_dataset, self.multimodal_overlay_manifests, split,
policy=MultimodalOverlayPolicy(
qa_ratio=self.multimodal_overlay_qa_ratio,
visual_only_ratio=self.multimodal_overlay_visual_only_ratio,
audio_only_ratio=1. - self.multimodal_overlay_qa_ratio - self.multimodal_overlay_visual_only_ratio),
seed=self.split_seed, strict_voice=self.multimodal_overlay_strict_voice,
visual_input_mode=self.multimodal_visual_input_mode)
audio_dataset = FullDuplexTransformedDataset(
raw_dataset,
self.transform,
transform_kwargs=transform_kwargs,
)
if self.multimodal_overlay_manifests or not self.swa_visual_qa_enabled:
return audio_dataset
if self.transform.repack.root40_actions:
raise ValueError("Root40 requires raw-pose unified overlay data, not historical pretransformed 39D SWA bodies")
from psi.data.seed.swa_visual_qa_dataset import (
MixedFullDuplexDataset,
SwaVisualQaDataset,
)
return MixedFullDuplexDataset(
audio_dataset,
SwaVisualQaDataset(
self.swa_source_plan,
split,
repeat=self.swa_train_repeat,
),
)
|