from __future__ import annotations import hashlib import contextlib import importlib.util import json import random import sqlite3 import sys import wave from collections import OrderedDict from pathlib import Path from types import SimpleNamespace import numpy as np import pytest import torch import torch.nn.functional as F from torch import nn from safetensors.torch import save_file from psi.config.transform import SeedPosttrainRepackTransform from psi.config.model_full_duplex_psi0 import FullDuplexPsi0ModelConfig from psi.data.seed.full_duplex_audio_command_dataset import ( AudioSource, BackgroundWaveformSource, CANONICAL_MOTION_MANIFEST_FORMAT, DISFLUENCY_UNCERTAINTY_ROLES, DisfluencyAudioSource, FullDuplexAudioCommandDataset, FullDuplexTransformedDataset, HiddenSlice, IDLE_TOKEN_ID, LISTEN_TOKEN_ID, MOTION_TOKEN_ID, SourceBackgroundAssignment, StationaryMotionSource, detect_clean_speech_onset_sample, has_known_tts_translation_mismatch, instruction_text_for_voice, load_canonical_motion_selection, ) from psi.data.seed.full_duplex_audio_contract import ( AUDIO_COMMAND_STREAMING_PROMPT, HFD_AUDIO_SOURCE_FORMAT, HFD_ONLINE_ENCODER_ATTENTION, HFD_ONLINE_ENCODER_BATCH_SIZE, HFD_ONLINE_ENCODER_CONTRACT, HFD_ONLINE_ENCODER_DTYPE, ) from psi.models.full_duplex_psi0 import FullDuplexPsi0Model, StreamingCausalMemory from psi.trainers.posttrain_full_duplex import ( PosttrainFullDuplexTrainer, SourceBatchingLoader, SourceLengthGroupedSampler, _source_prefetch_factor, average_full_duplex_gradients, ) from psi.data.seed.full_duplex_timing import ( aligned_motion_start_frame, complete_audio_clip_starts, ) from psi.data.seed.motion_repr import ( local_delta_from_world, quat_to_yaw_wxyz, rpy_to_quat_wxyz, wrap_angle, ) from psi.data.seed.source_split import select_source_row_ids def test_train_and_eval_are_disjoint_at_complete_source_level(): source_ids = ["source-a", "source-a", "source-b", "source-c", "source-c"] train_rows = select_source_row_ids( source_ids, split="train", val_ratio=0.25, split_seed=42 ) eval_rows = select_source_row_ids( source_ids, split="val", val_ratio=0.25, split_seed=42 ) train_sources = {source_ids[index] for index in train_rows} eval_sources = {source_ids[index] for index in eval_rows} assert train_sources.isdisjoint(eval_sources) def test_clean_speech_onset_ignores_leading_silence_and_pcm_click(): rate = 16_000 waveform = np.zeros(rate, dtype=np.float32) waveform[800] = 0.8 start = 4_000 time_axis = np.arange(rate - start, dtype=np.float32) / rate waveform[start:] = 0.1 * np.sin(2.0 * np.pi * 220.0 * time_axis) onset = detect_clean_speech_onset_sample(waveform, rate) assert start - 320 <= onset <= start + 160 def test_latest_instruction_text_matches_selected_tts_language(): record = { "row_id": 7, "instruction": "walk to the left", "instruction_zh": "向左走", } english = AudioSource( source_key="seed_audio/000007/female_01_en", clips=(), ) chinese = AudioSource( source_key="seed_audio/000007/male_01_zh", clips=(), ) assert instruction_text_for_voice(record, english) == "walk to the left" assert instruction_text_for_voice(record, chinese) == "向左走" def test_known_retro_translation_mismatch_excludes_only_chinese_voices(): assert has_known_tts_translation_mismatch( "retro dance jackson jump up", "后退舞步杰克逊跳起", ) assert not has_known_tts_translation_mismatch( "walk backward five steps", "后退五步", ) dataset = FullDuplexAudioCommandDataset.__new__( FullDuplexAudioCommandDataset ) dataset.audio_catalog = { "seed_audio/004603": ( AudioSource("seed_audio/004603/female_01_en", ()), AudioSource("seed_audio/004603/female_01_zh", ()), AudioSource("seed_audio/004603/male_01_en", ()), AudioSource("seed_audio/004603/male_01_zh", ()), ) } record = { "row_id": 1, "audio_dir": "seed_audio/004603", "_exclude_zh_voices": True, } assert dataset._session_voice_names(record) == ( "female_01_en", "male_01_en", ) def test_canonical_motion_manifest_loads_exact_bilingual_mapping(tmp_path): path = tmp_path / "canonical.json" path.write_text( json.dumps( { "format": CANONICAL_MOTION_MANIFEST_FORMAT, "excluded_ambiguous_chinese_instructions": [], "selections": [ { "instruction": "wave two hands in front", "instruction_zh": "双手在身前挥动", "filename": "wave_two_hands_R_001__A533", }, { "instruction": "walk forward", "instruction_zh": "向前走", "filename": "walk_forward_001__A100", }, ], }, ensure_ascii=False, ), encoding="utf-8", ) assert load_canonical_motion_selection(path) == { ( "wave two hands in front", "双手在身前挥动", ): "wave_two_hands_R_001__A533", ("walk forward", "向前走"): "walk_forward_001__A100", } def test_canonical_motion_manifest_rejects_duplicate_motion(tmp_path): path = tmp_path / "canonical.json" path.write_text( json.dumps( { "format": CANONICAL_MOTION_MANIFEST_FORMAT, "excluded_ambiguous_chinese_instructions": [], "selections": [ { "instruction": "first command", "instruction_zh": "第一条指令", "filename": "same_motion", }, { "instruction": "second command", "instruction_zh": "第二条指令", "filename": "same_motion", }, ], }, ensure_ascii=False, ), encoding="utf-8", ) with pytest.raises(RuntimeError, match="cannot represent two"): load_canonical_motion_selection(path) def test_canonical_motion_manifest_requires_ambiguous_chinese_exclusion( tmp_path, ): path = tmp_path / "canonical.json" path.write_text( json.dumps( { "format": CANONICAL_MOTION_MANIFEST_FORMAT, "excluded_ambiguous_chinese_instructions": [], "selections": [ { "instruction": "walk forward", "instruction_zh": "向前走", "filename": "walk_forward", }, { "instruction": "start walking forward", "instruction_zh": "向前走", "filename": "walk_forward_start", }, ], }, ensure_ascii=False, ), encoding="utf-8", ) with pytest.raises(RuntimeError, match="exclude every Chinese prompt"): load_canonical_motion_selection(path) def test_multiturn_partition_uses_every_split_record_once(): dataset = FullDuplexAudioCommandDataset.__new__( FullDuplexAudioCommandDataset ) dataset.multi_turn_sessions = True dataset.session_min_turns = 2 dataset.session_max_turns = 4 dataset.split = "train" dataset.split_seed = 42 records = [{"row_id": index} for index in range(23)] first = dataset._compose_sessions(records) second = dataset._compose_sessions(records) assert first == second assert all(2 <= len(session) <= 4 for session in first) flattened = [record["row_id"] for session in first for record in session] assert sorted(flattened) == list(range(23)) assert len(flattened) == len(set(flattened)) def _session_voice_fixture(split: str = "train"): dataset = FullDuplexAudioCommandDataset.__new__( FullDuplexAudioCommandDataset ) dataset.multi_turn_sessions = True dataset.session_min_turns = 2 dataset.session_max_turns = 4 dataset.session_voice_variants = 2 dataset.split = split dataset.split_seed = 42 records = [ {"row_id": index, "audio_dir": f"audio-{index}"} for index in range(32) ] voice_names = ( "female_01_en", "female_01_zh", "male_01_en", "male_01_zh", ) dataset.audio_catalog = { record["audio_dir"]: tuple( AudioSource( source_key=f"{record['audio_dir']}/{voice_name}", clips=(), packed_path=Path(f"/{record['audio_dir']}/{voice_name}"), packed_count=voice_index + 2, ) for voice_index, voice_name in enumerate(voice_names) ) for record in records } return dataset, records def test_one_to_four_partition_keeps_every_record_and_multiple_turn_lengths(): dataset, _ = _session_voice_fixture() dataset.session_min_turns = 1 dataset.session_voice_variants = 0 records = [{"row_id": i} for i in range(1000)] sessions = dataset._compose_sessions(records) assert sessions == dataset._compose_sessions(records) assert {len(s) for s in sessions} == {1, 2, 3, 4} assert sorted(r["row_id"] for s in sessions for r in s) == list(range(1000)) assert dataset._compose_sessions(records[:1]) == ((records[0],),) @pytest.mark.parametrize("split", ["train", "val"]) def test_single_turn_allowed_but_command_interruption_still_has_two_sources(split): dataset, records = _session_voice_fixture(split) dataset.session_min_turns = 1 dataset.interruption_sessions = True dataset.independent_task_pools = True dataset.interruption_min_start_seconds = 2. dataset.interruption_min_remaining_seconds = 1.5 dataset.motion_fps = 50 dataset.task_session_variants = { "turn-taking": 2, "interruption": 2, "emergency-stop": 2, "safe-hold": 0, "backchannel": 2, } dataset.control_task_session_ratio = .1 dataset.control_followup_ratio = .5 dataset._estimated_runtime_motion_frames = lambda r: 400 dataset._control_motion_is_eligible = lambda *args, **kwargs: True dataset._backchannel_motion_is_eligible = lambda *args, **kwargs: True sessions = dataset._compose_sessions(records) assert sessions == dataset._compose_sessions(records) for mode, repeats in dataset.task_session_variants.items(): selected = [s for s in sessions if s[0]["_session_interaction_mode"] == mode] count = repeats if split == "train" else int(repeats > 0) assert sorted(r["row_id"] for s in selected for r in s) == sorted(list(range(32))*count) if mode == "interruption": assert all(2 <= len(s) <= 4 for s in selected) elif selected: assert any(len(s) == 1 for s in selected) for session in selected: assert len({r["_session_voice_name"] for r in session}) == 1 if mode == "emergency-stop" and len(session) == 1: assert session[0]["_session_control_turn_index"] == 0 assert not session[0]["_session_control_has_followup"] def test_singleton_voice_bucket_does_not_require_a_different_speaker(): dataset, records = _session_voice_fixture("val") dataset.session_min_turns = 1 sessions = dataset._compose_sessions(records[:1]) assert len(sessions) == len(sessions[0]) == 1 assert sessions[0][0]["row_id"] == records[0]["row_id"] def test_train_session_voice_variants_fix_one_voice_and_expand_each_motion(): dataset, records = _session_voice_fixture("train") first = dataset._compose_sessions(records) second = dataset._compose_sessions(records) assert first == second assert sum(len(session) for session in first) == 2 * len(records) row_voices: dict[int, set[str]] = { int(record["row_id"]): set() for record in records } row_neighbours: dict[int, set[tuple[int, ...]]] = { int(record["row_id"]): set() for record in records } for session in first: assert 2 <= len(session) <= 4 assert len({record["_session_voice_name"] for record in session}) == 1 row_ids = tuple(sorted(int(record["row_id"]) for record in session)) for record in session: row_id = int(record["row_id"]) row_voices[row_id].add(str(record["_session_voice_name"])) row_neighbours[row_id].add(row_ids) selected = dataset._voice_for_record(record) assert Path(selected.source_key).name == record["_session_voice_name"] assert all(len(voices) == 2 for voices in row_voices.values()) assert any(len(neighbours) == 2 for neighbours in row_neighbours.values()) def test_validation_uses_one_deterministic_same_voice_session_variant(): dataset, records = _session_voice_fixture("val") sessions = dataset._compose_sessions(records) flattened = [record for session in sessions for record in session] assert len(flattened) == len(records) assert sorted(int(record["row_id"]) for record in flattened) == list( range(len(records)) ) assert all(record["_session_voice_variant"] == 0 for record in flattened) assert all( len({record["_session_voice_name"] for record in session}) == 1 for session in sessions ) @pytest.mark.parametrize("split", ["train", "val"]) def test_interruption_composition_is_exactly_one_primitive_use_per_mode(split): dataset, records = _session_voice_fixture(split) dataset.interruption_sessions = True dataset.interruption_ratio = 0.5 dataset.interruption_min_start_seconds = 2.0 dataset.interruption_max_start_seconds = 6.0 dataset.interruption_min_remaining_seconds = 1.5 dataset.motion_fps = 50 dataset._estimated_runtime_motion_frames = lambda record: ( 120 if int(record["row_id"]) % 9 == 0 else 400 ) first = dataset._compose_sessions(records) second = dataset._compose_sessions(records) assert first == second assert len( { session[0]["_session_interaction_mode"] for session in first[: min(8, len(first))] } ) == 2 by_mode: dict[str, list[dict]] = { "turn-taking": [], "interruption": [], } for session in first: modes = {record["_session_interaction_mode"] for record in session} voices = {record["_session_voice_name"] for record in session} assert len(modes) == len(voices) == 1 mode = modes.pop() by_mode[mode].extend(session) assert 2 <= len(session) <= 4 if mode == "interruption": assert all( dataset._interruption_motion_is_eligible(record) for record in session[:-1] ) expected_rows = list(range(len(records))) assert sorted(record["row_id"] for record in by_mode["turn-taking"]) == ( expected_rows ) assert sorted(record["row_id"] for record in by_mode["interruption"]) == ( expected_rows ) if split == "train": row_voices = { row_id: { record["_session_voice_name"] for mode_records in by_mode.values() for record in mode_records if int(record["row_id"]) == row_id } for row_id in expected_rows } assert all(len(voices) == 2 for voices in row_voices.values()) else: row_voices = { row_id: [ record["_session_voice_name"] for mode_records in by_mode.values() for record in mode_records if int(record["row_id"]) == row_id ] for row_id in expected_rows } assert all(len(set(voices)) == 1 for voices in row_voices.values()) def test_v21_independent_pools_preserve_complete_turn_taking_reference(): reference, reference_records = _session_voice_fixture("train") reference_sessions = reference._compose_sessions(reference_records) dataset, records = _session_voice_fixture("train") dataset.interruption_sessions = True dataset.interruption_ratio = 0.5 dataset.interruption_min_start_seconds = 2.0 dataset.interruption_max_start_seconds = 6.0 dataset.interruption_min_remaining_seconds = 1.5 dataset.motion_fps = 50 dataset.independent_task_pools = True dataset.task_session_variants = { "turn-taking": 2, "interruption": 2, "emergency-stop": 1, "safe-hold": 1, } dataset._estimated_runtime_motion_frames = lambda record: 400 dataset._control_motion_is_eligible = lambda *args, **kwargs: True sessions = dataset._compose_sessions(records) by_mode = { mode: [ session for session in sessions if session[0]["_session_interaction_mode"] == mode ] for mode in dataset.task_session_variants } def signature(values): return sorted( ( int(session[0]["_session_voice_variant"]), str(session[0]["_session_voice_name"]), tuple(int(record["row_id"]) for record in session), ) for session in values ) assert signature(by_mode["turn-taking"]) == signature(reference_sessions) assert sorted( int(session[0]["_session_pool_index"]) for session in by_mode["turn-taking"] ) == list(range(len(reference_sessions))) primitive_uses = { mode: sum(len(session) for session in values) for mode, values in by_mode.items() } assert primitive_uses == { "turn-taking": 2 * len(records), "interruption": 2 * len(records), "emergency-stop": len(records), "safe-hold": len(records), } assert dataset.control_task_rows_filtered == { "emergency-stop": 0, "safe-hold": 0, } def test_independent_pool_reuses_voice_only_after_full_coverage(): dataset, records = _session_voice_fixture("train") dataset.interruption_sessions = True dataset.interruption_ratio = 0.5 dataset.interruption_min_start_seconds = 2.0 dataset.interruption_max_start_seconds = 6.0 dataset.interruption_min_remaining_seconds = 1.5 dataset.motion_fps = 50 dataset.independent_task_pools = True dataset.task_session_variants = { "turn-taking": 5, "interruption": 1, "emergency-stop": 0, "safe-hold": 0, } dataset._estimated_runtime_motion_frames = lambda record: 400 sessions = dataset._compose_sessions(records) turn_rows = [ record for session in sessions if session[0]["_session_interaction_mode"] == "turn-taking" for record in session ] by_row = { row_id: [ str(record["_session_voice_name"]) for record in turn_rows if int(record["row_id"]) == row_id ] for row_id in range(len(records)) } assert all(len(voices) == 5 for voices in by_row.values()) assert all(len(set(voices)) == 4 for voices in by_row.values()) assert dataset.voice_reuse_after_coverage["turn-taking"] == len(records) assert dataset.voice_reuse_after_coverage["interruption"] == 0 def test_balanced_control_pools_are_eight_percent_with_one_event_per_session(): dataset, records = _session_voice_fixture("train") dataset.interruption_sessions = True dataset.interruption_ratio = 0.5 dataset.interruption_min_start_seconds = 2.0 dataset.interruption_max_start_seconds = 6.0 dataset.interruption_min_remaining_seconds = 1.5 dataset.motion_fps = 50 dataset.independent_task_pools = True dataset.task_session_variants = { "turn-taking": 2, "interruption": 2, "emergency-stop": 1, "safe-hold": 1, } dataset.control_task_session_ratio = 0.08 dataset.control_followup_ratio = 0.5 dataset._estimated_runtime_motion_frames = lambda record: 400 dataset._control_motion_is_eligible = lambda *args, **kwargs: True unbalanced = dataset._compose_sessions(records) sessions = dataset._balance_control_task_sessions(unbalanced) counts = { mode: sum( session[0]["_session_interaction_mode"] == mode for session in sessions ) for mode in dataset.task_session_variants } assert counts["turn-taking"] + counts["interruption"] > 0 assert counts["emergency-stop"] == counts["safe-hold"] assert counts["emergency-stop"] / len(sessions) == pytest.approx( 0.08, abs=0.01, ) control_sessions = [ session for session in sessions if session[0]["_session_interaction_mode"] in {"emergency-stop", "safe-hold"} ] for session in control_sessions: turn_indices = { int(record["_session_control_turn_index"]) for record in session } followup_flags = { bool(record["_session_control_has_followup"]) for record in session } assert len(turn_indices) == len(followup_flags) == 1 control_turn = turn_indices.pop() has_followup = followup_flags.pop() assert has_followup == (control_turn + 1 < len(session)) assert any( session[0]["_session_control_has_followup"] for session in control_sessions ) assert any( not session[0]["_session_control_has_followup"] for session in control_sessions ) def test_v22_backchannel_final_session_mix_is_approximately_60_15_10_15(): dataset, records = _session_voice_fixture("train") dataset.interruption_sessions = True dataset.interruption_ratio = 0.5 dataset.interruption_min_start_seconds = 2.0 dataset.interruption_max_start_seconds = 6.0 dataset.interruption_min_remaining_seconds = 1.5 dataset.motion_fps = 50 dataset.independent_task_pools = True dataset.task_session_variants = { "turn-taking": 4, "interruption": 1, "emergency-stop": 2, "safe-hold": 0, "backchannel": 2, } dataset.interruption_task_session_ratio = 0.15 dataset.control_task_session_ratio = 0.10 dataset.backchannel_task_session_ratio = 0.15 dataset.control_followup_ratio = 0.5 dataset._estimated_runtime_motion_frames = lambda record: 400 dataset._control_motion_is_eligible = lambda *args, **kwargs: True dataset._backchannel_motion_is_eligible = lambda *args, **kwargs: True sessions = dataset._balance_backchannel_task_sessions( dataset._compose_sessions(records) ) counts = { mode: sum( session[0]["_session_interaction_mode"] == mode for session in sessions ) for mode in ( "turn-taking", "interruption", "emergency-stop", "backchannel", ) } assert counts["turn-taking"] / len(sessions) == pytest.approx( 0.60, abs=0.02, ) assert counts["interruption"] / len(sessions) == pytest.approx( 0.15, abs=0.02, ) assert counts["emergency-stop"] / len(sessions) == pytest.approx( 0.10, abs=0.02, ) assert counts["backchannel"] / len(sessions) == pytest.approx( 0.15, abs=0.02, ) for session in sessions: if session[0]["_session_interaction_mode"] != "backchannel": continue event_turns = { int(record["_session_backchannel_turn_index"]) for record in session } assert len(event_turns) == 1 assert 0 <= event_turns.pop() < len(session) def test_v23_appends_disfluency_windows_without_changing_v22_selection(): dataset = FullDuplexAudioCommandDataset.__new__( FullDuplexAudioCommandDataset ) dataset.split = "train" dataset.split_seed = 42 dataset.task_session_variants = { "turn-taking": 4, "interruption": 1, "emergency-stop": 1, "safe-hold": 0, "backchannel": 2, "hesitation": 3, "pause": 2, } dataset.interruption_task_session_ratio = 0.15 dataset.control_task_session_ratio = 0.10 dataset.backchannel_task_session_ratio = 0.15 dataset.hesitation_task_window_ratio = 0.0 dataset.pause_task_window_ratio = 0.0 dataset._cached_session_max_stream_steps = lambda session: int( session[0]["estimated_steps"] ) def sessions(mode: str, count: int, *, offset: int): return tuple( ( { "row_id": offset + index, "_session_interaction_mode": mode, "estimated_steps": 10, }, ) for index in range(count) ) legacy_pool = ( *sessions("turn-taking", 100, offset=0), *sessions("interruption", 80, offset=1_000), *sessions("emergency-stop", 80, offset=2_000), *sessions("backchannel", 80, offset=3_000), ) v22 = dataset._balance_backchannel_task_sessions(legacy_pool) v22_ids = { int(session[0]["row_id"]) for session in v22 } dataset.hesitation_task_window_ratio = 0.12 dataset.pause_task_window_ratio = 0.08 v23 = dataset._balance_backchannel_task_sessions( ( *legacy_pool, *sessions("hesitation", 80, offset=4_000), *sessions("pause", 80, offset=5_000), ) ) legacy_ids_after = { int(session[0]["row_id"]) for session in v23 if session[0]["_session_interaction_mode"] not in {"hesitation", "pause"} } by_mode_windows = { mode: sum( int(session[0]["estimated_steps"]) for session in v23 if session[0]["_session_interaction_mode"] == mode ) for mode in ("hesitation", "pause") } total_windows = sum( int(session[0]["estimated_steps"]) for session in v23 ) assert legacy_ids_after == v22_ids assert by_mode_windows["hesitation"] / total_windows == pytest.approx( 0.12, abs=0.01, ) assert by_mode_windows["pause"] / total_windows == pytest.approx( 0.08, abs=0.01, ) def test_new_disfluency_rows_are_duration_stratified_and_exact_prompt_capped(): dataset = FullDuplexAudioCommandDataset.__new__( FullDuplexAudioCommandDataset ) dataset.split = "train" dataset.session_min_turns = 2 dataset.new_task_max_rows_per_instruction_train = 3 dataset.new_task_max_rows_per_instruction_val = 2 dataset.disfluency_instruction_segments = { ("seed_audio/shared", "female_01_en"): {"prefix": object(), "suffix": object()} } dataset._session_voice_names = lambda record: ("female_01_en",) records = [ { "row_id": index, "audio_dir": "seed_audio/shared", "instruction": "walk forward", "instruction_zh": "向前走", "actual_frames": frames, } for index, frames in enumerate((10, 20, 30, 40, 50)) ] selected = dataset._select_new_task_records(records) assert [record["actual_frames"] for record in selected] == [10, 30, 50] assert dataset.new_task_record_selection_summary == { "input_rows": 5, "eligible_rows": 5, "selected_rows": 3, "instruction_pairs": 1, "cap_per_instruction": 3, "unsupported_rows": 0, } def _write_test_pcm16(path: Path, values: np.ndarray, rate: int = 16_000) -> None: path.parent.mkdir(parents=True, exist_ok=True) pcm = np.clip(np.asarray(values), -0.999, 0.999) pcm = np.rint(pcm * 32767.0).astype(" DisfluencyAudioSource: with wave.open(str(path), "rb") as handle: count = handle.getnframes() return DisfluencyAudioSource( category=category, segment_id=segment_id, split="train" if category != "instruction_segment" else "shared", voice_name=voice_name, language="en", text=text, waveform_path=path, speech_samples=count, segment_role=role, ) final_roles = { "prefix": segment( "instruction_segment", "final-prefix", prefix_path, "walk", role="prefix" ), "suffix": segment( "instruction_segment", "final-suffix", suffix_path, "forward", role="suffix" ), } provisional_roles = { "prefix": segment( "instruction_segment", "provisional-prefix", provisional_path, "pick up", role="prefix", ), "suffix": segment( "instruction_segment", "unused-provisional-suffix", suffix_path, "the box", role="suffix", ), } filler = segment("hesitation_filler", "fill-1", filler_path, "um") repair = segment("repair_cue", "repair-1", repair_path, "I mean") dataset = FullDuplexAudioCommandDataset.__new__( FullDuplexAudioCommandDataset ) dataset.split = "train" dataset.audio_sampling_rate = rate dataset.motion_fps = 50 dataset.audio_sample_step_frames = 20 dataset.audio_window_seconds = 0.6 assert dataset.audio_stride_samples == stride assert dataset.audio_window_samples == 9_600 dataset.disfluency_instruction_segments = { ("seed_audio/final", voice_name): final_roles, ("seed_audio/provisional", voice_name): provisional_roles, } dataset.disfluency_marker_catalog = { voice_name: { "hesitation_filler": (filler,), "repair_cue": (repair,), } } voice = AudioSource( source_key=f"seed_audio/final/{voice_name}", clips=(), waveform_path=final_path, waveform_count=1, ) assignment = SourceBackgroundAssignment( source_key=voice.source_key, background_key="office", listen_start_index=3, listen_window_count=1, background_window_count=100, initial_idle_max_steps=7, max_motion_steps=0, safety_steps=2, mixed_audio_path=tmp_path / "unused.wav", original_audio_path=final_path, speech_samples=1_500, speech_scale=1.0, ) record = { "audio_dir": "seed_audio/final", "instruction": "walk forward", "instruction_zh": "向前走", } _, pause_assignment, pause_speech, pause_metadata = ( dataset._build_disfluency_turn_speech( record=record, voice=voice, assignment=assignment, subtype="silent-pause", gap_steps=2, ) ) pause_roles = [row["role"] for row in pause_metadata["components"]] assert pause_roles == [ "final-prefix", "background-only-pause", "final-suffix", ] assert pause_speech.size == 1_000 + 2 * stride + 1_200 np.testing.assert_allclose(pause_speech[1_000 : 1_000 + 2 * stride], 0.0) assert pause_assignment.speech_samples == pause_speech.size assert pause_assignment.listen_window_count == ( dataset._runtime_listen_window_count(pause_speech.size) ) _, _, repair_speech, repair_metadata = dataset._build_disfluency_turn_speech( record=record, voice=voice, assignment=assignment, subtype="self-repair", gap_steps=2, marker_id="repair-1", provisional_audio_dir="seed_audio/provisional", provisional_instruction="pick up the box", provisional_instruction_zh="拿起箱子", ) repair_roles = [row["role"] for row in repair_metadata["components"]] assert repair_roles == [ "provisional-prefix", "repair-background-before", "repair-cue", "repair-background-after", "complete-final-instruction", ] assert repair_speech.size == 900 + 2 * stride + 1_500 assert repair_metadata["provisional_instruction"] == "pick up the box" assert repair_metadata["final_instruction"] == "walk forward" _, _, implicit_speech, implicit_metadata = ( dataset._build_disfluency_turn_speech( record=record, voice=voice, assignment=assignment, subtype="self-repair", repair_style="implicit-restart", gap_steps=2, provisional_audio_dir="seed_audio/provisional", provisional_instruction="pick up the box", provisional_instruction_zh="拿起箱子", ) ) implicit_roles = [ row["role"] for row in implicit_metadata["components"] ] assert implicit_roles == [ "provisional-prefix", "repair-implicit-background", "complete-final-instruction", ] assert implicit_speech.size == 900 + 2 * stride + 1_500 assert implicit_metadata["marker_id"] is None assert implicit_metadata["marker_text"] == "" assert implicit_metadata["repair_style"] == "implicit-restart" assert "repair-implicit-background" in DISFLUENCY_UNCERTAINTY_ROLES def test_hesitation_pause_splitter_never_cuts_inside_domain_words(): module_path = ( Path(__file__).resolve().parents[1] / "scripts/data/generate_hfd_hesitation_pause.py" ) spec = importlib.util.spec_from_file_location( "hfd_test_hesitation_pause_generator", module_path, ) assert spec is not None and spec.loader is not None generator = importlib.util.module_from_spec(spec) sys.modules[spec.name] = generator spec.loader.exec_module(generator) pytest.importorskip("jieba") for text, protected in ( ("双臂交叉站立", "交叉"), ("拄拐顺时针缓慢走路", "顺时针"), ("向后退五步", "后退"), ): split = generator.split_chinese(text) assert split is not None prefix, suffix = split assert prefix + suffix == text boundary = len(prefix) protected_start = text.index(protected) assert not protected_start < boundary < protected_start + len(protected) def test_hesitation_pause_existing_index_is_relative_and_safe(tmp_path): module_path = ( Path(__file__).resolve().parents[1] / "scripts/data/generate_hfd_hesitation_pause.py" ) spec = importlib.util.spec_from_file_location( "hfd_test_hesitation_pause_resume_index", module_path, ) assert spec is not None and spec.loader is not None generator = importlib.util.module_from_spec(spec) sys.modules[spec.name] = generator spec.loader.exec_module(generator) index = tmp_path / "existing.txt" index.write_text( "audio/instruction_segments/000006/female_01_en_prefix.wav\n" "audio/markers/train/hes_fill_en_01/female_01_en.wav\n", encoding="utf-8", ) assert generator.load_existing_output_index(index, tmp_path) == { "audio/instruction_segments/000006/female_01_en_prefix.wav", "audio/markers/train/hes_fill_en_01/female_01_en.wav", } index.write_text("../outside.wav\n", encoding="utf-8") with pytest.raises(ValueError, match="Unsafe existing-output index"): generator.load_existing_output_index(index, tmp_path) def test_hesitation_pause_manifest_audit_rejects_runaway_tts(tmp_path): module_path = ( Path(__file__).resolve().parents[1] / "scripts/data/generate_hfd_hesitation_pause.py" ) spec = importlib.util.spec_from_file_location( "hfd_test_hesitation_pause_manifest_audit", module_path, ) assert spec is not None and spec.loader is not None generator = importlib.util.module_from_spec(spec) sys.modules[spec.name] = generator spec.loader.exec_module(generator) valid = tmp_path / "valid.wav" with wave.open(str(valid), "wb") as handle: handle.setnchannels(1) handle.setsampwidth(2) handle.setframerate(16_000) handle.writeframes(np.full(16_000, 1_024, dtype="= 0.8 for row in vocalized) assert all(float(row["min_duration_seconds"]) >= 0.4 for row in phrases) repair_styles = { str(row.get("repair_style", "explicit-cue")) for row in generator.REVIEW_EXAMPLES if row["subtype"] == "self-repair" } assert repair_styles == {"explicit-cue", "implicit-restart"} def test_longest_point_one_percent_is_dropped_inside_each_task_mode(): dataset = FullDuplexAudioCommandDataset.__new__( FullDuplexAudioCommandDataset ) dataset.long_session_drop_ratio = 0.001 sessions = tuple( ( { "row_id": mode_index * 2_000 + index, "_session_interaction_mode": mode, "estimated_steps": index + 1, }, ) for mode_index, mode in enumerate( ("turn-taking", "interruption", "emergency-stop") ) for index in range(1_000) ) dataset.sessions = sessions dataset._cached_session_max_stream_steps = lambda session: int( session[0]["estimated_steps"] ) dataset.long_session_filter_summary = {} retained = dataset._drop_longest_sessions_by_mode(sessions) assert len(retained) == len(sessions) - 3 for mode in ("turn-taking", "interruption", "emergency-stop"): values = [ session[0]["estimated_steps"] for session in retained if session[0]["_session_interaction_mode"] == mode ] assert len(values) == 999 assert max(values) == 999 assert dataset.long_session_filter_summary[mode]["dropped"] == 1 def test_control_command_only_uses_motion_with_complete_causal_tail(): dataset = FullDuplexAudioCommandDataset.__new__( FullDuplexAudioCommandDataset ) dataset.action_exec_horizon = 20 dataset.motion_fps = 50 dataset.turn_transition_frames = 50 dataset.interruption_min_start_seconds = 2.0 dataset.interruption_max_start_seconds = 6.0 dataset.control_command_catalog = { "emergency-stop": { "female_01_zh": ( SimpleNamespace(speech_samples=16_000), ) } } dataset._runtime_listen_window_count = lambda samples: 3 dataset._estimated_runtime_motion_frames = lambda record: int( record["frames"] ) # 100-frame onset + 60-frame complete command + 20-frame real tail. assert not dataset._control_motion_is_eligible( {"frames": 130}, intent="emergency-stop", voice_name="female_01_zh", ) assert dataset._control_motion_is_eligible( {"frames": 131}, intent="emergency-stop", voice_name="female_01_zh", ) start, diagnostics = dataset._sample_control_speech_start( np.zeros((180, 36), dtype=np.float32), reaction_frames=60, ) assert start == 100 assert diagnostics["minimum_remaining_frames"] == 20 def test_control_eligibility_respects_debug_motion_truncation(): dataset = FullDuplexAudioCommandDataset.__new__( FullDuplexAudioCommandDataset ) dataset.action_exec_horizon = 20 dataset.prefix_frames = 10 dataset.motion_fps = 50 dataset.turn_transition_frames = 50 dataset.dynamic_task_entry_blend = True dataset.control_min_start_seconds = 1.2 dataset.control_min_remaining_seconds = 2.0 dataset.control_command_catalog = { "emergency-stop": { "female_01_zh": (SimpleNamespace(speech_samples=16_000),) } } dataset._runtime_listen_window_count = lambda samples: 3 dataset._estimated_runtime_motion_frames = lambda record: 1_000 # Runtime truncation to 200 frames cannot contain a 60-frame onset, # 60-frame utterance and 100-frame post-command motion tail. dataset.max_motion_steps = 10 assert not dataset._control_motion_is_eligible( {}, intent="emergency-stop", voice_name="female_01_zh", ) dataset.max_motion_steps = 12 assert dataset._control_motion_is_eligible( {}, intent="emergency-stop", voice_name="female_01_zh", ) def test_activity_aware_stop_prefers_motion_over_stationary_tail(): dataset = FullDuplexAudioCommandDataset.__new__( FullDuplexAudioCommandDataset ) dataset.action_exec_horizon = 20 dataset.motion_fps = 50 dataset.turn_transition_frames = 50 dataset.interruption_min_start_seconds = 2.0 dataset.interruption_max_start_seconds = 6.0 dataset.control_min_start_seconds = 1.2 dataset.control_max_start_seconds = 4.0 dataset.control_min_remaining_seconds = 2.0 dataset.control_activity_aware_timing = True sequence = np.zeros((400, 36), dtype=np.float32) sequence[:, 32] = 1.0 # Real task motion is active from 1.0 s through 3.6 s, followed by a long # stationary tail. A 1.2 s command must be placed so both its onset and # its complete-response boundary still observe movement. sequence[50:181, 0] = np.arange(131, dtype=np.float32) * 0.02 sequence[181:, 0] = sequence[180, 0] random.seed(7) start, diagnostics = dataset._sample_control_speech_start( sequence, reaction_frames=60, ) assert 60 <= start <= 120 assert start + 60 + 100 <= len(sequence) assert diagnostics["active_candidate_count"] > 0 assert diagnostics["activity_fallback"] is False def test_safe_hold_waits_for_supported_low_velocity_pose(): dataset = FullDuplexAudioCommandDataset.__new__( FullDuplexAudioCommandDataset ) dataset.action_exec_horizon = 20 dataset.safe_hold_stability_window_frames = 20 sequence = np.zeros((260, 36), dtype=np.float32) sequence[:, 32] = 1.0 # The robot keeps moving through frame 159, then reaches and retains an # upright, supported pose. A safe hold must not freeze during that motion. sequence[:160, 0] = np.arange(160, dtype=np.float32) * 0.20 sequence[160:, 0] = sequence[159, 0] boundary = dataset._safe_hold_boundary(sequence, earliest_frame=100) assert boundary >= 180 assert boundary < len(sequence) def test_emergency_recovery_prefers_nearby_two_support_phase(): dataset = FullDuplexAudioCommandDataset.__new__( FullDuplexAudioCommandDataset ) dataset.safe_stationary_anchors = True dataset.action_exec_horizon = 20 dataset.turn_transition_frames = 50 dataset._stationary_joint_scale = np.ones(29, dtype=np.float32) neutral = np.zeros(36, dtype=np.float32) neutral[31] = 0.82 neutral[32] = 1.0 dataset._stationary_anchor_poses = neutral[None, :] sequence = np.repeat(neutral[None, :], 100, axis=0) # At the immediate stop boundary both leg chains are moving; shortly # afterwards they settle close to a two-support neutral pose. Recovery # searches every frame so it cannot miss a brief safe-support phase. sequence[:25, 0:12] = np.linspace( 0.0, 1.0, 25, dtype=np.float32 )[:, None] sequence[25:40, 0:12] = np.linspace( 1.0, 0.0, 15, dtype=np.float32 )[:, None] boundary, anchor = dataset._nearest_safe_recovery_boundary( sequence, earliest_frame=20, ) assert 39 <= boundary <= 41 np.testing.assert_allclose(anchor[:29], neutral[:29], atol=1e-6) def test_state_aware_serial_control_consumes_only_causal_physical_context(): torch.manual_seed(7) model = FullDuplexPsi0Model.__new__(FullDuplexPsi0Model) nn.Module.__init__(model) model.state_aware_serial_control = True model.action_dim = 3 memory_dim = 3 model.control_state_encoder = nn.Sequential( nn.LayerNorm(3), nn.Linear(3, memory_dim), nn.GELU() ) model.control_rtc_encoder = nn.Sequential( nn.LayerNorm(3), nn.Linear(3, memory_dim), nn.GELU() ) model.control_progress_encoder = nn.Sequential( nn.Linear(5, memory_dim), nn.GELU() ) model.previous_control_embedding = nn.Embedding(4, memory_dim) model.control_head = nn.Sequential( nn.LayerNorm(4 + 4 * memory_dim + 1), nn.Linear(4 + 4 * memory_dim + 1, memory_dim), nn.GELU(), nn.Linear(memory_dim, 3), ) semantic = torch.randn(1, 4, requires_grad=True) state = torch.tensor([[[0.1, -0.2, 0.3]]], requires_grad=True) progress = torch.tensor( [[0.25, 0.75, 0.25, 1.0, 0.0]], requires_grad=True ) rtc = torch.randn(1, 10, 3, requires_grad=True) rtc_mask = torch.ones(1, 10, dtype=torch.bool) logits = model._predict_control_logits( semantic, states=state, control_progress=progress, previous_control_ids=torch.tensor([MOTION_TOKEN_ID]), control_rtc_prefix=rtc, control_rtc_mask=rtc_mask, ) logits.square().sum().backward() assert logits.shape == (1, 3) for value in (semantic.grad, state.grad, progress.grad, rtc.grad): assert value is not None assert torch.isfinite(value).all() assert float(value.abs().sum()) > 0.0 # A masked source-start RTC contains no causal plan, so its numeric filler # cannot change the decision. source_start_a = model._predict_control_logits( semantic.detach(), states=state.detach(), control_progress=progress.detach(), previous_control_ids=torch.tensor([3]), control_rtc_prefix=torch.zeros_like(rtc), control_rtc_mask=torch.zeros_like(rtc_mask), ) source_start_b = model._predict_control_logits( semantic.detach(), states=state.detach(), control_progress=progress.detach(), previous_control_ids=torch.tensor([3]), control_rtc_prefix=torch.full_like(rtc, 999.0), control_rtc_mask=torch.zeros_like(rtc_mask), ) torch.testing.assert_close(source_start_a, source_start_b) with pytest.raises(ValueError, match="missing causal inputs"): model._predict_control_logits( semantic.detach(), states=state.detach(), control_progress=None, previous_control_ids=torch.tensor([3]), control_rtc_prefix=torch.zeros_like(rtc), control_rtc_mask=torch.zeros_like(rtc_mask), ) def test_semantic_only_serial_control_ignores_physical_context(): torch.manual_seed(11) model = FullDuplexPsi0Model.__new__(FullDuplexPsi0Model) nn.Module.__init__(model) model.state_aware_serial_control = False model.control_head = nn.Sequential( nn.LayerNorm(4), nn.Linear(4, 3), ) semantic = torch.randn(2, 4, requires_grad=True) logits_a = model._predict_control_logits( semantic, states=torch.randn(2, 1, 39), control_progress=torch.randn(2, 5), previous_control_ids=torch.tensor([0, 2]), control_rtc_prefix=torch.randn(2, 10, 39), control_rtc_mask=torch.ones(2, 10, dtype=torch.bool), ) logits_b = model._predict_control_logits( semantic, states=None, control_progress=None, previous_control_ids=None, control_rtc_prefix=None, control_rtc_mask=None, ) torch.testing.assert_close(logits_a, logits_b) logits_b.sum().backward() assert semantic.grad is not None assert float(semantic.grad.abs().sum()) > 0.0 def test_interruption_source_continues_old_motion_then_transitions_causally( monkeypatch, ): dataset = FullDuplexAudioCommandDataset.__new__( FullDuplexAudioCommandDataset ) records = tuple( { "row_id": row_id, "filename": f"motion-{row_id}", "instruction": f"instruction-{row_id}", "audio_dir": f"audio-{row_id}", "motion_dir": Path(f"/unused/motion-{row_id}"), "declared_frames": 350, "_session_interaction_mode": "interruption", } for row_id in (3, 7) ) dataset.records = list(records) dataset.sessions = (records,) dataset.multi_turn_sessions = True dataset.interruption_sessions = True dataset.interruption_ratio = 0.5 dataset.interruption_min_start_seconds = 2.0 dataset.interruption_max_start_seconds = 6.0 dataset.interruption_min_remaining_seconds = 1.5 dataset.session_min_turns = 2 dataset.session_max_turns = 4 dataset.fixed_voice_per_record = True dataset.action_chunk_size = 30 dataset.action_exec_horizon = 20 dataset.prefix_frames = 10 dataset.audio_sample_step_frames = 20 dataset.action_sample_step_frames = 1 dataset.audio_sampling_rate = 16_000 dataset.motion_fps = 50 dataset.audio_window_seconds = 0.6 dataset.initial_idle_max_steps = 7 dataset.stationary_blend_frames = 20 dataset.turn_transition_frames = 50 dataset.causal_listen_timing = True dataset.max_speech_onset_steps = 3 dataset.listen_reaction_steps = 1 dataset.max_motion_steps = 0 dataset.max_stream_steps = 1923 voices = tuple( AudioSource( source_key=f"audio-{row_id}/voice", clips=(), waveform_path=Path(f"/unused/voice-{row_id}.wav"), waveform_count=7, ) for row_id in (3, 7) ) dataset.audio_catalog = { "audio-3": (voices[0],), "audio-7": (voices[1],), } dataset.background_assignments = { voice.source_key: SourceBackgroundAssignment( source_key=voice.source_key, background_key="office", listen_start_index=20, listen_window_count=7, background_window_count=400, initial_idle_max_steps=7, max_motion_steps=30, safety_steps=5, original_audio_path=Path(f"/unused/{voice.source_key}.wav"), speech_samples=39_680, speech_scale=0.5, ) for voice in voices } motions = {} for record_index, record in enumerate(records): motion = np.zeros((350, 36), dtype=np.float32) motion[:, 0] = np.linspace(0.0, 0.4 + record_index, 350) motion[:, 29] = np.linspace(0.0, 4.0 + record_index, 350) motion[:, 31] = 0.8 motion[:, 32] = 1.0 motions[str(record["motion_dir"])] = motion monkeypatch.setattr(dataset, "_load_motion", lambda path: motions[str(path)]) monkeypatch.setattr( dataset, "_sample_stationary_anchor", lambda pose: np.asarray(pose, dtype=np.float32).copy(), ) monkeypatch.setattr( dataset, "_sample_stationary_sequence", lambda phase, frame_count, anchor_pose, **kwargs: ( np.repeat( np.asarray(anchor_pose, dtype=np.float32)[None, :], frame_count, axis=0, ), f"{phase}-fixture", ), ) clean_speech = np.zeros(39_680, dtype=np.float32) clean_speech[17_120:] = 0.1 monkeypatch.setattr( "psi.data.seed.full_duplex_audio_command_dataset._read_resampled_speech_float", lambda path, expected_rate: clean_speech.copy(), ) monkeypatch.setattr( dataset, "_build_session_audio_windows", lambda *, turn_audio, semantic_window_count: ( tuple( np.full(9600, float(index), dtype=np.float32) for index in range(semantic_window_count) ), "office", 100, ), ) monkeypatch.setattr( dataset, "_sample_interruption_prefix_frames", lambda task_sequence, next_reaction_frames: 300, ) random.seed(20) source = dataset[0] assert source["interaction_mode"] == "interruption" assert len(source["interruption_start_frames"]) == 1 speech_start = source["turn_speech_start_frames"][1] listen_start = source["turn_listen_start_frames"][1] assert speech_start == source["interruption_start_frames"][0] first_motion_start = source["turn_listen_start_frames"][0] + 7 * 20 assert 100 <= speech_start - first_motion_start <= 300 assert listen_start - speech_start == 100 assert all( source["steps"][frame]["control_label"] == MOTION_TOKEN_ID for frame in range(speech_start, listen_start) ) assert source["steps"][listen_start]["control_label"] == LISTEN_TOKEN_ID assert source["steps"][listen_start]["control_transition"] is True np.testing.assert_allclose( source["steps"][listen_start - 1]["actions"][0], source["steps"][listen_start]["states"][0], atol=1e-6, ) original_before = source["interruption_original_frames"][0] assert source["interruption_remaining_frames"][0] >= 75 assert source["interruption_terminal_hold_frames"] == [1] expected_progress = (original_before + 2) / 349.0 assert source["steps"][speech_start]["future_progress"][0, 0] == pytest.approx( expected_progress ) assert len(source["recovery_transition_frames"]) == 1 assert all( step["interaction_mode"] == "interruption" for step in source["steps"] ) def test_multiturn_source_keeps_one_cache_id_and_aligned_boundaries(monkeypatch): dataset = FullDuplexAudioCommandDataset.__new__( FullDuplexAudioCommandDataset ) records = tuple( { "row_id": row_id, "filename": f"motion-{row_id}", "instruction": f"instruction-{row_id}", "audio_dir": f"audio-{row_id}", "motion_dir": Path(f"/unused/motion-{row_id}"), "declared_frames": 96, } for row_id in (3, 7) ) dataset.records = list(records) dataset.sessions = (records,) dataset.multi_turn_sessions = True dataset.session_min_turns = 2 dataset.session_max_turns = 4 dataset.fixed_voice_per_record = True dataset.action_chunk_size = 30 dataset.action_exec_horizon = 20 dataset.prefix_frames = 10 dataset.audio_sample_step_frames = 20 dataset.action_sample_step_frames = 1 dataset.audio_sampling_rate = 16_000 dataset.motion_fps = 50 dataset.audio_window_seconds = 0.6 dataset.initial_idle_max_steps = 7 dataset.stationary_blend_frames = 20 dataset.turn_transition_frames = 50 dataset.causal_listen_timing = True dataset.max_speech_onset_steps = 3 dataset.listen_reaction_steps = 1 dataset.max_motion_steps = 0 dataset.max_stream_steps = 1923 voices = tuple( AudioSource( source_key=f"audio-{row_id}/voice", clips=(), waveform_path=Path(f"/unused/voice-{row_id}.wav"), waveform_count=7, ) for row_id in (3, 7) ) dataset.audio_catalog = { "audio-3": (voices[0],), "audio-7": (voices[1],), } dataset.background_assignments = { voice.source_key: SourceBackgroundAssignment( source_key=voice.source_key, background_key="office", listen_start_index=20, listen_window_count=7, background_window_count=400, initial_idle_max_steps=5, max_motion_steps=20, safety_steps=5, original_audio_path=Path(f"/unused/voice-{voice.source_key}.wav"), speech_samples=39_680, speech_scale=0.5, ) for voice in voices } motions = {} for record_index, record in enumerate(records): motion = np.zeros((40, 36), dtype=np.float32) motion[:, 0] = np.linspace(0.0, 0.2 + record_index, 40) motion[:, 29] = np.linspace(0.0, 0.5, 40) motion[:, 31] = 0.8 motion[:, 32] = 1.0 motions[str(record["motion_dir"])] = motion monkeypatch.setattr( dataset, "_load_motion", lambda path: motions[str(path)], ) monkeypatch.setattr( dataset, "_sample_stationary_anchor", lambda pose: np.asarray(pose, dtype=np.float32).copy(), ) monkeypatch.setattr( dataset, "_sample_stationary_sequence", lambda phase, frame_count, anchor_pose, **kwargs: ( np.repeat( np.asarray(anchor_pose, dtype=np.float32)[None, :], frame_count, axis=0, ), f"{phase}-fixture", ), ) # 1.07 s of leading silence needs four Qwen boundaries, deliberately # exceeding the historical expected budget of three. It is valid speech # and must extend the idle reaction phase instead of killing a worker. clean_speech = np.zeros(39_680, dtype=np.float32) clean_speech[17_120:] = 0.1 monkeypatch.setattr( "psi.data.seed.full_duplex_audio_command_dataset._read_resampled_speech_float", lambda path, expected_rate: clean_speech.copy(), ) def build_audio(*, turn_audio, semantic_window_count): assert len(turn_audio) == 2 windows = tuple( np.full(9600, float(index), dtype=np.float32) for index in range(semantic_window_count) ) return windows, "office", 100 monkeypatch.setattr(dataset, "_build_session_audio_windows", build_audio) random.seed(19) source = dataset[0] assert source["turn_count"] == 2 assert source["task_transition_frames"] == [50, 50] assert all(50 <= count <= 69 for count in source["recovery_transition_frames"]) assert all(frame % 20 == 0 for frame in source["turn_listen_start_frames"]) assert all( listen - speech == 100 for speech, listen in zip( source["turn_speech_start_frames"], source["turn_listen_start_frames"], ) ) assert source["turn_listen_reaction_frames"] == [100, 100] assert source["listen_transition_frames"] == 50 assert {step["source_id"] for step in source["steps"]} == { source["source_id"] } update_steps = [ step for step in source["steps"] if step["update_semantic_cache"] ] assert [step["background_window_index"] for step in update_steps] == list( range(100, 100 + len(update_steps)) ) transitions = [ step for step in update_steps[1:] if step["control_label"] != update_steps[step["semantic_window_index"] - 1]["control_label"] ] assert transitions assert all(step["control_transition"] for step in transitions) assert {step["turn_index"] for step in source["steps"]} == {0, 1} duration_targets = [ step for step in source["steps"] if step["motion_duration_target_mask"] ] assert len(duration_targets) == source["turn_count"] == 2 assert all(step["update_semantic_cache"] for step in duration_targets) assert all(step["control_label"] == MOTION_TOKEN_ID for step in duration_targets) assert [step["motion_duration_target_seconds"] for step in duration_targets] == [ 0.8, 0.8, ] text_targets = [ step for step in source["steps"] if step["latest_instruction_target_mask"] ] assert [step["latest_instruction_text"] for step in text_targets] == [ "instruction-3", "instruction-7", ] assert all(step["update_semantic_cache"] for step in text_targets) memory_targets = [ step for step in source["steps"] if step["instruction_memory_target_mask"] ] assert len(memory_targets) == 1 assert memory_targets[0]["turn_index"] == 1 assert memory_targets[0]["instruction_memory_slot"] == "previous" assert memory_targets[0]["instruction_memory_text"] == "instruction-3" assert memory_targets[0]["update_semantic_cache"] is True def test_multiturn_audio_uses_one_pcm_continuum(monkeypatch): dataset = FullDuplexAudioCommandDataset.__new__( FullDuplexAudioCommandDataset ) dataset.audio_sampling_rate = 16_000 dataset.audio_window_seconds = 0.6 dataset.audio_sample_step_frames = 20 dataset.motion_fps = 50 dataset.background_catalog = { "office": BackgroundWaveformSource( source_key="office", waveform_path=Path("/unused/office.wav"), window_count=20, ) } flat_background = np.linspace(-0.02, 0.02, 60_000, dtype=np.float32) monkeypatch.setattr( "psi.data.seed.full_duplex_audio_command_dataset._read_pcm16_wave_segment", lambda path, start_sample, sample_count, expected_rate: ( flat_background[:sample_count].copy() ), ) speech = { "/unused/one.wav": np.full(3_200, 0.1, dtype=np.float32), "/unused/two.wav": np.full(4_800, -0.1, dtype=np.float32), } monkeypatch.setattr( "psi.data.seed.full_duplex_audio_command_dataset._read_resampled_speech_float", lambda path, expected_rate: speech[str(path)].copy(), ) monkeypatch.setattr("random.randrange", lambda stop: 0) turn_audio = [] for source_key, path, samples, frame in ( ("one", "/unused/one.wav", 3_200, 20), ("two", "/unused/two.wav", 4_800, 100), ): turn_audio.append( { "listen_start_frame": frame, "assignment": SourceBackgroundAssignment( source_key=source_key, background_key="old-single-turn-background", listen_start_index=5, listen_window_count=2, background_window_count=20, initial_idle_max_steps=5, max_motion_steps=5, safety_steps=2, original_audio_path=Path(path), speech_samples=samples, speech_scale=0.5, ), } ) windows, key, start = dataset._build_session_audio_windows( turn_audio=turn_audio, semantic_window_count=8, ) assert key == "office" assert start == 0 assert len(windows) == 8 overlap = dataset.audio_window_samples - dataset.audio_stride_samples for left, right in zip(windows, windows[1:]): np.testing.assert_array_equal(left[-overlap:], right[:overlap]) assert not np.array_equal(windows[1], flat_background[6_400:16_000]) def test_turn_transition_is_slow_smooth_and_endpoint_exact(): start = np.zeros(36, dtype=np.float32) end = np.zeros(36, dtype=np.float32) start[31] = 0.80 end[:29] = 0.5 end[29:32] = np.asarray([1.0, -0.5, 0.86], dtype=np.float32) start[32] = 1.0 end[32] = 1.0 transition = FullDuplexAudioCommandDataset._build_pose_transition( start, end, 50, ) np.testing.assert_allclose(transition[0], start, atol=1e-7) np.testing.assert_allclose(transition[-1], end, atol=1e-7) # Quintic smootherstep approaches both endpoints much more gently than a # direct 20-frame interpolation; its first increment is below 0.01% of # the complete joint displacement. assert float(np.max(np.abs(transition[1, :29] - start[:29]))) < 5e-5 assert float(np.max(np.abs(end[:29] - transition[-2, :29]))) < 5e-5 np.testing.assert_allclose( np.linalg.norm(transition[:, 32:36], axis=-1), 1.0, atol=1e-6, ) def test_multiturn_task_entry_uses_relaxed_quintic_transition(): dataset = FullDuplexAudioCommandDataset.__new__( FullDuplexAudioCommandDataset ) dataset.multi_turn_sessions = True dataset.stationary_blend_frames = 20 anchor = np.zeros(36, dtype=np.float32) anchor[31] = 0.82 anchor[32] = 1.0 task = np.repeat(anchor[None, :], 80, axis=0) task[:, 0] = 0.6 sequence, transition_frames = dataset._build_task_sequence( task, stationary_anchor=anchor, transition_frames=50, ) assert transition_frames == 50 np.testing.assert_allclose(sequence[0], anchor, atol=1e-7) np.testing.assert_allclose(sequence[49], task[0], atol=1e-7) assert abs(float(sequence[1, 0] - anchor[0])) < 5e-5 assert abs(float(task[0, 0] - sequence[48, 0])) < 5e-5 def test_source_sequence_uses_30_20_10_rtc_and_phase_labels(monkeypatch): dataset = FullDuplexAudioCommandDataset.__new__(FullDuplexAudioCommandDataset) dataset.records = [ { "row_id": 7, "filename": "test_motion", "instruction": "test", "audio_dir": "seed_audio/000007", "motion_dir": Path("/unused/test_motion"), "declared_frames": 45, } ] slices = tuple( HiddenSlice(frame_id, Path(f"/unused/{frame_id}"), 0, 1) for frame_id in (0, 20, 40) ) dataset.audio_catalog = { "seed_audio/000007": ( AudioSource("seed_audio/000007/voice", slices), ) } dataset.white_noise_hidden = tuple( torch.full((4, 8), float(value)) for value in (9, 10, 11) ) dataset.fixed_voice_per_record = False dataset.action_chunk_size = 30 dataset.action_exec_horizon = 20 dataset.prefix_frames = 10 dataset.max_motion_steps = 0 dataset.initial_idle_max_steps = 2 dataset.stationary_blend_frames = 20 motion = np.arange(45 * 36, dtype=np.float32).reshape(45, 36) monkeypatch.setattr(dataset, "_load_motion", lambda path: motion) monkeypatch.setattr( FullDuplexAudioCommandDataset, "_load_slice", staticmethod(lambda item: torch.full((4, 8), float(item.frame_id))), ) monkeypatch.setattr( dataset, "_sample_stationary_sequence", lambda phase, frame_count, anchor_pose: ( np.repeat( np.asarray(anchor_pose, dtype=np.float32)[None, :], frame_count, axis=0, ), f"test-{phase}", ), ) monkeypatch.setattr( dataset, "_sample_stationary_anchor", lambda task_start_pose: np.asarray(task_start_pose, dtype=np.float32), ) monkeypatch.setattr( dataset, "_build_task_sequence", lambda task_motion, stationary_anchor: ( np.asarray(task_motion, dtype=np.float32), 1, ), ) monkeypatch.setattr("random.randint", lambda low, high: 2) monkeypatch.setattr("random.randrange", lambda stop: 1) source = dataset[0] assert source["white_noise_start_index"] == 1 assert source["idle_steps"] == 2 assert source["listen_steps"] == 3 assert source["motion_steps"] == 2 assert source["action_idle_steps"] == 40 assert source["action_listen_steps"] == 60 assert source["action_motion_steps"] == 34 assert [step["phase"] for step in source["steps"]].count("idle") == 40 assert [step["phase"] for step in source["steps"]].count("listen") == 60 assert [step["phase"] for step in source["steps"]].count("motion") == 34 assert [ index for index, step in enumerate(source["steps"]) if step["update_semantic_cache"] ] == [0, 20, 40, 60, 80, 100, 120] assert all( step["control_label"] == IDLE_TOKEN_ID for step in source["steps"][:40] ) assert all( step["control_label"] == LISTEN_TOKEN_ID for step in source["steps"][40:100] ) assert all( step["control_label"] == MOTION_TOKEN_ID for step in source["steps"][100:] ) # A 30-frame target crossing a state boundary now keeps one control label # per future frame. It is no longer falsely described as all-idle or # all-listen by the scalar current-window token. np.testing.assert_array_equal( source["steps"][20]["action_control_labels"], np.asarray( [IDLE_TOKEN_ID] * 19 + [LISTEN_TOKEN_ID] * 11, dtype=np.int64, ), ) np.testing.assert_array_equal( source["steps"][80]["action_control_labels"], np.asarray( [LISTEN_TOKEN_ID] * 19 + [MOTION_TOKEN_ID] * 11, dtype=np.int64, ), ) assert source["steps"][40]["control_transition"] is True assert source["steps"][100]["control_transition"] is True assert source["steps"][41]["control_transition"] is False assert [ float(source["steps"][index]["vlm_hidden_states"][0, 0]) for index in (0, 20, 40, 60, 80, 100, 120) ] == [10.0, 11.0, 0.0, 20.0, 40.0, 9.0, 10.0] assert "vlm_hidden_states" not in source["steps"][1] first_motion = source["steps"][100] np.testing.assert_array_equal(first_motion["actions"], motion[1:31]) np.testing.assert_array_equal(first_motion["states"][0], motion[0]) second_motion = source["steps"][101] np.testing.assert_array_equal(second_motion["actions"], motion[2:32]) np.testing.assert_array_equal(second_motion["states"][0], motion[1]) np.testing.assert_array_equal( second_motion["action_reference_states"][0], motion[1], ) final_motion = source["steps"][-1] np.testing.assert_array_equal(final_motion["actions"][:11], motion[34:45]) np.testing.assert_array_equal( final_motion["actions"][11:], np.repeat(motion[44:45], 19, axis=0), ) assert final_motion["action_mask"].sum() == 11 def test_textop_rtc_first_delta_uses_current_state_before_future_horizon(): actions = np.zeros((30, 36), dtype=np.float32) actions[:, 29] = np.arange(10, 40, dtype=np.float32) actions[:, 32] = 1.0 observation_state = np.zeros((1, 36), dtype=np.float32) observation_state[0, 29] = 29.0 observation_state[0, 32] = 1.0 action_reference_state = np.zeros((1, 36), dtype=np.float32) action_reference_state[0, 29] = 9.0 action_reference_state[0, 32] = 1.0 transformed = SeedPosttrainRepackTransform( action_chunk_size=30, rtc_prefix_frames=10, textop_actions=True, )( { "actions": actions, "action_mask": np.ones(30, dtype=np.float32), "states": observation_state, "action_reference_states": action_reference_state, } ) assert transformed["actions"].shape == (30, 39) assert transformed["actions"][0, 29] == pytest.approx(1.0) def test_dense_action_only_steps_skip_qwen_audio_transform(): class Stage: def __init__(self, name, calls): self.name = name self.calls = calls def __call__(self, data, **kwargs): del kwargs self.calls.append(self.name) output = dict(data) if self.name == "field": output["actions_mask"] = output.pop("action_mask") if self.name == "model": assert "audio_waveform" in output output["stream_input_ids"] = torch.ones(3, dtype=torch.long) return output class Transform: def __init__(self): self.calls = [] self.repack = Stage("repack", self.calls) self.field = Stage("field", self.calls) self.model = Stage("model", self.calls) def __call__(self, data, **kwargs): data = self.repack(data, **kwargs) data = self.field(data, **kwargs) return self.model(data, **kwargs) base = { "source_id": "source-a", "source_audio_key": "audio-a", "phase": "listen", "control_label": LISTEN_TOKEN_ID, "control_token": "", "frame_id": 0, "semantic_frame_id": 0, "semantic_window_index": 0, "actions": np.zeros((30, 39), dtype=np.float32), "action_mask": np.ones((30, 39), dtype=np.float32), "states": np.zeros((1, 39), dtype=np.float32), "progress": np.zeros(5, dtype=np.float32), "future_progress": np.zeros((30, 5), dtype=np.float32), "motion_duration_target_seconds": 4.25, "motion_duration_target_mask": False, } update = { **base, "step_index": 0, "action_substep": 0, "update_semantic_cache": True, "audio_waveform": np.zeros(9_600, dtype=np.float32), "motion_duration_target_mask": True, } reuse = { **base, "step_index": 1, "action_substep": 1, "update_semantic_cache": False, } class RawDataset: def __len__(self): return 1 def __getitem__(self, index): assert index == 0 return {"source_id": "source-a", "steps": [update, reuse]} def estimated_stream_steps(self, index): assert index == 0 return 2 transform = Transform() dataset = FullDuplexTransformedDataset(RawDataset(), transform) source = dataset[0] assert transform.calls == ["repack", "field", "model", "repack", "field"] assert "stream_input_ids" in source["steps"][0] assert "stream_input_ids" not in source["steps"][1] assert source["steps"][0]["motion_duration_target_seconds"] == 4.25 assert source["steps"][0]["motion_duration_target_mask"] is True assert source["steps"][1]["motion_duration_target_mask"] is False def test_real_seed_repack_preserves_motion_duration_supervision(): class IdentityStage: def __call__(self, data, **kwargs): del kwargs return data class Transform: def __init__(self): self.repack = SeedPosttrainRepackTransform( action_chunk_size=30, rtc_prefix_frames=10, textop_actions=False, pad_action_dim=39, pad_state_dim=39, ) self.field = IdentityStage() self.model = IdentityStage() def __call__(self, data, **kwargs): data = self.repack(data, **kwargs) data = self.field(data, **kwargs) return self.model(data, **kwargs) raw_step = { "source_id": "source-duration", "source_audio_key": "audio-duration", "phase": "motion", "control_label": MOTION_TOKEN_ID, "control_token": "", "control_transition": True, "action_control_labels": np.full(30, MOTION_TOKEN_ID), "update_semantic_cache": True, "step_index": 0, "frame_id": 0, "semantic_frame_id": 0, "semantic_window_index": 0, "action_substep": 0, "turn_index": 0, "actions": np.zeros((30, 39), dtype=np.float32), "action_mask": np.ones((30, 39), dtype=np.float32), "states": np.zeros((1, 39), dtype=np.float32), "progress": np.zeros(5, dtype=np.float32), "future_progress": np.zeros((30, 5), dtype=np.float32), "instruction": "walk", "motion_duration_target_seconds": 6.5, "motion_duration_target_mask": True, "latest_instruction_text": "walk", "latest_instruction_target_mask": True, } class RawDataset: def __len__(self): return 1 def __getitem__(self, index): assert index == 0 return {"source_id": "source-duration", "steps": [raw_step]} def estimated_stream_steps(self, index): assert index == 0 return 1 source = FullDuplexTransformedDataset(RawDataset(), Transform())[0] assert source["steps"][0]["motion_duration_target_seconds"] == 6.5 assert source["steps"][0]["motion_duration_target_mask"] is True assert source["steps"][0]["latest_instruction_text"] == "walk" assert source["steps"][0]["latest_instruction_target_mask"] is True assert source["steps"][0]["instruction_memory_text"] == "" assert source["steps"][0]["instruction_memory_slot"] == "" assert source["steps"][0]["instruction_memory_target_mask"] is False def test_continuous_background_indices_are_adjacent_without_wrap(monkeypatch): dataset = FullDuplexAudioCommandDataset.__new__( FullDuplexAudioCommandDataset ) dataset.records = [ { "row_id": 7, "filename": "test_motion", "instruction": "test", "audio_dir": "seed_audio/000007", "motion_dir": Path("/unused/test_motion"), "declared_frames": 45, } ] slices = tuple( HiddenSlice(frame_id, Path(f"/unused/{frame_id}"), 0, 1) for frame_id in (0, 20, 40) ) voice = AudioSource("seed_audio/000007/voice", slices) dataset.audio_catalog = {"seed_audio/000007": (voice,)} dataset.continuous_background = True dataset.background_assignments = { voice.source_key: SourceBackgroundAssignment( source_key=voice.source_key, background_key="sessions/office", listen_start_index=100, listen_window_count=3, background_window_count=300, initial_idle_max_steps=2, max_motion_steps=3, safety_steps=10, ) } dataset.white_noise_hidden = () dataset.fixed_voice_per_record = True dataset.action_chunk_size = 30 dataset.action_exec_horizon = 20 dataset.prefix_frames = 10 dataset.max_motion_steps = 0 dataset.initial_idle_max_steps = 2 dataset.stationary_blend_frames = 20 motion = np.arange(45 * 36, dtype=np.float32).reshape(45, 36) monkeypatch.setattr(dataset, "_load_motion", lambda path: motion) monkeypatch.setattr( FullDuplexAudioCommandDataset, "_load_slice", staticmethod(lambda item: torch.full((4, 8), float(item.frame_id))), ) loaded_background_blocks = [] def load_background_block(source_key, start, count): assert source_key == "sessions/office" loaded_background_blocks.append((start, count)) return tuple( torch.full((4, 8), float(index)) for index in range(start, start + count) ) monkeypatch.setattr( dataset, "_load_background_window_block", load_background_block, ) monkeypatch.setattr( dataset, "_sample_stationary_sequence", lambda phase, frame_count, anchor_pose: ( np.repeat( np.asarray(anchor_pose, dtype=np.float32)[None, :], frame_count, axis=0, ), f"test-{phase}", ), ) monkeypatch.setattr( dataset, "_sample_stationary_anchor", lambda task_start_pose: np.asarray(task_start_pose, dtype=np.float32), ) monkeypatch.setattr( dataset, "_build_task_sequence", lambda task_motion, stationary_anchor: ( np.asarray(task_motion, dtype=np.float32), 1, ), ) monkeypatch.setattr("random.randint", lambda low, high: 2) source = dataset[0] assert loaded_background_blocks == [(98, 2), (103, 2)] assert source["background_key"] == "sessions/office" assert source["background_listen_start_index"] == 100 assert source["listen_steps"] == 3 assert source["motion_steps"] == 2 def test_overfit_voice_selection_is_stable_per_record(monkeypatch): voices = tuple( AudioSource(f"seed_audio/voice_{index}", ()) for index in range(4) ) dataset = FullDuplexAudioCommandDataset.__new__(FullDuplexAudioCommandDataset) dataset.fixed_voice_per_record = True selected = [dataset._select_voice(voices, row_id=7) for _ in range(5)] assert selected == [voices[3]] * 5 dataset.fixed_voice_per_record = False monkeypatch.setattr("random.choice", lambda values: values[1]) assert dataset._select_voice(voices, row_id=7) is voices[1] def test_broken_tts_duration_outliers_are_filtered_before_sampling(): def voice(name: str, windows: int) -> AudioSource: return AudioSource( name, (), waveform_path=Path(f"/unused/{name}.wav"), waveform_count=windows, ) dataset = FullDuplexAudioCommandDataset.__new__( FullDuplexAudioCommandDataset ) dataset.max_voice_windows = 30 dataset.max_voice_median_ratio = 3.0 catalog = { "normal-with-corruption": ( voice("short-a", 7), voice("short-b", 8), voice("broken", 196), ), # Defensive fallback keeps a custom long-command directory usable. "all-long": (voice("long-a", 35), voice("long-b", 40)), } filtered, removed = dataset._filter_voice_duration_outliers(catalog) assert [item.source_key for item in filtered["normal-with-corruption"]] == [ "short-a", "short-b", ] assert [item.source_key for item in filtered["all-long"]] == ["long-a"] assert removed == 2 def test_length_estimate_uses_fixed_voice_and_converts_metadata_fps(): dataset = FullDuplexAudioCommandDataset.__new__( FullDuplexAudioCommandDataset ) dataset.records = [ { "row_id": 1, "audio_dir": "seed_audio/one", "declared_frames": 580, } ] dataset.audio_catalog = { "seed_audio/one": ( AudioSource("voice-a", (), packed_path=Path("a"), packed_count=7), AudioSource("voice-b", (), packed_path=Path("b"), packed_count=10), ) } dataset.fixed_voice_per_record = True dataset.motion_fps = 50 dataset.motion_metadata_fps = 120 dataset.stationary_blend_frames = 20 dataset.prefix_frames = 10 dataset.action_exec_horizon = 20 dataset.initial_idle_max_steps = 5 dataset.max_motion_steps = 0 dataset.max_stream_steps = 1923 # 580 metadata frames -> 242 exported 50 Hz frames. With the 19-frame # transition there are 250 valid dense motion starts. assert dataset._estimated_runtime_motion_frames(dataset.records[0]) == 242 assert dataset.estimated_stream_steps(0) == (2 + 10) * 20 + 250 assert dataset._record_max_stream_steps(dataset.records[0]) == 5 + 10 + 13 def test_validation_source_augmentation_is_repeatable_and_restores_rng(): dataset = FullDuplexAudioCommandDataset.__new__( FullDuplexAudioCommandDataset ) dataset.records = [{"row_id": 17}] dataset.split = "val" dataset.split_seed = 42 dataset.deterministic_eval = True dataset._build_source = lambda index: { "index": index, "voice": random.randrange(8), "idle": random.randint(0, 5), "anchor": random.random(), } random.seed(1234) state_before = random.getstate() first = dataset[0] assert random.getstate() == state_before second = dataset[0] assert first == second assert random.getstate() == state_before def test_dataset_prefers_compact_source_indexes_and_reads_one_wav( tmp_path, ): source_dir = ( tmp_path / "audio" / "qwen3_tts_seed_audio_command" / "audio_command_text" / "sources" ) source_path = ( source_dir / "seed_audio" / "000007" / "voice.safetensors" ) source_path.parent.mkdir(parents=True) hidden = torch.arange(32, dtype=torch.bfloat16).reshape(8, 4) save_file( { "hidden_states": hidden, "offsets": torch.tensor([0, 3, 8], dtype=torch.int64), "frame_ids": torch.tensor([0, 20], dtype=torch.int64), }, source_path, metadata={ "format": HFD_AUDIO_SOURCE_FORMAT, "encoder_contract": HFD_ONLINE_ENCODER_CONTRACT, "encoder_batch_size": str(HFD_ONLINE_ENCODER_BATCH_SIZE), "encoder_dtype": HFD_ONLINE_ENCODER_DTYPE, "attention_implementation": HFD_ONLINE_ENCODER_ATTENTION, "instruction_sha1": hashlib.sha1( AUDIO_COMMAND_STREAMING_PROMPT.encode("utf-8") ).hexdigest(), "count": "2", "window_seconds": "0.600000000", "step_frames": "20", "motion_fps": "50", "sampling_rate": "16000", }, ) index_path = source_dir / "index.ws1.rank00.sqlite" with sqlite3.connect(index_path) as conn: conn.execute( "CREATE TABLE sources (" "key TEXT PRIMARY KEY, path TEXT NOT NULL, count INTEGER NOT NULL, " "window_seconds REAL NOT NULL, step_frames INTEGER NOT NULL, " "motion_fps INTEGER NOT NULL, sampling_rate INTEGER NOT NULL)" ) conn.execute( "INSERT INTO sources VALUES (?, ?, ?, ?, ?, ?, ?)", ( "seed_audio/000007/voice", "seed_audio/000007/voice.safetensors", 2, 0.6, 20, 50, 16000, ), ) dataset = FullDuplexAudioCommandDataset.__new__( FullDuplexAudioCommandDataset ) dataset.hidden_root = tmp_path dataset.source_name = "qwen3_tts_seed_audio_command" dataset.task_name = "audio_command_text" dataset.audio_window_seconds = 0.6 dataset.audio_sample_step_frames = 20 dataset.motion_fps = 50 dataset.audio_sampling_rate = 16000 catalog = dataset._load_audio_catalog() voice = catalog["seed_audio/000007"][0] windows = dataset._load_voice_hidden(voice) assert dataset.hidden_layout == "source" assert voice.packed_path == source_path.resolve() assert voice.clips == () assert voice.window_count == 2 assert torch.equal(windows[0], hidden[:3]) assert torch.equal(windows[1], hidden[3:]) assert windows[0].untyped_storage().data_ptr() == ( windows[1].untyped_storage().data_ptr() ) def test_stationary_motion_pool_is_kinematically_gated_and_blended(monkeypatch): anchor = np.zeros(36, dtype=np.float32) anchor[29:32] = [1.0, 2.0, 0.8] anchor[32] = 1.0 natural = np.repeat(anchor[None, :], 80, axis=0) natural[:, 0] = np.sin(np.linspace(0.0, 2.0 * np.pi, 80)) * 0.2 natural[:, 29] = 4.0 + np.sin(np.linspace(0.0, 2.0 * np.pi, 80)) * 0.02 natural[:, 30] = -3.0 natural[:, 31] = 0.92 + np.sin(np.linspace(0.0, np.pi, 80)) * 0.01 dataset = FullDuplexAudioCommandDataset.__new__( FullDuplexAudioCommandDataset ) candidate = StationaryMotionSource( "safe-neutral", Path("/unused/safe-neutral"), len(natural), "neutral_idle", ) dataset.stationary_motion_pools = { "idle": (candidate,), "listen": (candidate,), } dataset.stationary_blend_frames = 10 monkeypatch.setattr(dataset, "_load_motion", lambda path: natural) monkeypatch.setattr("random.randint", lambda low, high: 0) sequence, name = dataset._sample_stationary_sequence( "idle", frame_count=40, anchor_pose=anchor, ) assert name == "safe-neutral" np.testing.assert_allclose(sequence[0], anchor, atol=1e-6) np.testing.assert_allclose(sequence[-1], anchor, atol=1e-6) assert np.max(np.abs(sequence[10:-10, 0] - anchor[0])) > 0.05 np.testing.assert_allclose( sequence[10, 29:31], anchor[29:31] + natural[10, 29:31] - natural[0, 29:31], atol=1e-6, ) np.testing.assert_allclose(sequence[10, 31], natural[10, 31], atol=1e-6) np.testing.assert_allclose( np.linalg.norm(sequence[:, 32:36], axis=-1), np.ones(40), atol=1e-6, ) task_start = anchor.copy() task_start[29:32] = [7.0, 8.0, 0.35] sampled_anchor = dataset._sample_stationary_anchor(task_start) np.testing.assert_allclose(sampled_anchor[29:31], task_start[29:31], atol=1e-6) np.testing.assert_allclose(sampled_anchor[31], natural[0, 31], atol=1e-6) np.testing.assert_allclose(sampled_anchor[:29], natural[0, :29], atol=1e-6) locomoting = natural.copy() locomoting[:, 29] += np.linspace(0.0, 1.0, len(locomoting)) assert dataset._stationary_motion_is_safe(natural) assert not dataset._stationary_motion_is_safe(locomoting) def test_v24_idle_uses_motion_residual_instead_of_unrelated_absolute_pose( monkeypatch, ): anchor = np.zeros(36, dtype=np.float32) anchor[0] = 0.20 anchor[31] = 0.82 anchor[32] = 1.0 recorded = np.repeat(anchor[None, :], 60, axis=0) # The recorded clip is quiet, but its absolute arm pose is unrelated to # this session's current pose. Only its subtle joint-1 residual is useful. recorded[:, 0] = 1.50 recorded[:, 1] = 0.05 * np.sin( np.linspace(0.0, 2.0 * np.pi, len(recorded), dtype=np.float32) ) source = StationaryMotionSource( "quiet-different-arm-pose", Path("/unused/quiet-different-arm-pose"), len(recorded), "neutral_idle", ) dataset = FullDuplexAudioCommandDataset.__new__( FullDuplexAudioCommandDataset ) dataset.stationary_motion_pools = { "idle": (source,), "listen": (source,), } dataset.stationary_blend_frames = 5 dataset.stationary_heading_alignment = False dataset.safe_stationary_anchors = False monkeypatch.setattr(dataset, "_load_motion", lambda path: recorded) monkeypatch.setattr("random.randint", lambda low, high: 0) dataset.stationary_idle_residual_retargeting = False legacy, _ = dataset._sample_stationary_sequence( "idle", frame_count=60, anchor_pose=anchor, ) assert float(np.max(np.abs(legacy[:, 0] - anchor[0]))) > 1.0 dataset.stationary_idle_residual_retargeting = True retargeted, _ = dataset._sample_stationary_sequence( "idle", frame_count=60, anchor_pose=anchor, ) np.testing.assert_allclose(retargeted[:, 0], anchor[0], atol=1e-6) assert float(np.max(np.abs(retargeted[:, 1] - anchor[1]))) > 0.04 np.testing.assert_allclose(retargeted[0], anchor, atol=1e-6) np.testing.assert_allclose(retargeted[-1], anchor, atol=1e-6) def test_v24_idle_gate_rejects_jump_cuts_and_large_gestures(): clean = np.zeros((100, 36), dtype=np.float32) clean[:, 31] = 0.82 clean[:, 32] = 1.0 clean[:, 0] = 0.08 * np.sin( np.linspace(0.0, 2.0 * np.pi, len(clean), dtype=np.float32) ) jump_cut = clean.copy() jump_cut[50:, 3] += 0.20 gesture = clean.copy() gesture[:, 4] = np.linspace(0.0, 0.60, len(gesture), dtype=np.float32) assert FullDuplexAudioCommandDataset._idle_motion_is_quiet_and_continuous( clean ) assert not FullDuplexAudioCommandDataset._idle_motion_is_quiet_and_continuous( jump_cut ) assert not FullDuplexAudioCommandDataset._idle_motion_is_quiet_and_continuous( gesture ) def test_safe_stationary_audit_rejects_static_whole_body_outlier(monkeypatch): dataset = FullDuplexAudioCommandDataset.__new__( FullDuplexAudioCommandDataset ) dataset.motion_fps = 50 dataset.stationary_motion_pools = {"idle": (), "listen": ()} sources = tuple( StationaryMotionSource( name, Path(f"/unused/{name}"), 100, "neutral_idle", ) for name in ("neutral-a", "neutral-b", "corrupt-static") ) dataset.stationary_motion_pools["idle"] = sources motions = {} for source in sources: motion = np.zeros((100, 36), dtype=np.float32) motion[:, 31] = 0.82 motion[:, 32] = 1.0 motions[str(source.motion_dir)] = motion motions["/unused/neutral-b"][:, 0] = 0.03 # Static alone is not sufficient: this mimics the real malformed idle clip # whose arm joints are many robust deviations away from the neutral pool. motions["/unused/corrupt-static"][:, 22:29] = 2.0 monkeypatch.setattr( dataset, "_load_motion", lambda path: motions[str(path)], ) dataset._initialise_safe_stationary_anchors() assert "/unused/corrupt-static" not in dataset._safe_idle_motion_paths assert dataset.stationary_audit_summary["accepted_clips"] == 2 assert dataset.stationary_audit_summary["rejected_clips"] == 1 assert dataset.stationary_audit_summary["anchor_frames"] > 0 query = motions["/unused/neutral-b"][50].copy() query[29:31] = [4.0, -2.0] anchor, _ = dataset._nearest_safe_stationary_anchor(query) np.testing.assert_allclose(anchor[:29], query[:29], atol=1e-6) np.testing.assert_allclose(anchor[29:31], query[29:31], atol=1e-6) def test_stationary_entry_blends_from_executed_pose_to_safe_anchor(monkeypatch): anchor = np.zeros(36, dtype=np.float32) anchor[31] = 0.82 anchor[32] = 1.0 entry = anchor.copy() entry[0] = 0.8 natural = np.repeat(anchor[None, :], 80, axis=0) natural[:, 1] = 0.2 source = StationaryMotionSource( "listen", Path("/unused/listen"), len(natural), "listening", ) dataset = FullDuplexAudioCommandDataset.__new__( FullDuplexAudioCommandDataset ) dataset.stationary_motion_pools = {"idle": (source,), "listen": (source,)} dataset.stationary_blend_frames = 10 dataset.turn_transition_frames = 50 dataset.stationary_heading_alignment = True dataset.safe_stationary_anchors = False monkeypatch.setattr(dataset, "_load_motion", lambda path: natural) monkeypatch.setattr("random.randint", lambda low, high: 0) sequence, _ = dataset._sample_stationary_sequence( "listen", frame_count=80, anchor_pose=anchor, blend_frames=50, entry_pose=entry, ) np.testing.assert_allclose(sequence[0], entry, atol=1e-6) np.testing.assert_allclose(sequence[-1], anchor, atol=1e-6) assert float(sequence[25, 0]) < float(entry[0]) def test_stationary_motion_heading_is_reexpressed_in_current_anchor(monkeypatch): anchor = np.zeros(36, dtype=np.float32) anchor[29:32] = [2.0, -1.0, 0.8] anchor_yaw = 0.5 * np.pi anchor[32:36] = rpy_to_quat_wxyz(0.0, 0.0, anchor_yaw) candidate_motion = np.repeat(anchor[None, :], 40, axis=0) candidate_motion[:, 29:31] = [8.0, 4.0] recorded_yaw = -0.5 * np.pi relative_yaw = np.linspace(0.0, 0.1, 40, dtype=np.float32) candidate_motion[:, 32:36] = np.stack( [ rpy_to_quat_wxyz(0.0, 0.0, recorded_yaw + float(offset)) for offset in relative_yaw ] ) dataset = FullDuplexAudioCommandDataset.__new__( FullDuplexAudioCommandDataset ) source = StationaryMotionSource( "listen-facing-another-way", Path("/unused/listen-facing-another-way"), len(candidate_motion), "listening", ) dataset.stationary_motion_pools = { "idle": (source,), "listen": (source,), } dataset.stationary_blend_frames = 5 dataset.stationary_heading_alignment = True monkeypatch.setattr(dataset, "_load_motion", lambda path: candidate_motion) monkeypatch.setattr("random.randint", lambda low, high: 0) sequence, _ = dataset._sample_stationary_sequence( "listen", frame_count=40, anchor_pose=anchor, ) output_yaw = np.unwrap( np.asarray( [quat_to_yaw_wxyz(frame[32:36]) for frame in sequence] ) ) assert abs(float(wrap_angle(output_yaw[0] - anchor_yaw))) < 1e-6 assert abs(float(wrap_angle(output_yaw[-1] - anchor_yaw))) < 1e-6 # The candidate's small relative turn is preserved, while its unrelated # 180-degree recording orientation is removed. assert np.max(np.abs(output_yaw - anchor_yaw)) < 0.11 def test_dense_stream_rows_carry_two_source_local_context_tokens(): dataset = FullDuplexAudioCommandDataset.__new__( FullDuplexAudioCommandDataset ) dataset.action_chunk_size = 3 dataset.prefix_frames = 1 sequence = np.zeros((8, 36), dtype=np.float32) sequence[:, 0] = np.arange(8, dtype=np.float32) sequence[:, 29] = np.arange(8, dtype=np.float32) * 0.1 sequence[:, 32] = 1.0 rows = dataset._stream_action_rows(sequence, start=4) actions, mask, state, reference, context, context_reference = rows np.testing.assert_allclose(actions, sequence[5:8]) np.testing.assert_array_equal(mask, np.ones(3, dtype=np.float32)) np.testing.assert_allclose(state, sequence[4]) np.testing.assert_allclose(reference, sequence[4]) np.testing.assert_allclose(context, sequence[2:4 + 1][1:]) np.testing.assert_allclose(context_reference, sequence[2]) start_rows = dataset._stream_action_rows(sequence, start=0) np.testing.assert_allclose(start_rows[4], np.repeat(sequence[0:1], 2, axis=0)) np.testing.assert_allclose(start_rows[5], sequence[0]) def test_task_motion_transition_starts_from_upright_anchor_without_jump(): dataset = FullDuplexAudioCommandDataset.__new__( FullDuplexAudioCommandDataset ) dataset.stationary_blend_frames = 20 anchor = np.zeros(36, dtype=np.float32) anchor[0] = -0.5 anchor[29:32] = [1.0, 2.0, 0.8] anchor[32] = 1.0 task = np.repeat(anchor[None, :], 60, axis=0) task[:, 0] = 0.75 task[:, 31] = 0.5 sequence, transition_frames = dataset._build_task_sequence( task, stationary_anchor=anchor, ) assert transition_frames == 20 assert len(sequence) == len(task) + transition_frames - 1 np.testing.assert_allclose(sequence[0], anchor, atol=1e-6) np.testing.assert_allclose(sequence[transition_frames - 1], task[0], atol=1e-6) np.testing.assert_allclose(sequence[transition_frames], task[1], atol=1e-6) assert np.max(np.linalg.norm(np.diff(sequence[:, :29], axis=0), axis=1)) < 0.11 def test_dynamic_task_entry_blends_into_moving_prefix_without_phase_restart(): dataset = FullDuplexAudioCommandDataset.__new__( FullDuplexAudioCommandDataset ) dataset.stationary_blend_frames = 20 dataset.dynamic_task_entry_blend = True anchor = np.zeros(36, dtype=np.float32) anchor[31] = 0.82 anchor[32] = 1.0 task = np.repeat(anchor[None, :], 80, axis=0) task[:, 0] = np.linspace(0.6, 1.4, len(task), dtype=np.float32) sequence, transition_frames = dataset._build_task_sequence( task, stationary_anchor=anchor, ) assert transition_frames == 20 assert len(sequence) == len(task) np.testing.assert_allclose(sequence[0], anchor, atol=1e-6) np.testing.assert_allclose(sequence[19], task[19], atol=1e-6) # At the crossfade boundary the real trajectory simply continues; it never # jumps back to primitive frame zero or restarts its phase. np.testing.assert_allclose(sequence[20], task[20], atol=1e-6) np.testing.assert_allclose( sequence[20, :29] - sequence[19, :29], task[20, :29] - task[19, :29], atol=1e-6, ) np.testing.assert_array_equal( dataset._task_original_frame_indices( sequence_frames=len(sequence), motion_frames=len(task), transition_frames=transition_frames, ), np.arange(len(task)), ) def test_task_motion_is_rigidly_aligned_to_stationary_anchor_heading(): dataset = FullDuplexAudioCommandDataset.__new__( FullDuplexAudioCommandDataset ) dataset.stationary_blend_frames = 5 anchor = np.zeros(36, dtype=np.float32) anchor[29:32] = [10.0, -3.0, 0.8] anchor_yaw = -0.5 * np.pi anchor[32:36] = rpy_to_quat_wxyz(0.0, 0.0, anchor_yaw) task = np.zeros((4, 36), dtype=np.float32) task[:, :29] = np.arange(4, dtype=np.float32)[:, None] task[:, 29] = 1.0 task[:, 30] = 2.0 + np.arange(4, dtype=np.float32) task[:, 31] = 0.8 + 0.05 * np.arange(4, dtype=np.float32) task_yaw = 0.5 * np.pi yaw_progress = 0.1 * np.arange(4, dtype=np.float32) task[:, 32:36] = np.stack( [ rpy_to_quat_wxyz(0.0, 0.0, float(task_yaw + offset)) for offset in yaw_progress ] ) sequence, transition_frames = dataset._build_task_sequence( task, stationary_anchor=anchor, ) aligned = sequence[transition_frames - 1 :] np.testing.assert_allclose(sequence[0], anchor, atol=1e-6) np.testing.assert_allclose(aligned[0, 29:32], anchor[29:32], atol=1e-6) np.testing.assert_allclose(aligned[:, :29], task[:, :29], atol=1e-6) np.testing.assert_allclose( aligned[:, 31] - aligned[0, 31], task[:, 31] - task[0, 31], atol=1e-6, ) aligned_yaw = quat_to_yaw_wxyz(aligned[:, 32:36]) np.testing.assert_allclose( wrap_angle(aligned_yaw - anchor_yaw), yaw_progress, atol=1e-6, ) original_local_delta = local_delta_from_world( task[:-1, 29:32], task[1:, 29:32], task[:-1, 32:36], ) aligned_local_delta = local_delta_from_world( aligned[:-1, 29:32], aligned[1:, 29:32], aligned[:-1, 32:36], ) np.testing.assert_allclose( aligned_local_delta, original_local_delta, atol=1e-6, ) np.testing.assert_allclose( np.linalg.norm(aligned[:, 32:36], axis=-1), np.ones(len(aligned)), atol=1e-6, ) def test_kv_cache_is_shared_only_inside_one_source(): memory = StreamingCausalMemory( input_dim=8, hidden_dim=16, num_layers=2, num_heads=4, dropout=0.0, max_cache_tokens=32, ) cache_a = memory.begin_source(["source-a"]) pooled = torch.randn(1, 8) _, cache_a, _ = memory( pooled, cache_a, source_ids=["source-a"], update_cache=True ) _, cache_a, _ = memory( pooled, cache_a, source_ids=["source-a"], update_cache=True ) assert cache_a.sequence_length == 2 # False is reserved for a trainer-inserted padding row after the source has # ended; it must not insert the placeholder token into semantic history. _, padded_cache, _ = memory( torch.randn(1, 8), cache_a, source_ids=["source-a"], update_cache=False ) assert padded_cache.sequence_length == 2 for before, after in zip(cache_a.keys, padded_cache.keys): assert before is after with pytest.raises(RuntimeError, match="across sources"): memory(pooled, cache_a, source_ids=["source-b"], update_cache=True) cache_b = memory.begin_source(["source-b"]) assert cache_b.sequence_length == 0 def test_batched_kv_cache_tracks_mixed_active_and_padded_sources(): memory = StreamingCausalMemory( input_dim=8, hidden_dim=16, num_layers=2, num_heads=4, dropout=0.0, max_cache_tokens=32, ).eval() cache = memory.begin_source(["source-a", "source-b"]) _, cache, _ = memory( torch.randn(2, 8), cache, source_ids=["source-a", "source-b"], update_cache=[True, True], ) assert cache.sequence_lengths == (1, 1) # A has ended while B still has an active audio window. The shared physical # cache grows once, but the new position is invalid for A and valid for B. _, cache, _ = memory( torch.randn(2, 8), cache, source_ids=["source-a", "source-b"], update_cache=[False, True], ) assert cache.sequence_lengths == (1, 2) assert cache.valid_mask is not None assert cache.valid_mask.tolist() == [[True, False], [True, True]] with pytest.raises(RuntimeError, match="different source lengths"): _ = cache.sequence_length before_keys = cache.keys _, padded_cache, _ = memory( torch.randn(2, 8), cache, source_ids=["source-a", "source-b"], update_cache=[False, False], ) assert padded_cache.sequence_lengths == (1, 2) assert all(before is after for before, after in zip(before_keys, padded_cache.keys)) def test_trainer_stream_forward_is_a_real_two_source_batch(): config = FullDuplexPsi0ModelConfig( streaming_qwen_enabled=False, action_dim=3, action_chunk_size=30, action_exec_horizon=20, observation_horizon=1, odim=3, n_conditions=0, hidden_dim=48, num_blocks=1, dim_feedforward=64, nhead=4, view_feature_dim=8, use_future_progress_embedding=False, rtc=True, max_delay=11, streaming_memory_dim=16, streaming_memory_layers=1, streaming_memory_heads=4, streaming_max_cache_tokens=8, ) model = FullDuplexPsi0Model(config, vlm_model=None).eval() captured_timesteps = [] original_forward = model.forward def capture_forward(*args, **kwargs): captured_timesteps.append(kwargs["timestep"].detach().clone()) return original_forward(*args, **kwargs) model.forward = capture_forward trainer = PosttrainFullDuplexTrainer.__new__(PosttrainFullDuplexTrainer) trainer.device = torch.device("cpu") trainer.rtc_train_overlap = 10 trainer.train_diffusion_steps = 1_000 trainer.loss_w = torch.ones(3) trainer.loss_group_w = torch.ones(3) class FixedSampler: def sample_like(self, actions): return SimpleNamespace( sigmas=torch.tensor([0.25, 0.75], dtype=torch.float32), noise=torch.zeros_like(actions), ) def make_step( source_id: str, *, listen: bool, value: float, step_index: int, ): return { "source_id": source_id, "step_index": step_index, "vlm_hidden_states": torch.full((5, 8), value), "attention_mask": torch.ones(5, dtype=torch.long), "actions": torch.full((30, 3), value), "actions_mask": torch.ones(30, 3), "states": torch.zeros(1, 3), "progress": torch.zeros(5), "future_progress": torch.zeros(30, 5), "update_semantic_cache": listen, "control_label": LISTEN_TOKEN_ID if listen else MOTION_TOKEN_ID, } source_ids = ["source-a", "source-b"] cache = model.begin_source(source_ids) losses, cache = trainer._forward_stream_batch( model, source_ids, [ make_step("source-a", listen=True, value=1.0, step_index=0), make_step("source-b", listen=True, value=2.0, step_index=0), ], cache, FixedSampler(), ) assert losses["action_loss"].shape == (2,) assert losses["control_loss"].shape == (2,) assert torch.isfinite(losses["action_loss"]).all() assert cache.sequence_lengths == (1, 1) # The first source window has no previous prediction, hence no clean RTC # prefix is leaked even though the sampler fallback requests delay 10. assert torch.all(captured_timesteps[0][0] == 250.0) assert torch.all(captured_timesteps[0][1] == 750.0) losses, cache = trainer._forward_stream_batch( model, source_ids, [ make_step("source-a", listen=False, value=3.0, step_index=1), make_step("source-b", listen=True, value=4.0, step_index=1), ], cache, FixedSampler(), ) assert losses["action_loss"].shape == (2,) assert cache.sequence_lengths == (1, 2) assert torch.all(captured_timesteps[1][:, :10] == 0.0) @pytest.mark.parametrize("semantics_version", [12, 15, 19, 20]) def test_framewise_control_uses_versioned_future_mask_and_rtc_prefix( semantics_version, ): trainer = PosttrainFullDuplexTrainer.__new__(PosttrainFullDuplexTrainer) trainer.device = torch.device("cpu") trainer.rtc_train_overlap = 10 trainer.train_diffusion_steps = 1_000 trainer.loss_w = torch.ones(39) trainer.loss_group_w = torch.ones(39) trainer.cfg = SimpleNamespace( train=SimpleNamespace( full_duplex_semantics_version=semantics_version, joint_velocity_loss_weight=0.10, joint_acceleration_loss_weight=0.05, root_continuity_loss_weight=0.10, stationary_root_height_loss_weight=1.0, ), model=SimpleNamespace(max_delay=11), ) class FixedSampler: def sample_like(self, actions): return SimpleNamespace( sigmas=torch.tensor([0.5], dtype=torch.float32), noise=torch.zeros_like(actions), delay=torch.tensor([10], dtype=torch.long), ) class FakeModel: def __init__(self): self.captured_timestep = None self._latest_reusable_condition = None def __call__(self, **kwargs): self.captured_timestep = kwargs["timestep"].detach().clone() batch = kwargs["action_samples"].shape[0] logits = torch.zeros((batch, 3), dtype=torch.float32) probabilities = torch.softmax(logits, dim=-1) condition = torch.zeros((batch, 1, 8), dtype=torch.float32) mask = torch.ones((batch, 1), dtype=torch.long) self._latest_reusable_condition = { "condition_hidden_states": condition, "condition_attention_mask": mask, "control_logits": logits, "control_probabilities": probabilities, "control_condition_weights": probabilities, "control_entropy": torch.full((batch,), float(np.log(3.0))), "memory_token": condition, "motion_duration_log1p_seconds": torch.zeros( batch, dtype=torch.float32 ), } return SimpleNamespace( action=torch.zeros_like(kwargs["action_samples"]), control_logits=logits, control_probabilities=probabilities, control_entropy=torch.full((batch,), float(np.log(3.0))), motion_duration_log1p_seconds=torch.zeros( batch, dtype=torch.float32 ), latest_instruction_loss=None, latest_instruction_token_correct=None, latest_instruction_token_count=None, kv_cache=kwargs["kv_cache"], ) actions = torch.zeros((30, 39), dtype=torch.float32) actions[:, 36] = 2.0 step = { "source_id": "source-a", "step_index": 20, "vlm_hidden_states": torch.zeros((5, 8)), "attention_mask": torch.ones(5, dtype=torch.long), "actions": actions, "actions_mask": torch.ones((30, 39)), "states": torch.zeros((1, 39)), "smoothness_context_actions": torch.zeros((2, 39)), "progress": torch.zeros(5), "future_progress": torch.zeros((30, 5)), "update_semantic_cache": True, "control_label": LISTEN_TOKEN_ID, "action_control_labels": torch.tensor( [LISTEN_TOKEN_ID] * 15 + [MOTION_TOKEN_ID] * 15, dtype=torch.long, ), "control_transition": True, "motion_duration_target_seconds": 0.0, "motion_duration_target_mask": False, "latest_instruction_text": "", "latest_instruction_target_mask": False, } model = FakeModel() cache = SimpleNamespace() losses, _ = trainer._forward_stream_batch( model, ["source-a"], [step], cache, FixedSampler(), ) # Historical v12/v15 regenerate all 30 frames and mask the cross-phase # tail. v19 preserves the previous ten-frame RTC prefix and supervises the # complete valid future horizon so that preceding windows learn that tail. if semantics_version in {19, 20}: assert torch.all(model.captured_timestep[:, :10] == 0.0) assert torch.all(model.captured_timestep[:, 10:] == 500.0) else: assert torch.all(model.captured_timestep == 500.0) assert losses["cross_phase_frames"].tolist() == [15.0] assert losses["supervised_phase_frames"].tolist() == [ 30.0 if semantics_version in {19, 20} else 15.0 ] assert losses["control_transition"].tolist() == [1.0] assert float(losses["stationary_root_height_loss"].item()) > 0.0 @pytest.mark.parametrize("semantics_version", [18, 19, 20]) def test_completed_motion_progress_survives_cross_control_rtc_tail( semantics_version, ): trainer = PosttrainFullDuplexTrainer.__new__(PosttrainFullDuplexTrainer) trainer.device = torch.device("cpu") trainer.rtc_train_overlap = 10 trainer.train_diffusion_steps = 1_000 trainer.loss_w = torch.ones(39) trainer.loss_group_w = torch.ones(39) trainer.cfg = SimpleNamespace( train=SimpleNamespace( full_duplex_semantics_version=semantics_version, joint_velocity_loss_weight=0.10, joint_acceleration_loss_weight=0.05, root_continuity_loss_weight=0.10, stationary_root_height_loss_weight=1.0, ), model=SimpleNamespace(max_delay=11), ) class FixedSampler: def sample_like(self, actions): return SimpleNamespace( sigmas=torch.tensor([0.5], dtype=torch.float32), noise=torch.zeros_like(actions), delay=torch.tensor([10], dtype=torch.long), ) class FakeModel: def __init__(self): self.future_progress = None self._latest_reusable_condition = None def __call__(self, **kwargs): self.future_progress = kwargs["future_progress"].detach().clone() batch = kwargs["action_samples"].shape[0] logits = torch.zeros((batch, 3), dtype=torch.float32) probabilities = torch.softmax(logits, dim=-1) condition = torch.zeros((batch, 1, 8), dtype=torch.float32) mask = torch.ones((batch, 1), dtype=torch.long) self._latest_reusable_condition = { "condition_hidden_states": condition, "condition_attention_mask": mask, "control_logits": logits, "control_probabilities": probabilities, "control_condition_weights": probabilities, "control_entropy": torch.full((batch,), float(np.log(3.0))), "memory_token": condition, "motion_duration_log1p_seconds": torch.zeros( batch, dtype=torch.float32 ), } return SimpleNamespace( action=torch.zeros_like(kwargs["action_samples"]), control_logits=logits, control_probabilities=probabilities, control_entropy=torch.full((batch,), float(np.log(3.0))), motion_duration_log1p_seconds=torch.zeros( batch, dtype=torch.float32 ), latest_instruction_loss=None, latest_instruction_token_correct=None, latest_instruction_token_count=None, kv_cache=kwargs["kv_cache"], ) future_progress = torch.zeros((30, 5), dtype=torch.float32) future_progress[:15] = torch.tensor( [0.8, 0.2, 0.8, -0.5, 0.5], dtype=torch.float32 ) step = { "source_id": "source-a", "step_index": 100, "vlm_hidden_states": torch.zeros((5, 8)), "attention_mask": torch.ones(5, dtype=torch.long), "actions": torch.zeros((30, 39), dtype=torch.float32), "actions_mask": torch.ones((30, 39), dtype=torch.float32), "states": torch.zeros((1, 39), dtype=torch.float32), "smoothness_context_actions": torch.zeros((2, 39), dtype=torch.float32), "progress": torch.tensor([0.8, 0.2, 0.8, -0.5, 0.5]), # The raw GT timeline changes to stationary p=0 in the tail. "future_progress": future_progress, "update_semantic_cache": True, "control_label": MOTION_TOKEN_ID, "action_control_labels": torch.tensor( [MOTION_TOKEN_ID] * 15 + [IDLE_TOKEN_ID] * 15, dtype=torch.long, ), # This is the final motion window, not the next idle window. "control_transition": False, "motion_duration_target_seconds": 0.0, "motion_duration_target_mask": False, "latest_instruction_text": "", "latest_instruction_target_mask": False, } model = FakeModel() trainer._forward_stream_batch( model, ["source-a"], [step], SimpleNamespace(), FixedSampler(), ) torch.testing.assert_close( model.future_progress[0, :15], future_progress[:15], ) expected_tail = ( torch.tensor([1.0, 0.0, 1.0, 0.0, 1.0]).expand(15, -1) if semantics_version in {19, 20} else future_progress[15:] ) torch.testing.assert_close(model.future_progress[0, 15:], expected_tail) @pytest.mark.skip( reason=( "Legacy cached-hidden optimizer test; HFDv2 native-Qwen accumulation " "is covered by test_hfdv2_qwen_streaming.py" ) ) def test_training_step_calls_one_parallel_forward_per_stream_position(): config = FullDuplexPsi0ModelConfig( streaming_qwen_enabled=False, action_dim=3, action_chunk_size=30, action_exec_horizon=20, observation_horizon=1, odim=3, n_conditions=0, hidden_dim=48, num_blocks=1, dim_feedforward=64, nhead=4, view_feature_dim=8, use_future_progress_embedding=False, rtc=True, max_delay=11, streaming_memory_dim=16, streaming_memory_layers=1, streaming_memory_heads=4, streaming_max_cache_tokens=8, ) model = FullDuplexPsi0Model(config, vlm_model=None) model.enable_full_source_gradient_checkpointing() class FakeAccelerator: num_processes = 1 sync_gradients = True @staticmethod def accumulate(model): del model return contextlib.nullcontext() @staticmethod def no_sync(model): del model return contextlib.nullcontext() @staticmethod def autocast(): return contextlib.nullcontext() @staticmethod def backward(loss): loss.backward() @staticmethod def unwrap_model(model): return model @staticmethod def gather(value): return value class FixedSampler: @staticmethod def sample_like(actions): return SimpleNamespace( sigmas=torch.full((actions.shape[0],), 0.5), noise=torch.zeros_like(actions), ) class Scheduler: def __init__(self): self.step_calls = 0 def step(self): self.step_calls += 1 def get_last_lr(self): return [1e-4] trainer = PosttrainFullDuplexTrainer.__new__(PosttrainFullDuplexTrainer) trainer.device = torch.device("cpu") trainer.cfg = SimpleNamespace( train=SimpleNamespace( tbptt_unroll_steps=0, control_loss_weight=1.0, data_parallel="ddp", gradient_accumulation_steps=1, skip_resumed_steps=False, ) ) trainer.model = model trainer.accelerator = FakeAccelerator() trainer.flow_random_sampler = FixedSampler() trainer.rtc_train_overlap = 10 trainer.train_diffusion_steps = 1_000 trainer.loss_w = torch.ones(3) trainer.loss_group_w = torch.ones(3) class CountingSgd(torch.optim.SGD): def __init__(self, parameters): super().__init__(parameters, lr=1e-3) self.step_calls = 0 def step(self, closure=None): self.step_calls += 1 return super().step(closure) trainer.optimizer = CountingSgd(model.parameters()) trainer.lr_scheduler = Scheduler() def make_step(source_id: str, phase: str, value: float): labels = { "idle": IDLE_TOKEN_ID, "listen": LISTEN_TOKEN_ID, "motion": MOTION_TOKEN_ID, } return { "source_id": source_id, "vlm_hidden_states": torch.full((5, 8), value), "attention_mask": torch.ones(5, dtype=torch.long), "actions": torch.full((30, 3), value), "actions_mask": torch.ones(30, 3), "states": torch.zeros(1, 3), "progress": torch.zeros(5), "future_progress": torch.zeros(30, 5), "update_semantic_cache": True, "control_label": labels[phase], "phase": phase, } batch = { "sources": [ { "source_id": "source-a", "idle_steps": 1, "listen_steps": 1, "motion_steps": 1, "steps": [ make_step("source-a", "idle", 0.5), make_step("source-a", "listen", 1.0), make_step("source-a", "motion", 2.0), ], }, { "source_id": "source-b", "idle_steps": 0, "listen_steps": 2, "motion_steps": 1, "steps": [ make_step("source-b", "listen", 3.0), make_step("source-b", "motion", 4.0), make_step("source-b", "listen", 5.0), ], }, ] } seen_batch_sizes: list[int] = [] seen_cache_lengths: list[tuple[int, ...]] = [] def record_batch_size(module, args, kwargs): del module, args seen_batch_sizes.append(int(kwargs["vlm_hidden_states"].shape[0])) original_forward_stream_batch = trainer._forward_stream_batch def record_cache_lengths(model, source_ids, steps, cache, sampler): seen_cache_lengths.append(cache.sequence_lengths) return original_forward_stream_batch( model, source_ids, steps, cache, sampler ) trainer._forward_stream_batch = record_cache_lengths hook = model.register_forward_pre_hook(record_batch_size, with_kwargs=True) try: events = list( trainer.iter_optimizer_steps( batch, global_step_start=0, local_step=0, ) ) finally: hook.remove() # Three parallel time positions, not five serial source forwards. assert seen_batch_sizes == [2, 2, 2] assert seen_cache_lengths == [(0, 0), (1, 1), (2, 2)] assert len(events) == 1 assert trainer.optimizer.step_calls == 1 assert trainer.lr_scheduler.step_calls == 1 sync, metrics, source_group_end = events[0] assert sync is True assert source_group_end is True assert metrics["source_step_seconds"] > 0.0 assert metrics["active_windows_per_second"] > 0.0 assert np.isfinite(metrics["loss"]) assert metrics["loss"] == pytest.approx( metrics["loss_action"] + trainer.train_cfg.control_loss_weight * metrics["loss_control"] ) assert { "loss_objective", "loss_window_mean", "sources_per_second", "optimizer_source_windows", "parallel_source_batch", "peak_memory_allocated_gib", "peak_memory_reserved_gib", "idle_steps", "listen_steps", "motion_steps", "full_source_bptt", "source_group_complete", "chunks_in_source_group", "chunk_index", "optimizer_chunk_steps", }.isdisjoint(metrics) # A max-step stop never starts and truncates a source graph. seen_cache_lengths.clear() stopped_events = list( trainer.iter_optimizer_steps( batch, global_step_start=1, local_step=1, max_optimizer_events=0, ) ) assert stopped_events == [] assert trainer.optimizer.step_calls == 1 assert trainer.lr_scheduler.step_calls == 1 assert seen_cache_lengths == [] assert trainer._last_cache_state == "empty_source_boundary" def test_fully_padded_rank_still_advances_one_flow_draw(): trainer = PosttrainFullDuplexTrainer.__new__(PosttrainFullDuplexTrainer) trainer.device = torch.device("cpu") class CountingSampler: def __init__(self): self.shapes = [] def sample_like(self, actions): self.shapes.append(tuple(actions.shape)) return None trainer.flow_random_sampler = CountingSampler() trainer._advance_padded_flow_rng( SimpleNamespace(action_exec_horizon=20, action_dim=3), source_batch_size=2, ) assert trainer.flow_random_sampler.shapes == [(2, 30, 3)] def test_cache_free_checkpoint_marker_records_window_step_training( tmp_path, monkeypatch, ): contract = { "format": "hfd_qwen_serial_train_infer_contract_v6", "identity": "test", } data_contract = { "format": "hfd_continuous_office_production_v1", "source_count": 2, "mixed_window_count": 9, "background_source_count": 1, "background_window_count": 20, "initial_idle_max_steps": 5, "stationary_blend_frames": 20, "safety_steps": 10, "assignment_seed": 260730, "assignment_sha256": "assignment", } monkeypatch.setattr( "psi.trainers.posttrain_full_duplex." "_active_continuous_data_contract_identity", lambda: data_contract, ) trainer = PosttrainFullDuplexTrainer.__new__(PosttrainFullDuplexTrainer) trainer.cfg = SimpleNamespace( train=SimpleNamespace( full_duplex_semantics_version=11, joint_velocity_loss_weight=0.10, joint_acceleration_loss_weight=0.05, root_continuity_loss_weight=0.10, tbptt_unroll_steps=0, full_source_gradient_checkpointing=True, qwen_kv_gradient_horizon_windows=1, source_accumulation_groups=1, train_batch_size=8, ), model=SimpleNamespace( streaming_qwen_trainable_last_layers=2, streaming_qwen_train_final_norm=True, control_condition_train_mode="hard", control_condition_inference_mode="hard", ), ) trainer._hfdv2_checkpoint_contract = lambda: contract class FakeAccelerator: is_main_process = True @staticmethod def wait_for_everyone(): return None trainer.accelerator = FakeAccelerator() trainer._last_cache_state = "empty_source_boundary" trainer._last_source_group_complete = True trainer._write_full_duplex_boundary_marker( str(tmp_path), global_step=7, ) marker = json.loads( (tmp_path / "full_duplex_source_boundary.json").read_text() ) assert marker["global_step"] == 7 assert marker["full_duplex_semantics_version"] == 11 assert marker["architecture"] == "hfd_qwen_native_kv_serial_control" assert marker["online_raw_audio"] is True assert marker["semantic_memory"] == "qwen_dynamic_cache" assert marker["action_streaming_memory"] is False assert marker["serial_control_conditioning"] is True assert marker["control_condition_train_mode"] == "hard" assert marker["control_condition_inference_mode"] == "hard" assert marker["control_condition_source"] == "predicted_no_ground_truth" assert marker["joint_velocity_loss_weight"] == 0.10 assert marker["joint_acceleration_loss_weight"] == 0.05 assert marker["root_continuity_loss_weight"] == 0.10 assert marker["optimizer_scope"] == "one_aligned_stream_window" assert marker["optimizer_chunk_steps"] == 1 assert marker["source_tail_loss_normalization"] == ( "fixed_global_source_capacity" ) assert marker["source_slot_policy"] == ( "qwen_boundary_dynamic_refill_v1" ) assert marker["qwen_kv_gradient_horizon_windows"] == 1 assert marker["source_accumulation_groups"] == 1 assert marker["effective_sources_per_gpu"] == 8 assert marker["action_gradient_checkpointing"] is True assert marker["cache_state"] == "empty_source_boundary" assert marker["source_group_complete"] is True assert marker["train_infer_contract"] == contract assert marker["continuous_audio_data_contract"] == data_contract def test_small_full_duplex_model_has_finite_action_and_control_gradients(): config = FullDuplexPsi0ModelConfig( streaming_qwen_enabled=False, action_dim=3, action_chunk_size=30, action_exec_horizon=20, observation_horizon=1, odim=3, n_conditions=0, hidden_dim=48, num_blocks=1, dim_feedforward=64, nhead=4, view_feature_dim=8, use_future_progress_embedding=False, rtc=True, max_delay=11, streaming_memory_dim=16, streaming_memory_layers=1, streaming_memory_heads=4, streaming_max_cache_tokens=8, ) model = FullDuplexPsi0Model(config, vlm_model=None) cache = model.begin_source(["source-a"]) output = model( source_ids=["source-a"], kv_cache=cache, update_semantic_cache=True, vlm_hidden_states=torch.randn(1, 5, 8), attention_mask=torch.ones(1, 5, dtype=torch.long), action_samples=torch.randn(1, 20, 3), known_action_prefix=torch.randn(1, 10, 3), states=torch.randn(1, 1, 3), timestep=torch.tensor([500.0]), ) assert output.action.shape == (1, 20, 3) assert output.control_logits.shape == (1, 3) loss = output.action.float().square().mean() + F.cross_entropy( output.control_logits.float(), torch.tensor([LISTEN_TOKEN_ID]) ) assert torch.isfinite(loss) loss.backward() assert model.control_head[-1].weight.grad is not None assert torch.isfinite(model.control_head[-1].weight.grad).all() @pytest.mark.parametrize("action_recompute", [False, True]) def test_late_window_loss_reaches_early_source_input_through_kv_graph( action_recompute, ): config = FullDuplexPsi0ModelConfig( streaming_qwen_enabled=False, action_dim=3, action_chunk_size=30, action_exec_horizon=20, observation_horizon=1, odim=3, n_conditions=0, hidden_dim=48, num_blocks=1, dim_feedforward=64, nhead=4, view_feature_dim=8, use_future_progress_embedding=False, rtc=True, max_delay=11, streaming_memory_dim=16, streaming_memory_layers=1, streaming_memory_heads=4, streaming_max_cache_tokens=8, ) model = FullDuplexPsi0Model(config, vlm_model=None) if action_recompute: model.enable_full_source_gradient_checkpointing() cache = model.begin_source(["source-a"]) early_hidden = torch.randn(1, 5, 8, requires_grad=True) early = model( source_ids=["source-a"], kv_cache=cache, update_semantic_cache=True, vlm_hidden_states=early_hidden, attention_mask=torch.ones(1, 5, dtype=torch.long), action_samples=torch.randn(1, 20, 3), known_action_prefix=torch.randn(1, 10, 3), states=torch.randn(1, 1, 3), timestep=torch.tensor([500.0]), ) late = model( source_ids=["source-a"], kv_cache=early.kv_cache, update_semantic_cache=True, vlm_hidden_states=torch.randn(1, 5, 8), attention_mask=torch.ones(1, 5, dtype=torch.long), action_samples=torch.randn(1, 20, 3), known_action_prefix=torch.randn(1, 10, 3), states=torch.randn(1, 1, 3), timestep=torch.tensor([500.0]), ) late_loss = late.action.float().square().mean() + F.cross_entropy( late.control_logits.float(), torch.tensor([MOTION_TOKEN_ID]), ) late_loss.backward() assert early_hidden.grad is not None assert torch.isfinite(early_hidden.grad).all() assert float(early_hidden.grad.abs().sum()) > 0.0 def test_stream_condition_is_encoded_once_across_flow_solver_steps(): config = FullDuplexPsi0ModelConfig( streaming_qwen_enabled=False, action_dim=3, action_chunk_size=30, action_exec_horizon=20, observation_horizon=1, odim=3, n_conditions=0, hidden_dim=48, num_blocks=1, dim_feedforward=64, nhead=4, view_feature_dim=8, use_future_progress_embedding=False, rtc=True, max_delay=11, streaming_memory_dim=16, streaming_memory_layers=1, streaming_memory_heads=4, streaming_max_cache_tokens=8, ) model = FullDuplexPsi0Model(config, vlm_model=None).eval() cache = model.begin_source(["source-a"]) speech = model.encode_stream_input( source_ids=["source-a"], kv_cache=cache, update_semantic_cache=True, vlm_hidden_states=torch.randn(1, 5, 8), attention_mask=torch.ones(1, 5, dtype=torch.long), ) assert speech.kv_cache.sequence_length == 1 for timestep in (900.0, 100.0): action = model.forward_action_conditioned( condition=speech, action_samples=torch.randn(1, 20, 3), known_action_prefix=torch.randn(1, 10, 3), states=torch.randn(1, 1, 3), timestep=torch.tensor([timestep]), ) assert action.shape == (1, 20, 3) assert speech.kv_cache.sequence_length == 1 padded = model.encode_stream_input( source_ids=["source-a"], kv_cache=speech.kv_cache, update_semantic_cache=False, vlm_hidden_states=torch.randn(1, 4, 8), attention_mask=torch.ones(1, 4, dtype=torch.long), ) assert padded.kv_cache.sequence_length == 1 def test_length_group_sampler_balances_each_global_rank_group(): class Dataset: lengths = [100, 1, 98, 2, 96, 3, 94, 4] def __len__(self): return len(self.lengths) def estimated_stream_steps(self, index): return self.lengths[index] dataset = Dataset() sampler = SourceLengthGroupedSampler( dataset, world_size=2, batch_size=1, seed=17 ) assert len(sampler) // sampler.group_size == 4 first = list(sampler) sampler.set_epoch(1) second = list(sampler) assert sorted(first) == list(range(len(dataset))) assert sorted(second) == list(range(len(dataset))) assert first != second for offset in range(0, len(first), 2): pair = [dataset.lengths[index] for index in first[offset : offset + 2]] assert max(pair) - min(pair) <= 3 def test_length_group_sampler_balances_long_source_work_across_ranks(): class Dataset: lengths = [1_000, 900, 800, 700, 600, 500, 400, 300] def __len__(self): return len(self.lengths) def estimated_stream_steps(self, index): return self.lengths[index] dataset = Dataset() sampler = SourceLengthGroupedSampler( dataset, world_size=2, batch_size=4, seed=23 ) order = list(sampler) rank_rows = [order[rank::2] for rank in range(2)] assert all(len(rows) == 4 for rows in rank_rows) assert { max(dataset.lengths[index] for index in rows) for rows in rank_rows } == {900, 1_000} for offset in range(0, len(order), 2): aligned = [ dataset.lengths[index] for index in order[offset : offset + 2] ] assert max(aligned) - min(aligned) <= 100 def test_source_batching_loader_groups_independently_loaded_sources(): singleton_loader = [ {"sources": [{"source_id": f"source-{index}"}]} for index in range(6) ] loader = SourceBatchingLoader(singleton_loader, batch_size=3) assert len(loader) == 2 batches = list(loader) assert [ [source["source_id"] for source in batch["sources"]] for batch in batches ] == [ ["source-0", "source-1", "source-2"], ["source-3", "source-4", "source-5"], ] def test_source_batching_loader_rebuckets_materialized_exact_lengths(): lengths = [8, 1, 7, 2] singleton_loader = [ { "sources": [ { "source_id": f"source-{index}", "steps": [{} for _ in range(length)], } ] } for index, length in enumerate(lengths) ] loader = SourceBatchingLoader( singleton_loader, batch_size=2, length_lookahead_batches=2, ) batches = list(loader) assert [ [len(source["steps"]) for source in batch["sources"]] for batch in batches ] == [[1, 2], [7, 8]] assert sorted( source["source_id"] for batch in batches for source in batch["sources"] ) == [f"source-{index}" for index in range(4)] def test_source_batching_loader_flushes_short_final_lookahead_block(): singleton_loader = [ { "sources": [ { "source_id": f"source-{index}", "steps": [{} for _ in range(length)], } ] } for index, length in enumerate([6, 1, 5, 2, 4, 3]) ] loader = SourceBatchingLoader( singleton_loader, batch_size=2, length_lookahead_batches=2, ) assert [ [len(source["steps"]) for source in batch["sources"]] for batch in loader ] == [[1, 2], [5, 6], [3, 4]] def test_source_prefetch_defaults_to_two_complete_local_batches(monkeypatch): monkeypatch.delenv("PSI_DATALOADER_PREFETCH_FACTOR", raising=False) assert _source_prefetch_factor(workers=16, local_source_batch=64) == 8 monkeypatch.setenv("PSI_DATALOADER_PREFETCH_FACTOR", "3") assert _source_prefetch_factor(workers=16, local_source_batch=64) == 3 def test_source_prefetch_keeps_two_batches_ready_for_exact_lookahead(monkeypatch): monkeypatch.delenv("PSI_DATALOADER_PREFETCH_FACTOR", raising=False) assert ( _source_prefetch_factor( workers=16, local_source_batch=128, ) == 16 ) def test_session_voice_lookup_index_preserves_exact_audio_source(): dataset = FullDuplexAudioCommandDataset.__new__( FullDuplexAudioCommandDataset ) voices = ( AudioSource("seed_audio/000001/female_01_en", ()), AudioSource("seed_audio/000001/female_01_zh", ()), ) dataset.audio_catalog = {"seed_audio/000001": voices} dataset._audio_catalog_by_voice_name = { "seed_audio/000001": { dataset._voice_name(voice): voice for voice in voices } } record = { "row_id": 1, "audio_dir": "seed_audio/000001", "_session_voice_name": "female_01_zh", } assert dataset._voice_for_record(record) is voices[1] record["_exclude_zh_voices"] = True with pytest.raises(RuntimeError, match="unavailable or ambiguous"): dataset._voice_for_record(record) def test_motion_cache_is_reserved_for_reused_stationary_sources( monkeypatch, tmp_path ): dataset = FullDuplexAudioCommandDataset.__new__( FullDuplexAudioCommandDataset ) stationary = tmp_path / "stationary" task = tmp_path / "task" dataset.motion_cache_size = 256 dataset._motion_cache = OrderedDict() dataset._stationary_motion_paths = frozenset({str(stationary)}) reads = {str(stationary): 0, str(task): 0} class FakeFrame: def __init__(self, values): self.values = values def to_numpy(self, dtype): return self.values.astype(dtype) def fake_read_csv(path): motion_dir = str(Path(path).parent) reads[motion_dir] += 1 columns = { "joint_pos.csv": 29, "body_pos.csv": 3, "body_quat.csv": 4, }[Path(path).name] return FakeFrame(np.zeros((2, columns), dtype=np.float32)) monkeypatch.setattr( "psi.data.seed.full_duplex_audio_command_dataset.pd.read_csv", fake_read_csv, ) dataset._load_motion(stationary) dataset._load_motion(stationary) dataset._load_motion(task) dataset._load_motion(task) assert reads[str(stationary)] == 3 assert reads[str(task)] == 6 assert list(dataset._motion_cache) == [str(stationary)] def test_manual_gradient_average_includes_remote_missing_grad(monkeypatch): first = torch.nn.Parameter(torch.zeros(2)) second = torch.nn.Parameter(torch.zeros(1)) first.grad = torch.tensor([2.0, 4.0]) second.grad = None monkeypatch.setattr(torch.distributed, "is_initialized", lambda: True) def fake_all_reduce(flat, op): del op flat.add_(torch.tensor([4.0, 8.0, 6.0])) monkeypatch.setattr(torch.distributed, "all_reduce", fake_all_reduce) average_full_duplex_gradients([first, second], world_size=2, bucket_cap_mb=1) torch.testing.assert_close(first.grad, torch.tensor([3.0, 6.0])) torch.testing.assert_close(second.grad, torch.tensor([3.0])) def test_audio_windows_and_motion_start_use_integer_sample_timeline(): assert complete_audio_clip_starts(9_599) == [] assert complete_audio_clip_starts(16_100) == [0, 6_400] assert aligned_motion_start_frame(16_000) == 50 assert aligned_motion_start_frame(16_100) == 51