Download tests/test_full_duplex_audio_command.py from Tinghong-Ye22/code-HFD: direct link, hf CLI and curl.
- Browser
- Download file 144 kB
-
https://huggingface.co/Tinghong-Ye22/code-HFD/resolve/main/tests/test_full_duplex_audio_command.py
- Command line
-
hf download hf://Tinghong-Ye22/code-HFD/tests/test_full_duplex_audio_command.py
-
curl -L -o test_full_duplex_audio_command.py https://huggingface.co/Tinghong-Ye22/code-HFD/resolve/main/tests/test_full_duplex_audio_command.py
144 kB
from __future__ import annotations
import hashlib
import contextlib
import importlib.util
import json
import random
import sqlite3
import sys
import wave
from collections import OrderedDict
from pathlib import Path
from types import SimpleNamespace
import numpy as np
import pytest
import torch
import torch.nn.functional as F
from torch import nn
from safetensors.torch import save_file
from psi.config.transform import SeedPosttrainRepackTransform
from psi.config.model_full_duplex_psi0 import FullDuplexPsi0ModelConfig
from psi.data.seed.full_duplex_audio_command_dataset import (
AudioSource,
BackgroundWaveformSource,
CANONICAL_MOTION_MANIFEST_FORMAT,
DISFLUENCY_UNCERTAINTY_ROLES,
DisfluencyAudioSource,
FullDuplexAudioCommandDataset,
FullDuplexTransformedDataset,
HiddenSlice,
IDLE_TOKEN_ID,
LISTEN_TOKEN_ID,
MOTION_TOKEN_ID,
SourceBackgroundAssignment,
StationaryMotionSource,
detect_clean_speech_onset_sample,
has_known_tts_translation_mismatch,
instruction_text_for_voice,
load_canonical_motion_selection,
)
from psi.data.seed.full_duplex_audio_contract import (
AUDIO_COMMAND_STREAMING_PROMPT,
HFD_AUDIO_SOURCE_FORMAT,
HFD_ONLINE_ENCODER_ATTENTION,
HFD_ONLINE_ENCODER_BATCH_SIZE,
HFD_ONLINE_ENCODER_CONTRACT,
HFD_ONLINE_ENCODER_DTYPE,
)
from psi.models.full_duplex_psi0 import FullDuplexPsi0Model, StreamingCausalMemory
from psi.trainers.posttrain_full_duplex import (
PosttrainFullDuplexTrainer,
SourceBatchingLoader,
SourceLengthGroupedSampler,
_source_prefetch_factor,
average_full_duplex_gradients,
)
from psi.data.seed.full_duplex_timing import (
aligned_motion_start_frame,
complete_audio_clip_starts,
)
from psi.data.seed.motion_repr import (
local_delta_from_world,
quat_to_yaw_wxyz,
rpy_to_quat_wxyz,
wrap_angle,
)
from psi.data.seed.source_split import select_source_row_ids
def test_train_and_eval_are_disjoint_at_complete_source_level():
source_ids = ["source-a", "source-a", "source-b", "source-c", "source-c"]
train_rows = select_source_row_ids(
source_ids, split="train", val_ratio=0.25, split_seed=42
)
eval_rows = select_source_row_ids(
source_ids, split="val", val_ratio=0.25, split_seed=42
)
train_sources = {source_ids[index] for index in train_rows}
eval_sources = {source_ids[index] for index in eval_rows}
assert train_sources.isdisjoint(eval_sources)
def test_clean_speech_onset_ignores_leading_silence_and_pcm_click():
rate = 16_000
waveform = np.zeros(rate, dtype=np.float32)
waveform[800] = 0.8
start = 4_000
time_axis = np.arange(rate - start, dtype=np.float32) / rate
waveform[start:] = 0.1 * np.sin(2.0 * np.pi * 220.0 * time_axis)
onset = detect_clean_speech_onset_sample(waveform, rate)
assert start - 320 <= onset <= start + 160
def test_latest_instruction_text_matches_selected_tts_language():
record = {
"row_id": 7,
"instruction": "walk to the left",
"instruction_zh": "向左走",
}
english = AudioSource(
source_key="seed_audio/000007/female_01_en",
clips=(),
)
chinese = AudioSource(
source_key="seed_audio/000007/male_01_zh",
clips=(),
)
assert instruction_text_for_voice(record, english) == "walk to the left"
assert instruction_text_for_voice(record, chinese) == "向左走"
def test_known_retro_translation_mismatch_excludes_only_chinese_voices():
assert has_known_tts_translation_mismatch(
"retro dance jackson jump up",
"后退舞步杰克逊跳起",
)
assert not has_known_tts_translation_mismatch(
"walk backward five steps",
"后退五步",
)
dataset = FullDuplexAudioCommandDataset.__new__(
FullDuplexAudioCommandDataset
)
dataset.audio_catalog = {
"seed_audio/004603": (
AudioSource("seed_audio/004603/female_01_en", ()),
AudioSource("seed_audio/004603/female_01_zh", ()),
AudioSource("seed_audio/004603/male_01_en", ()),
AudioSource("seed_audio/004603/male_01_zh", ()),
)
}
record = {
"row_id": 1,
"audio_dir": "seed_audio/004603",
"_exclude_zh_voices": True,
}
assert dataset._session_voice_names(record) == (
"female_01_en",
"male_01_en",
)
def test_canonical_motion_manifest_loads_exact_bilingual_mapping(tmp_path):
path = tmp_path / "canonical.json"
path.write_text(
json.dumps(
{
"format": CANONICAL_MOTION_MANIFEST_FORMAT,
"excluded_ambiguous_chinese_instructions": [],
"selections": [
{
"instruction": "wave two hands in front",
"instruction_zh": "双手在身前挥动",
"filename": "wave_two_hands_R_001__A533",
},
{
"instruction": "walk forward",
"instruction_zh": "向前走",
"filename": "walk_forward_001__A100",
},
],
},
ensure_ascii=False,
),
encoding="utf-8",
)
assert load_canonical_motion_selection(path) == {
(
"wave two hands in front",
"双手在身前挥动",
): "wave_two_hands_R_001__A533",
("walk forward", "向前走"): "walk_forward_001__A100",
}
def test_canonical_motion_manifest_rejects_duplicate_motion(tmp_path):
path = tmp_path / "canonical.json"
path.write_text(
json.dumps(
{
"format": CANONICAL_MOTION_MANIFEST_FORMAT,
"excluded_ambiguous_chinese_instructions": [],
"selections": [
{
"instruction": "first command",
"instruction_zh": "第一条指令",
"filename": "same_motion",
},
{
"instruction": "second command",
"instruction_zh": "第二条指令",
"filename": "same_motion",
},
],
},
ensure_ascii=False,
),
encoding="utf-8",
)
with pytest.raises(RuntimeError, match="cannot represent two"):
load_canonical_motion_selection(path)
def test_canonical_motion_manifest_requires_ambiguous_chinese_exclusion(
tmp_path,
):
path = tmp_path / "canonical.json"
path.write_text(
json.dumps(
{
"format": CANONICAL_MOTION_MANIFEST_FORMAT,
"excluded_ambiguous_chinese_instructions": [],
"selections": [
{
"instruction": "walk forward",
"instruction_zh": "向前走",
"filename": "walk_forward",
},
{
"instruction": "start walking forward",
"instruction_zh": "向前走",
"filename": "walk_forward_start",
},
],
},
ensure_ascii=False,
),
encoding="utf-8",
)
with pytest.raises(RuntimeError, match="exclude every Chinese prompt"):
load_canonical_motion_selection(path)
def test_multiturn_partition_uses_every_split_record_once():
dataset = FullDuplexAudioCommandDataset.__new__(
FullDuplexAudioCommandDataset
)
dataset.multi_turn_sessions = True
dataset.session_min_turns = 2
dataset.session_max_turns = 4
dataset.split = "train"
dataset.split_seed = 42
records = [{"row_id": index} for index in range(23)]
first = dataset._compose_sessions(records)
second = dataset._compose_sessions(records)
assert first == second
assert all(2 <= len(session) <= 4 for session in first)
flattened = [record["row_id"] for session in first for record in session]
assert sorted(flattened) == list(range(23))
assert len(flattened) == len(set(flattened))
def _session_voice_fixture(split: str = "train"):
dataset = FullDuplexAudioCommandDataset.__new__(
FullDuplexAudioCommandDataset
)
dataset.multi_turn_sessions = True
dataset.session_min_turns = 2
dataset.session_max_turns = 4
dataset.session_voice_variants = 2
dataset.split = split
dataset.split_seed = 42
records = [
{"row_id": index, "audio_dir": f"audio-{index}"}
for index in range(32)
]
voice_names = (
"female_01_en",
"female_01_zh",
"male_01_en",
"male_01_zh",
)
dataset.audio_catalog = {
record["audio_dir"]: tuple(
AudioSource(
source_key=f"{record['audio_dir']}/{voice_name}",
clips=(),
packed_path=Path(f"/{record['audio_dir']}/{voice_name}"),
packed_count=voice_index + 2,
)
for voice_index, voice_name in enumerate(voice_names)
)
for record in records
}
return dataset, records
def test_one_to_four_partition_keeps_every_record_and_multiple_turn_lengths():
dataset, _ = _session_voice_fixture()
dataset.session_min_turns = 1
dataset.session_voice_variants = 0
records = [{"row_id": i} for i in range(1000)]
sessions = dataset._compose_sessions(records)
assert sessions == dataset._compose_sessions(records)
assert {len(s) for s in sessions} == {1, 2, 3, 4}
assert sorted(r["row_id"] for s in sessions for r in s) == list(range(1000))
assert dataset._compose_sessions(records[:1]) == ((records[0],),)
@pytest.mark.parametrize("split", ["train", "val"])
def test_single_turn_allowed_but_command_interruption_still_has_two_sources(split):
dataset, records = _session_voice_fixture(split)
dataset.session_min_turns = 1
dataset.interruption_sessions = True
dataset.independent_task_pools = True
dataset.interruption_min_start_seconds = 2.
dataset.interruption_min_remaining_seconds = 1.5
dataset.motion_fps = 50
dataset.task_session_variants = {
"turn-taking": 2, "interruption": 2, "emergency-stop": 2,
"safe-hold": 0, "backchannel": 2,
}
dataset.control_task_session_ratio = .1
dataset.control_followup_ratio = .5
dataset._estimated_runtime_motion_frames = lambda r: 400
dataset._control_motion_is_eligible = lambda *args, **kwargs: True
dataset._backchannel_motion_is_eligible = lambda *args, **kwargs: True
sessions = dataset._compose_sessions(records)
assert sessions == dataset._compose_sessions(records)
for mode, repeats in dataset.task_session_variants.items():
selected = [s for s in sessions if s[0]["_session_interaction_mode"] == mode]
count = repeats if split == "train" else int(repeats > 0)
assert sorted(r["row_id"] for s in selected for r in s) == sorted(list(range(32))*count)
if mode == "interruption":
assert all(2 <= len(s) <= 4 for s in selected)
elif selected:
assert any(len(s) == 1 for s in selected)
for session in selected:
assert len({r["_session_voice_name"] for r in session}) == 1
if mode == "emergency-stop" and len(session) == 1:
assert session[0]["_session_control_turn_index"] == 0
assert not session[0]["_session_control_has_followup"]
def test_singleton_voice_bucket_does_not_require_a_different_speaker():
dataset, records = _session_voice_fixture("val")
dataset.session_min_turns = 1
sessions = dataset._compose_sessions(records[:1])
assert len(sessions) == len(sessions[0]) == 1
assert sessions[0][0]["row_id"] == records[0]["row_id"]
def test_train_session_voice_variants_fix_one_voice_and_expand_each_motion():
dataset, records = _session_voice_fixture("train")
first = dataset._compose_sessions(records)
second = dataset._compose_sessions(records)
assert first == second
assert sum(len(session) for session in first) == 2 * len(records)
row_voices: dict[int, set[str]] = {
int(record["row_id"]): set() for record in records
}
row_neighbours: dict[int, set[tuple[int, ...]]] = {
int(record["row_id"]): set() for record in records
}
for session in first:
assert 2 <= len(session) <= 4
assert len({record["_session_voice_name"] for record in session}) == 1
row_ids = tuple(sorted(int(record["row_id"]) for record in session))
for record in session:
row_id = int(record["row_id"])
row_voices[row_id].add(str(record["_session_voice_name"]))
row_neighbours[row_id].add(row_ids)
selected = dataset._voice_for_record(record)
assert Path(selected.source_key).name == record["_session_voice_name"]
assert all(len(voices) == 2 for voices in row_voices.values())
assert any(len(neighbours) == 2 for neighbours in row_neighbours.values())
def test_validation_uses_one_deterministic_same_voice_session_variant():
dataset, records = _session_voice_fixture("val")
sessions = dataset._compose_sessions(records)
flattened = [record for session in sessions for record in session]
assert len(flattened) == len(records)
assert sorted(int(record["row_id"]) for record in flattened) == list(
range(len(records))
)
assert all(record["_session_voice_variant"] == 0 for record in flattened)
assert all(
len({record["_session_voice_name"] for record in session}) == 1
for session in sessions
)
@pytest.mark.parametrize("split", ["train", "val"])
def test_interruption_composition_is_exactly_one_primitive_use_per_mode(split):
dataset, records = _session_voice_fixture(split)
dataset.interruption_sessions = True
dataset.interruption_ratio = 0.5
dataset.interruption_min_start_seconds = 2.0
dataset.interruption_max_start_seconds = 6.0
dataset.interruption_min_remaining_seconds = 1.5
dataset.motion_fps = 50
dataset._estimated_runtime_motion_frames = lambda record: (
120 if int(record["row_id"]) % 9 == 0 else 400
)
first = dataset._compose_sessions(records)
second = dataset._compose_sessions(records)
assert first == second
assert len(
{
session[0]["_session_interaction_mode"]
for session in first[: min(8, len(first))]
}
) == 2
by_mode: dict[str, list[dict]] = {
"turn-taking": [],
"interruption": [],
}
for session in first:
modes = {record["_session_interaction_mode"] for record in session}
voices = {record["_session_voice_name"] for record in session}
assert len(modes) == len(voices) == 1
mode = modes.pop()
by_mode[mode].extend(session)
assert 2 <= len(session) <= 4
if mode == "interruption":
assert all(
dataset._interruption_motion_is_eligible(record)
for record in session[:-1]
)
expected_rows = list(range(len(records)))
assert sorted(record["row_id"] for record in by_mode["turn-taking"]) == (
expected_rows
)
assert sorted(record["row_id"] for record in by_mode["interruption"]) == (
expected_rows
)
if split == "train":
row_voices = {
row_id: {
record["_session_voice_name"]
for mode_records in by_mode.values()
for record in mode_records
if int(record["row_id"]) == row_id
}
for row_id in expected_rows
}
assert all(len(voices) == 2 for voices in row_voices.values())
else:
row_voices = {
row_id: [
record["_session_voice_name"]
for mode_records in by_mode.values()
for record in mode_records
if int(record["row_id"]) == row_id
]
for row_id in expected_rows
}
assert all(len(set(voices)) == 1 for voices in row_voices.values())
def test_v21_independent_pools_preserve_complete_turn_taking_reference():
reference, reference_records = _session_voice_fixture("train")
reference_sessions = reference._compose_sessions(reference_records)
dataset, records = _session_voice_fixture("train")
dataset.interruption_sessions = True
dataset.interruption_ratio = 0.5
dataset.interruption_min_start_seconds = 2.0
dataset.interruption_max_start_seconds = 6.0
dataset.interruption_min_remaining_seconds = 1.5
dataset.motion_fps = 50
dataset.independent_task_pools = True
dataset.task_session_variants = {
"turn-taking": 2,
"interruption": 2,
"emergency-stop": 1,
"safe-hold": 1,
}
dataset._estimated_runtime_motion_frames = lambda record: 400
dataset._control_motion_is_eligible = lambda *args, **kwargs: True
sessions = dataset._compose_sessions(records)
by_mode = {
mode: [
session
for session in sessions
if session[0]["_session_interaction_mode"] == mode
]
for mode in dataset.task_session_variants
}
def signature(values):
return sorted(
(
int(session[0]["_session_voice_variant"]),
str(session[0]["_session_voice_name"]),
tuple(int(record["row_id"]) for record in session),
)
for session in values
)
assert signature(by_mode["turn-taking"]) == signature(reference_sessions)
assert sorted(
int(session[0]["_session_pool_index"])
for session in by_mode["turn-taking"]
) == list(range(len(reference_sessions)))
primitive_uses = {
mode: sum(len(session) for session in values)
for mode, values in by_mode.items()
}
assert primitive_uses == {
"turn-taking": 2 * len(records),
"interruption": 2 * len(records),
"emergency-stop": len(records),
"safe-hold": len(records),
}
assert dataset.control_task_rows_filtered == {
"emergency-stop": 0,
"safe-hold": 0,
}
def test_independent_pool_reuses_voice_only_after_full_coverage():
dataset, records = _session_voice_fixture("train")
dataset.interruption_sessions = True
dataset.interruption_ratio = 0.5
dataset.interruption_min_start_seconds = 2.0
dataset.interruption_max_start_seconds = 6.0
dataset.interruption_min_remaining_seconds = 1.5
dataset.motion_fps = 50
dataset.independent_task_pools = True
dataset.task_session_variants = {
"turn-taking": 5,
"interruption": 1,
"emergency-stop": 0,
"safe-hold": 0,
}
dataset._estimated_runtime_motion_frames = lambda record: 400
sessions = dataset._compose_sessions(records)
turn_rows = [
record
for session in sessions
if session[0]["_session_interaction_mode"] == "turn-taking"
for record in session
]
by_row = {
row_id: [
str(record["_session_voice_name"])
for record in turn_rows
if int(record["row_id"]) == row_id
]
for row_id in range(len(records))
}
assert all(len(voices) == 5 for voices in by_row.values())
assert all(len(set(voices)) == 4 for voices in by_row.values())
assert dataset.voice_reuse_after_coverage["turn-taking"] == len(records)
assert dataset.voice_reuse_after_coverage["interruption"] == 0
def test_balanced_control_pools_are_eight_percent_with_one_event_per_session():
dataset, records = _session_voice_fixture("train")
dataset.interruption_sessions = True
dataset.interruption_ratio = 0.5
dataset.interruption_min_start_seconds = 2.0
dataset.interruption_max_start_seconds = 6.0
dataset.interruption_min_remaining_seconds = 1.5
dataset.motion_fps = 50
dataset.independent_task_pools = True
dataset.task_session_variants = {
"turn-taking": 2,
"interruption": 2,
"emergency-stop": 1,
"safe-hold": 1,
}
dataset.control_task_session_ratio = 0.08
dataset.control_followup_ratio = 0.5
dataset._estimated_runtime_motion_frames = lambda record: 400
dataset._control_motion_is_eligible = lambda *args, **kwargs: True
unbalanced = dataset._compose_sessions(records)
sessions = dataset._balance_control_task_sessions(unbalanced)
counts = {
mode: sum(
session[0]["_session_interaction_mode"] == mode
for session in sessions
)
for mode in dataset.task_session_variants
}
assert counts["turn-taking"] + counts["interruption"] > 0
assert counts["emergency-stop"] == counts["safe-hold"]
assert counts["emergency-stop"] / len(sessions) == pytest.approx(
0.08,
abs=0.01,
)
control_sessions = [
session
for session in sessions
if session[0]["_session_interaction_mode"]
in {"emergency-stop", "safe-hold"}
]
for session in control_sessions:
turn_indices = {
int(record["_session_control_turn_index"])
for record in session
}
followup_flags = {
bool(record["_session_control_has_followup"])
for record in session
}
assert len(turn_indices) == len(followup_flags) == 1
control_turn = turn_indices.pop()
has_followup = followup_flags.pop()
assert has_followup == (control_turn + 1 < len(session))
assert any(
session[0]["_session_control_has_followup"]
for session in control_sessions
)
assert any(
not session[0]["_session_control_has_followup"]
for session in control_sessions
)
def test_v22_backchannel_final_session_mix_is_approximately_60_15_10_15():
dataset, records = _session_voice_fixture("train")
dataset.interruption_sessions = True
dataset.interruption_ratio = 0.5
dataset.interruption_min_start_seconds = 2.0
dataset.interruption_max_start_seconds = 6.0
dataset.interruption_min_remaining_seconds = 1.5
dataset.motion_fps = 50
dataset.independent_task_pools = True
dataset.task_session_variants = {
"turn-taking": 4,
"interruption": 1,
"emergency-stop": 2,
"safe-hold": 0,
"backchannel": 2,
}
dataset.interruption_task_session_ratio = 0.15
dataset.control_task_session_ratio = 0.10
dataset.backchannel_task_session_ratio = 0.15
dataset.control_followup_ratio = 0.5
dataset._estimated_runtime_motion_frames = lambda record: 400
dataset._control_motion_is_eligible = lambda *args, **kwargs: True
dataset._backchannel_motion_is_eligible = lambda *args, **kwargs: True
sessions = dataset._balance_backchannel_task_sessions(
dataset._compose_sessions(records)
)
counts = {
mode: sum(
session[0]["_session_interaction_mode"] == mode
for session in sessions
)
for mode in (
"turn-taking",
"interruption",
"emergency-stop",
"backchannel",
)
}
assert counts["turn-taking"] / len(sessions) == pytest.approx(
0.60,
abs=0.02,
)
assert counts["interruption"] / len(sessions) == pytest.approx(
0.15,
abs=0.02,
)
assert counts["emergency-stop"] / len(sessions) == pytest.approx(
0.10,
abs=0.02,
)
assert counts["backchannel"] / len(sessions) == pytest.approx(
0.15,
abs=0.02,
)
for session in sessions:
if session[0]["_session_interaction_mode"] != "backchannel":
continue
event_turns = {
int(record["_session_backchannel_turn_index"])
for record in session
}
assert len(event_turns) == 1
assert 0 <= event_turns.pop() < len(session)
def test_v23_appends_disfluency_windows_without_changing_v22_selection():
dataset = FullDuplexAudioCommandDataset.__new__(
FullDuplexAudioCommandDataset
)
dataset.split = "train"
dataset.split_seed = 42
dataset.task_session_variants = {
"turn-taking": 4,
"interruption": 1,
"emergency-stop": 1,
"safe-hold": 0,
"backchannel": 2,
"hesitation": 3,
"pause": 2,
}
dataset.interruption_task_session_ratio = 0.15
dataset.control_task_session_ratio = 0.10
dataset.backchannel_task_session_ratio = 0.15
dataset.hesitation_task_window_ratio = 0.0
dataset.pause_task_window_ratio = 0.0
dataset._cached_session_max_stream_steps = lambda session: int(
session[0]["estimated_steps"]
)
def sessions(mode: str, count: int, *, offset: int):
return tuple(
(
{
"row_id": offset + index,
"_session_interaction_mode": mode,
"estimated_steps": 10,
},
)
for index in range(count)
)
legacy_pool = (
*sessions("turn-taking", 100, offset=0),
*sessions("interruption", 80, offset=1_000),
*sessions("emergency-stop", 80, offset=2_000),
*sessions("backchannel", 80, offset=3_000),
)
v22 = dataset._balance_backchannel_task_sessions(legacy_pool)
v22_ids = {
int(session[0]["row_id"])
for session in v22
}
dataset.hesitation_task_window_ratio = 0.12
dataset.pause_task_window_ratio = 0.08
v23 = dataset._balance_backchannel_task_sessions(
(
*legacy_pool,
*sessions("hesitation", 80, offset=4_000),
*sessions("pause", 80, offset=5_000),
)
)
legacy_ids_after = {
int(session[0]["row_id"])
for session in v23
if session[0]["_session_interaction_mode"]
not in {"hesitation", "pause"}
}
by_mode_windows = {
mode: sum(
int(session[0]["estimated_steps"])
for session in v23
if session[0]["_session_interaction_mode"] == mode
)
for mode in ("hesitation", "pause")
}
total_windows = sum(
int(session[0]["estimated_steps"])
for session in v23
)
assert legacy_ids_after == v22_ids
assert by_mode_windows["hesitation"] / total_windows == pytest.approx(
0.12,
abs=0.01,
)
assert by_mode_windows["pause"] / total_windows == pytest.approx(
0.08,
abs=0.01,
)
def test_new_disfluency_rows_are_duration_stratified_and_exact_prompt_capped():
dataset = FullDuplexAudioCommandDataset.__new__(
FullDuplexAudioCommandDataset
)
dataset.split = "train"
dataset.session_min_turns = 2
dataset.new_task_max_rows_per_instruction_train = 3
dataset.new_task_max_rows_per_instruction_val = 2
dataset.disfluency_instruction_segments = {
("seed_audio/shared", "female_01_en"): {"prefix": object(), "suffix": object()}
}
dataset._session_voice_names = lambda record: ("female_01_en",)
records = [
{
"row_id": index,
"audio_dir": "seed_audio/shared",
"instruction": "walk forward",
"instruction_zh": "向前走",
"actual_frames": frames,
}
for index, frames in enumerate((10, 20, 30, 40, 50))
]
selected = dataset._select_new_task_records(records)
assert [record["actual_frames"] for record in selected] == [10, 30, 50]
assert dataset.new_task_record_selection_summary == {
"input_rows": 5,
"eligible_rows": 5,
"selected_rows": 3,
"instruction_pairs": 1,
"cap_per_instruction": 3,
"unsupported_rows": 0,
}
def _write_test_pcm16(path: Path, values: np.ndarray, rate: int = 16_000) -> None:
path.parent.mkdir(parents=True, exist_ok=True)
pcm = np.clip(np.asarray(values), -0.999, 0.999)
pcm = np.rint(pcm * 32767.0).astype("<i2")
with wave.open(str(path), "wb") as handle:
handle.setnchannels(1)
handle.setsampwidth(2)
handle.setframerate(rate)
handle.writeframes(pcm.tobytes())
def test_disfluency_speech_keeps_pause_and_repair_inside_one_causal_utterance(
tmp_path,
):
rate = 16_000
stride = 6_400
voice_name = "female_01_en"
prefix_path = tmp_path / "final-prefix.wav"
suffix_path = tmp_path / "final-suffix.wav"
provisional_path = tmp_path / "provisional-prefix.wav"
filler_path = tmp_path / "filler.wav"
repair_path = tmp_path / "repair.wav"
final_path = tmp_path / "final-complete.wav"
_write_test_pcm16(prefix_path, np.full(1_000, 0.10, np.float32))
_write_test_pcm16(suffix_path, np.full(1_200, 0.20, np.float32))
_write_test_pcm16(provisional_path, np.full(900, -0.10, np.float32))
_write_test_pcm16(filler_path, np.full(800, 0.30, np.float32))
_write_test_pcm16(repair_path, np.full(700, -0.30, np.float32))
_write_test_pcm16(final_path, np.full(1_500, 0.15, np.float32))
def segment(
category: str,
segment_id: str,
path: Path,
text: str,
*,
role: str = "",
) -> DisfluencyAudioSource:
with wave.open(str(path), "rb") as handle:
count = handle.getnframes()
return DisfluencyAudioSource(
category=category,
segment_id=segment_id,
split="train" if category != "instruction_segment" else "shared",
voice_name=voice_name,
language="en",
text=text,
waveform_path=path,
speech_samples=count,
segment_role=role,
)
final_roles = {
"prefix": segment(
"instruction_segment", "final-prefix", prefix_path, "walk", role="prefix"
),
"suffix": segment(
"instruction_segment", "final-suffix", suffix_path, "forward", role="suffix"
),
}
provisional_roles = {
"prefix": segment(
"instruction_segment",
"provisional-prefix",
provisional_path,
"pick up",
role="prefix",
),
"suffix": segment(
"instruction_segment",
"unused-provisional-suffix",
suffix_path,
"the box",
role="suffix",
),
}
filler = segment("hesitation_filler", "fill-1", filler_path, "um")
repair = segment("repair_cue", "repair-1", repair_path, "I mean")
dataset = FullDuplexAudioCommandDataset.__new__(
FullDuplexAudioCommandDataset
)
dataset.split = "train"
dataset.audio_sampling_rate = rate
dataset.motion_fps = 50
dataset.audio_sample_step_frames = 20
dataset.audio_window_seconds = 0.6
assert dataset.audio_stride_samples == stride
assert dataset.audio_window_samples == 9_600
dataset.disfluency_instruction_segments = {
("seed_audio/final", voice_name): final_roles,
("seed_audio/provisional", voice_name): provisional_roles,
}
dataset.disfluency_marker_catalog = {
voice_name: {
"hesitation_filler": (filler,),
"repair_cue": (repair,),
}
}
voice = AudioSource(
source_key=f"seed_audio/final/{voice_name}",
clips=(),
waveform_path=final_path,
waveform_count=1,
)
assignment = SourceBackgroundAssignment(
source_key=voice.source_key,
background_key="office",
listen_start_index=3,
listen_window_count=1,
background_window_count=100,
initial_idle_max_steps=7,
max_motion_steps=0,
safety_steps=2,
mixed_audio_path=tmp_path / "unused.wav",
original_audio_path=final_path,
speech_samples=1_500,
speech_scale=1.0,
)
record = {
"audio_dir": "seed_audio/final",
"instruction": "walk forward",
"instruction_zh": "向前走",
}
_, pause_assignment, pause_speech, pause_metadata = (
dataset._build_disfluency_turn_speech(
record=record,
voice=voice,
assignment=assignment,
subtype="silent-pause",
gap_steps=2,
)
)
pause_roles = [row["role"] for row in pause_metadata["components"]]
assert pause_roles == [
"final-prefix",
"background-only-pause",
"final-suffix",
]
assert pause_speech.size == 1_000 + 2 * stride + 1_200
np.testing.assert_allclose(pause_speech[1_000 : 1_000 + 2 * stride], 0.0)
assert pause_assignment.speech_samples == pause_speech.size
assert pause_assignment.listen_window_count == (
dataset._runtime_listen_window_count(pause_speech.size)
)
_, _, repair_speech, repair_metadata = dataset._build_disfluency_turn_speech(
record=record,
voice=voice,
assignment=assignment,
subtype="self-repair",
gap_steps=2,
marker_id="repair-1",
provisional_audio_dir="seed_audio/provisional",
provisional_instruction="pick up the box",
provisional_instruction_zh="拿起箱子",
)
repair_roles = [row["role"] for row in repair_metadata["components"]]
assert repair_roles == [
"provisional-prefix",
"repair-background-before",
"repair-cue",
"repair-background-after",
"complete-final-instruction",
]
assert repair_speech.size == 900 + 2 * stride + 1_500
assert repair_metadata["provisional_instruction"] == "pick up the box"
assert repair_metadata["final_instruction"] == "walk forward"
_, _, implicit_speech, implicit_metadata = (
dataset._build_disfluency_turn_speech(
record=record,
voice=voice,
assignment=assignment,
subtype="self-repair",
repair_style="implicit-restart",
gap_steps=2,
provisional_audio_dir="seed_audio/provisional",
provisional_instruction="pick up the box",
provisional_instruction_zh="拿起箱子",
)
)
implicit_roles = [
row["role"] for row in implicit_metadata["components"]
]
assert implicit_roles == [
"provisional-prefix",
"repair-implicit-background",
"complete-final-instruction",
]
assert implicit_speech.size == 900 + 2 * stride + 1_500
assert implicit_metadata["marker_id"] is None
assert implicit_metadata["marker_text"] == ""
assert implicit_metadata["repair_style"] == "implicit-restart"
assert "repair-implicit-background" in DISFLUENCY_UNCERTAINTY_ROLES
def test_hesitation_pause_splitter_never_cuts_inside_domain_words():
module_path = (
Path(__file__).resolve().parents[1]
/ "scripts/data/generate_hfd_hesitation_pause.py"
)
spec = importlib.util.spec_from_file_location(
"hfd_test_hesitation_pause_generator",
module_path,
)
assert spec is not None and spec.loader is not None
generator = importlib.util.module_from_spec(spec)
sys.modules[spec.name] = generator
spec.loader.exec_module(generator)
pytest.importorskip("jieba")
for text, protected in (
("双臂交叉站立", "交叉"),
("拄拐顺时针缓慢走路", "顺时针"),
("向后退五步", "后退"),
):
split = generator.split_chinese(text)
assert split is not None
prefix, suffix = split
assert prefix + suffix == text
boundary = len(prefix)
protected_start = text.index(protected)
assert not protected_start < boundary < protected_start + len(protected)
def test_hesitation_pause_existing_index_is_relative_and_safe(tmp_path):
module_path = (
Path(__file__).resolve().parents[1]
/ "scripts/data/generate_hfd_hesitation_pause.py"
)
spec = importlib.util.spec_from_file_location(
"hfd_test_hesitation_pause_resume_index",
module_path,
)
assert spec is not None and spec.loader is not None
generator = importlib.util.module_from_spec(spec)
sys.modules[spec.name] = generator
spec.loader.exec_module(generator)
index = tmp_path / "existing.txt"
index.write_text(
"audio/instruction_segments/000006/female_01_en_prefix.wav\n"
"audio/markers/train/hes_fill_en_01/female_01_en.wav\n",
encoding="utf-8",
)
assert generator.load_existing_output_index(index, tmp_path) == {
"audio/instruction_segments/000006/female_01_en_prefix.wav",
"audio/markers/train/hes_fill_en_01/female_01_en.wav",
}
index.write_text("../outside.wav\n", encoding="utf-8")
with pytest.raises(ValueError, match="Unsafe existing-output index"):
generator.load_existing_output_index(index, tmp_path)
def test_hesitation_pause_manifest_audit_rejects_runaway_tts(tmp_path):
module_path = (
Path(__file__).resolve().parents[1]
/ "scripts/data/generate_hfd_hesitation_pause.py"
)
spec = importlib.util.spec_from_file_location(
"hfd_test_hesitation_pause_manifest_audit",
module_path,
)
assert spec is not None and spec.loader is not None
generator = importlib.util.module_from_spec(spec)
sys.modules[spec.name] = generator
spec.loader.exec_module(generator)
valid = tmp_path / "valid.wav"
with wave.open(str(valid), "wb") as handle:
handle.setnchannels(1)
handle.setsampwidth(2)
handle.setframerate(16_000)
handle.writeframes(np.full(16_000, 1_024, dtype="<i2").tobytes())
fields = generator._manifest_audio_fields(valid)
assert fields["speech_samples_16k"] == 16_000
assert fields["duration_seconds"] == pytest.approx(1.0)
assert fields["sha256"] == hashlib.sha256(valid.read_bytes()).hexdigest()
runaway = tmp_path / "runaway.wav"
with wave.open(str(runaway), "wb") as handle:
handle.setnchannels(1)
handle.setsampwidth(2)
handle.setframerate(16_000)
handle.writeframes(np.zeros(20 * 16_000, dtype="<i2").tobytes())
with pytest.raises(RuntimeError, match="invalid PCM16/16kHz/mono"):
generator._manifest_audio_fields(runaway)
def test_v24_hesitation_inventory_has_long_fillers_and_implicit_repair():
module_path = (
Path(__file__).resolve().parents[1]
/ "scripts/data/generate_hfd_hesitation_pause.py"
)
spec = importlib.util.spec_from_file_location(
"hfd_test_hesitation_pause_v24_inventory",
module_path,
)
assert spec is not None and spec.loader is not None
generator = importlib.util.module_from_spec(spec)
sys.modules[spec.name] = generator
spec.loader.exec_module(generator)
vocalized = [
row
for row in generator.MARKERS
if row.get("delivery") == "vocalized_filler"
]
phrases = [
row
for row in generator.MARKERS
if row.get("delivery") == "phrase_filler"
]
assert vocalized
assert phrases
assert all(float(row["min_duration_seconds"]) >= 0.8 for row in vocalized)
assert all(float(row["min_duration_seconds"]) >= 0.4 for row in phrases)
repair_styles = {
str(row.get("repair_style", "explicit-cue"))
for row in generator.REVIEW_EXAMPLES
if row["subtype"] == "self-repair"
}
assert repair_styles == {"explicit-cue", "implicit-restart"}
def test_longest_point_one_percent_is_dropped_inside_each_task_mode():
dataset = FullDuplexAudioCommandDataset.__new__(
FullDuplexAudioCommandDataset
)
dataset.long_session_drop_ratio = 0.001
sessions = tuple(
(
{
"row_id": mode_index * 2_000 + index,
"_session_interaction_mode": mode,
"estimated_steps": index + 1,
},
)
for mode_index, mode in enumerate(
("turn-taking", "interruption", "emergency-stop")
)
for index in range(1_000)
)
dataset.sessions = sessions
dataset._cached_session_max_stream_steps = lambda session: int(
session[0]["estimated_steps"]
)
dataset.long_session_filter_summary = {}
retained = dataset._drop_longest_sessions_by_mode(sessions)
assert len(retained) == len(sessions) - 3
for mode in ("turn-taking", "interruption", "emergency-stop"):
values = [
session[0]["estimated_steps"]
for session in retained
if session[0]["_session_interaction_mode"] == mode
]
assert len(values) == 999
assert max(values) == 999
assert dataset.long_session_filter_summary[mode]["dropped"] == 1
def test_control_command_only_uses_motion_with_complete_causal_tail():
dataset = FullDuplexAudioCommandDataset.__new__(
FullDuplexAudioCommandDataset
)
dataset.action_exec_horizon = 20
dataset.motion_fps = 50
dataset.turn_transition_frames = 50
dataset.interruption_min_start_seconds = 2.0
dataset.interruption_max_start_seconds = 6.0
dataset.control_command_catalog = {
"emergency-stop": {
"female_01_zh": (
SimpleNamespace(speech_samples=16_000),
)
}
}
dataset._runtime_listen_window_count = lambda samples: 3
dataset._estimated_runtime_motion_frames = lambda record: int(
record["frames"]
)
# 100-frame onset + 60-frame complete command + 20-frame real tail.
assert not dataset._control_motion_is_eligible(
{"frames": 130},
intent="emergency-stop",
voice_name="female_01_zh",
)
assert dataset._control_motion_is_eligible(
{"frames": 131},
intent="emergency-stop",
voice_name="female_01_zh",
)
start, diagnostics = dataset._sample_control_speech_start(
np.zeros((180, 36), dtype=np.float32),
reaction_frames=60,
)
assert start == 100
assert diagnostics["minimum_remaining_frames"] == 20
def test_control_eligibility_respects_debug_motion_truncation():
dataset = FullDuplexAudioCommandDataset.__new__(
FullDuplexAudioCommandDataset
)
dataset.action_exec_horizon = 20
dataset.prefix_frames = 10
dataset.motion_fps = 50
dataset.turn_transition_frames = 50
dataset.dynamic_task_entry_blend = True
dataset.control_min_start_seconds = 1.2
dataset.control_min_remaining_seconds = 2.0
dataset.control_command_catalog = {
"emergency-stop": {
"female_01_zh": (SimpleNamespace(speech_samples=16_000),)
}
}
dataset._runtime_listen_window_count = lambda samples: 3
dataset._estimated_runtime_motion_frames = lambda record: 1_000
# Runtime truncation to 200 frames cannot contain a 60-frame onset,
# 60-frame utterance and 100-frame post-command motion tail.
dataset.max_motion_steps = 10
assert not dataset._control_motion_is_eligible(
{},
intent="emergency-stop",
voice_name="female_01_zh",
)
dataset.max_motion_steps = 12
assert dataset._control_motion_is_eligible(
{},
intent="emergency-stop",
voice_name="female_01_zh",
)
def test_activity_aware_stop_prefers_motion_over_stationary_tail():
dataset = FullDuplexAudioCommandDataset.__new__(
FullDuplexAudioCommandDataset
)
dataset.action_exec_horizon = 20
dataset.motion_fps = 50
dataset.turn_transition_frames = 50
dataset.interruption_min_start_seconds = 2.0
dataset.interruption_max_start_seconds = 6.0
dataset.control_min_start_seconds = 1.2
dataset.control_max_start_seconds = 4.0
dataset.control_min_remaining_seconds = 2.0
dataset.control_activity_aware_timing = True
sequence = np.zeros((400, 36), dtype=np.float32)
sequence[:, 32] = 1.0
# Real task motion is active from 1.0 s through 3.6 s, followed by a long
# stationary tail. A 1.2 s command must be placed so both its onset and
# its complete-response boundary still observe movement.
sequence[50:181, 0] = np.arange(131, dtype=np.float32) * 0.02
sequence[181:, 0] = sequence[180, 0]
random.seed(7)
start, diagnostics = dataset._sample_control_speech_start(
sequence,
reaction_frames=60,
)
assert 60 <= start <= 120
assert start + 60 + 100 <= len(sequence)
assert diagnostics["active_candidate_count"] > 0
assert diagnostics["activity_fallback"] is False
def test_safe_hold_waits_for_supported_low_velocity_pose():
dataset = FullDuplexAudioCommandDataset.__new__(
FullDuplexAudioCommandDataset
)
dataset.action_exec_horizon = 20
dataset.safe_hold_stability_window_frames = 20
sequence = np.zeros((260, 36), dtype=np.float32)
sequence[:, 32] = 1.0
# The robot keeps moving through frame 159, then reaches and retains an
# upright, supported pose. A safe hold must not freeze during that motion.
sequence[:160, 0] = np.arange(160, dtype=np.float32) * 0.20
sequence[160:, 0] = sequence[159, 0]
boundary = dataset._safe_hold_boundary(sequence, earliest_frame=100)
assert boundary >= 180
assert boundary < len(sequence)
def test_emergency_recovery_prefers_nearby_two_support_phase():
dataset = FullDuplexAudioCommandDataset.__new__(
FullDuplexAudioCommandDataset
)
dataset.safe_stationary_anchors = True
dataset.action_exec_horizon = 20
dataset.turn_transition_frames = 50
dataset._stationary_joint_scale = np.ones(29, dtype=np.float32)
neutral = np.zeros(36, dtype=np.float32)
neutral[31] = 0.82
neutral[32] = 1.0
dataset._stationary_anchor_poses = neutral[None, :]
sequence = np.repeat(neutral[None, :], 100, axis=0)
# At the immediate stop boundary both leg chains are moving; shortly
# afterwards they settle close to a two-support neutral pose. Recovery
# searches every frame so it cannot miss a brief safe-support phase.
sequence[:25, 0:12] = np.linspace(
0.0, 1.0, 25, dtype=np.float32
)[:, None]
sequence[25:40, 0:12] = np.linspace(
1.0, 0.0, 15, dtype=np.float32
)[:, None]
boundary, anchor = dataset._nearest_safe_recovery_boundary(
sequence,
earliest_frame=20,
)
assert 39 <= boundary <= 41
np.testing.assert_allclose(anchor[:29], neutral[:29], atol=1e-6)
def test_state_aware_serial_control_consumes_only_causal_physical_context():
torch.manual_seed(7)
model = FullDuplexPsi0Model.__new__(FullDuplexPsi0Model)
nn.Module.__init__(model)
model.state_aware_serial_control = True
model.action_dim = 3
memory_dim = 3
model.control_state_encoder = nn.Sequential(
nn.LayerNorm(3), nn.Linear(3, memory_dim), nn.GELU()
)
model.control_rtc_encoder = nn.Sequential(
nn.LayerNorm(3), nn.Linear(3, memory_dim), nn.GELU()
)
model.control_progress_encoder = nn.Sequential(
nn.Linear(5, memory_dim), nn.GELU()
)
model.previous_control_embedding = nn.Embedding(4, memory_dim)
model.control_head = nn.Sequential(
nn.LayerNorm(4 + 4 * memory_dim + 1),
nn.Linear(4 + 4 * memory_dim + 1, memory_dim),
nn.GELU(),
nn.Linear(memory_dim, 3),
)
semantic = torch.randn(1, 4, requires_grad=True)
state = torch.tensor([[[0.1, -0.2, 0.3]]], requires_grad=True)
progress = torch.tensor(
[[0.25, 0.75, 0.25, 1.0, 0.0]], requires_grad=True
)
rtc = torch.randn(1, 10, 3, requires_grad=True)
rtc_mask = torch.ones(1, 10, dtype=torch.bool)
logits = model._predict_control_logits(
semantic,
states=state,
control_progress=progress,
previous_control_ids=torch.tensor([MOTION_TOKEN_ID]),
control_rtc_prefix=rtc,
control_rtc_mask=rtc_mask,
)
logits.square().sum().backward()
assert logits.shape == (1, 3)
for value in (semantic.grad, state.grad, progress.grad, rtc.grad):
assert value is not None
assert torch.isfinite(value).all()
assert float(value.abs().sum()) > 0.0
# A masked source-start RTC contains no causal plan, so its numeric filler
# cannot change the decision.
source_start_a = model._predict_control_logits(
semantic.detach(),
states=state.detach(),
control_progress=progress.detach(),
previous_control_ids=torch.tensor([3]),
control_rtc_prefix=torch.zeros_like(rtc),
control_rtc_mask=torch.zeros_like(rtc_mask),
)
source_start_b = model._predict_control_logits(
semantic.detach(),
states=state.detach(),
control_progress=progress.detach(),
previous_control_ids=torch.tensor([3]),
control_rtc_prefix=torch.full_like(rtc, 999.0),
control_rtc_mask=torch.zeros_like(rtc_mask),
)
torch.testing.assert_close(source_start_a, source_start_b)
with pytest.raises(ValueError, match="missing causal inputs"):
model._predict_control_logits(
semantic.detach(),
states=state.detach(),
control_progress=None,
previous_control_ids=torch.tensor([3]),
control_rtc_prefix=torch.zeros_like(rtc),
control_rtc_mask=torch.zeros_like(rtc_mask),
)
def test_semantic_only_serial_control_ignores_physical_context():
torch.manual_seed(11)
model = FullDuplexPsi0Model.__new__(FullDuplexPsi0Model)
nn.Module.__init__(model)
model.state_aware_serial_control = False
model.control_head = nn.Sequential(
nn.LayerNorm(4),
nn.Linear(4, 3),
)
semantic = torch.randn(2, 4, requires_grad=True)
logits_a = model._predict_control_logits(
semantic,
states=torch.randn(2, 1, 39),
control_progress=torch.randn(2, 5),
previous_control_ids=torch.tensor([0, 2]),
control_rtc_prefix=torch.randn(2, 10, 39),
control_rtc_mask=torch.ones(2, 10, dtype=torch.bool),
)
logits_b = model._predict_control_logits(
semantic,
states=None,
control_progress=None,
previous_control_ids=None,
control_rtc_prefix=None,
control_rtc_mask=None,
)
torch.testing.assert_close(logits_a, logits_b)
logits_b.sum().backward()
assert semantic.grad is not None
assert float(semantic.grad.abs().sum()) > 0.0
def test_interruption_source_continues_old_motion_then_transitions_causally(
monkeypatch,
):
dataset = FullDuplexAudioCommandDataset.__new__(
FullDuplexAudioCommandDataset
)
records = tuple(
{
"row_id": row_id,
"filename": f"motion-{row_id}",
"instruction": f"instruction-{row_id}",
"audio_dir": f"audio-{row_id}",
"motion_dir": Path(f"/unused/motion-{row_id}"),
"declared_frames": 350,
"_session_interaction_mode": "interruption",
}
for row_id in (3, 7)
)
dataset.records = list(records)
dataset.sessions = (records,)
dataset.multi_turn_sessions = True
dataset.interruption_sessions = True
dataset.interruption_ratio = 0.5
dataset.interruption_min_start_seconds = 2.0
dataset.interruption_max_start_seconds = 6.0
dataset.interruption_min_remaining_seconds = 1.5
dataset.session_min_turns = 2
dataset.session_max_turns = 4
dataset.fixed_voice_per_record = True
dataset.action_chunk_size = 30
dataset.action_exec_horizon = 20
dataset.prefix_frames = 10
dataset.audio_sample_step_frames = 20
dataset.action_sample_step_frames = 1
dataset.audio_sampling_rate = 16_000
dataset.motion_fps = 50
dataset.audio_window_seconds = 0.6
dataset.initial_idle_max_steps = 7
dataset.stationary_blend_frames = 20
dataset.turn_transition_frames = 50
dataset.causal_listen_timing = True
dataset.max_speech_onset_steps = 3
dataset.listen_reaction_steps = 1
dataset.max_motion_steps = 0
dataset.max_stream_steps = 1923
voices = tuple(
AudioSource(
source_key=f"audio-{row_id}/voice",
clips=(),
waveform_path=Path(f"/unused/voice-{row_id}.wav"),
waveform_count=7,
)
for row_id in (3, 7)
)
dataset.audio_catalog = {
"audio-3": (voices[0],),
"audio-7": (voices[1],),
}
dataset.background_assignments = {
voice.source_key: SourceBackgroundAssignment(
source_key=voice.source_key,
background_key="office",
listen_start_index=20,
listen_window_count=7,
background_window_count=400,
initial_idle_max_steps=7,
max_motion_steps=30,
safety_steps=5,
original_audio_path=Path(f"/unused/{voice.source_key}.wav"),
speech_samples=39_680,
speech_scale=0.5,
)
for voice in voices
}
motions = {}
for record_index, record in enumerate(records):
motion = np.zeros((350, 36), dtype=np.float32)
motion[:, 0] = np.linspace(0.0, 0.4 + record_index, 350)
motion[:, 29] = np.linspace(0.0, 4.0 + record_index, 350)
motion[:, 31] = 0.8
motion[:, 32] = 1.0
motions[str(record["motion_dir"])] = motion
monkeypatch.setattr(dataset, "_load_motion", lambda path: motions[str(path)])
monkeypatch.setattr(
dataset,
"_sample_stationary_anchor",
lambda pose: np.asarray(pose, dtype=np.float32).copy(),
)
monkeypatch.setattr(
dataset,
"_sample_stationary_sequence",
lambda phase, frame_count, anchor_pose, **kwargs: (
np.repeat(
np.asarray(anchor_pose, dtype=np.float32)[None, :],
frame_count,
axis=0,
),
f"{phase}-fixture",
),
)
clean_speech = np.zeros(39_680, dtype=np.float32)
clean_speech[17_120:] = 0.1
monkeypatch.setattr(
"psi.data.seed.full_duplex_audio_command_dataset._read_resampled_speech_float",
lambda path, expected_rate: clean_speech.copy(),
)
monkeypatch.setattr(
dataset,
"_build_session_audio_windows",
lambda *, turn_audio, semantic_window_count: (
tuple(
np.full(9600, float(index), dtype=np.float32)
for index in range(semantic_window_count)
),
"office",
100,
),
)
monkeypatch.setattr(
dataset,
"_sample_interruption_prefix_frames",
lambda task_sequence, next_reaction_frames: 300,
)
random.seed(20)
source = dataset[0]
assert source["interaction_mode"] == "interruption"
assert len(source["interruption_start_frames"]) == 1
speech_start = source["turn_speech_start_frames"][1]
listen_start = source["turn_listen_start_frames"][1]
assert speech_start == source["interruption_start_frames"][0]
first_motion_start = source["turn_listen_start_frames"][0] + 7 * 20
assert 100 <= speech_start - first_motion_start <= 300
assert listen_start - speech_start == 100
assert all(
source["steps"][frame]["control_label"] == MOTION_TOKEN_ID
for frame in range(speech_start, listen_start)
)
assert source["steps"][listen_start]["control_label"] == LISTEN_TOKEN_ID
assert source["steps"][listen_start]["control_transition"] is True
np.testing.assert_allclose(
source["steps"][listen_start - 1]["actions"][0],
source["steps"][listen_start]["states"][0],
atol=1e-6,
)
original_before = source["interruption_original_frames"][0]
assert source["interruption_remaining_frames"][0] >= 75
assert source["interruption_terminal_hold_frames"] == [1]
expected_progress = (original_before + 2) / 349.0
assert source["steps"][speech_start]["future_progress"][0, 0] == pytest.approx(
expected_progress
)
assert len(source["recovery_transition_frames"]) == 1
assert all(
step["interaction_mode"] == "interruption"
for step in source["steps"]
)
def test_multiturn_source_keeps_one_cache_id_and_aligned_boundaries(monkeypatch):
dataset = FullDuplexAudioCommandDataset.__new__(
FullDuplexAudioCommandDataset
)
records = tuple(
{
"row_id": row_id,
"filename": f"motion-{row_id}",
"instruction": f"instruction-{row_id}",
"audio_dir": f"audio-{row_id}",
"motion_dir": Path(f"/unused/motion-{row_id}"),
"declared_frames": 96,
}
for row_id in (3, 7)
)
dataset.records = list(records)
dataset.sessions = (records,)
dataset.multi_turn_sessions = True
dataset.session_min_turns = 2
dataset.session_max_turns = 4
dataset.fixed_voice_per_record = True
dataset.action_chunk_size = 30
dataset.action_exec_horizon = 20
dataset.prefix_frames = 10
dataset.audio_sample_step_frames = 20
dataset.action_sample_step_frames = 1
dataset.audio_sampling_rate = 16_000
dataset.motion_fps = 50
dataset.audio_window_seconds = 0.6
dataset.initial_idle_max_steps = 7
dataset.stationary_blend_frames = 20
dataset.turn_transition_frames = 50
dataset.causal_listen_timing = True
dataset.max_speech_onset_steps = 3
dataset.listen_reaction_steps = 1
dataset.max_motion_steps = 0
dataset.max_stream_steps = 1923
voices = tuple(
AudioSource(
source_key=f"audio-{row_id}/voice",
clips=(),
waveform_path=Path(f"/unused/voice-{row_id}.wav"),
waveform_count=7,
)
for row_id in (3, 7)
)
dataset.audio_catalog = {
"audio-3": (voices[0],),
"audio-7": (voices[1],),
}
dataset.background_assignments = {
voice.source_key: SourceBackgroundAssignment(
source_key=voice.source_key,
background_key="office",
listen_start_index=20,
listen_window_count=7,
background_window_count=400,
initial_idle_max_steps=5,
max_motion_steps=20,
safety_steps=5,
original_audio_path=Path(f"/unused/voice-{voice.source_key}.wav"),
speech_samples=39_680,
speech_scale=0.5,
)
for voice in voices
}
motions = {}
for record_index, record in enumerate(records):
motion = np.zeros((40, 36), dtype=np.float32)
motion[:, 0] = np.linspace(0.0, 0.2 + record_index, 40)
motion[:, 29] = np.linspace(0.0, 0.5, 40)
motion[:, 31] = 0.8
motion[:, 32] = 1.0
motions[str(record["motion_dir"])] = motion
monkeypatch.setattr(
dataset,
"_load_motion",
lambda path: motions[str(path)],
)
monkeypatch.setattr(
dataset,
"_sample_stationary_anchor",
lambda pose: np.asarray(pose, dtype=np.float32).copy(),
)
monkeypatch.setattr(
dataset,
"_sample_stationary_sequence",
lambda phase, frame_count, anchor_pose, **kwargs: (
np.repeat(
np.asarray(anchor_pose, dtype=np.float32)[None, :],
frame_count,
axis=0,
),
f"{phase}-fixture",
),
)
# 1.07 s of leading silence needs four Qwen boundaries, deliberately
# exceeding the historical expected budget of three. It is valid speech
# and must extend the idle reaction phase instead of killing a worker.
clean_speech = np.zeros(39_680, dtype=np.float32)
clean_speech[17_120:] = 0.1
monkeypatch.setattr(
"psi.data.seed.full_duplex_audio_command_dataset._read_resampled_speech_float",
lambda path, expected_rate: clean_speech.copy(),
)
def build_audio(*, turn_audio, semantic_window_count):
assert len(turn_audio) == 2
windows = tuple(
np.full(9600, float(index), dtype=np.float32)
for index in range(semantic_window_count)
)
return windows, "office", 100
monkeypatch.setattr(dataset, "_build_session_audio_windows", build_audio)
random.seed(19)
source = dataset[0]
assert source["turn_count"] == 2
assert source["task_transition_frames"] == [50, 50]
assert all(50 <= count <= 69 for count in source["recovery_transition_frames"])
assert all(frame % 20 == 0 for frame in source["turn_listen_start_frames"])
assert all(
listen - speech == 100
for speech, listen in zip(
source["turn_speech_start_frames"],
source["turn_listen_start_frames"],
)
)
assert source["turn_listen_reaction_frames"] == [100, 100]
assert source["listen_transition_frames"] == 50
assert {step["source_id"] for step in source["steps"]} == {
source["source_id"]
}
update_steps = [
step for step in source["steps"] if step["update_semantic_cache"]
]
assert [step["background_window_index"] for step in update_steps] == list(
range(100, 100 + len(update_steps))
)
transitions = [
step
for step in update_steps[1:]
if step["control_label"]
!= update_steps[step["semantic_window_index"] - 1]["control_label"]
]
assert transitions
assert all(step["control_transition"] for step in transitions)
assert {step["turn_index"] for step in source["steps"]} == {0, 1}
duration_targets = [
step
for step in source["steps"]
if step["motion_duration_target_mask"]
]
assert len(duration_targets) == source["turn_count"] == 2
assert all(step["update_semantic_cache"] for step in duration_targets)
assert all(step["control_label"] == MOTION_TOKEN_ID for step in duration_targets)
assert [step["motion_duration_target_seconds"] for step in duration_targets] == [
0.8,
0.8,
]
text_targets = [
step
for step in source["steps"]
if step["latest_instruction_target_mask"]
]
assert [step["latest_instruction_text"] for step in text_targets] == [
"instruction-3",
"instruction-7",
]
assert all(step["update_semantic_cache"] for step in text_targets)
memory_targets = [
step
for step in source["steps"]
if step["instruction_memory_target_mask"]
]
assert len(memory_targets) == 1
assert memory_targets[0]["turn_index"] == 1
assert memory_targets[0]["instruction_memory_slot"] == "previous"
assert memory_targets[0]["instruction_memory_text"] == "instruction-3"
assert memory_targets[0]["update_semantic_cache"] is True
def test_multiturn_audio_uses_one_pcm_continuum(monkeypatch):
dataset = FullDuplexAudioCommandDataset.__new__(
FullDuplexAudioCommandDataset
)
dataset.audio_sampling_rate = 16_000
dataset.audio_window_seconds = 0.6
dataset.audio_sample_step_frames = 20
dataset.motion_fps = 50
dataset.background_catalog = {
"office": BackgroundWaveformSource(
source_key="office",
waveform_path=Path("/unused/office.wav"),
window_count=20,
)
}
flat_background = np.linspace(-0.02, 0.02, 60_000, dtype=np.float32)
monkeypatch.setattr(
"psi.data.seed.full_duplex_audio_command_dataset._read_pcm16_wave_segment",
lambda path, start_sample, sample_count, expected_rate: (
flat_background[:sample_count].copy()
),
)
speech = {
"/unused/one.wav": np.full(3_200, 0.1, dtype=np.float32),
"/unused/two.wav": np.full(4_800, -0.1, dtype=np.float32),
}
monkeypatch.setattr(
"psi.data.seed.full_duplex_audio_command_dataset._read_resampled_speech_float",
lambda path, expected_rate: speech[str(path)].copy(),
)
monkeypatch.setattr("random.randrange", lambda stop: 0)
turn_audio = []
for source_key, path, samples, frame in (
("one", "/unused/one.wav", 3_200, 20),
("two", "/unused/two.wav", 4_800, 100),
):
turn_audio.append(
{
"listen_start_frame": frame,
"assignment": SourceBackgroundAssignment(
source_key=source_key,
background_key="old-single-turn-background",
listen_start_index=5,
listen_window_count=2,
background_window_count=20,
initial_idle_max_steps=5,
max_motion_steps=5,
safety_steps=2,
original_audio_path=Path(path),
speech_samples=samples,
speech_scale=0.5,
),
}
)
windows, key, start = dataset._build_session_audio_windows(
turn_audio=turn_audio,
semantic_window_count=8,
)
assert key == "office"
assert start == 0
assert len(windows) == 8
overlap = dataset.audio_window_samples - dataset.audio_stride_samples
for left, right in zip(windows, windows[1:]):
np.testing.assert_array_equal(left[-overlap:], right[:overlap])
assert not np.array_equal(windows[1], flat_background[6_400:16_000])
def test_turn_transition_is_slow_smooth_and_endpoint_exact():
start = np.zeros(36, dtype=np.float32)
end = np.zeros(36, dtype=np.float32)
start[31] = 0.80
end[:29] = 0.5
end[29:32] = np.asarray([1.0, -0.5, 0.86], dtype=np.float32)
start[32] = 1.0
end[32] = 1.0
transition = FullDuplexAudioCommandDataset._build_pose_transition(
start,
end,
50,
)
np.testing.assert_allclose(transition[0], start, atol=1e-7)
np.testing.assert_allclose(transition[-1], end, atol=1e-7)
# Quintic smootherstep approaches both endpoints much more gently than a
# direct 20-frame interpolation; its first increment is below 0.01% of
# the complete joint displacement.
assert float(np.max(np.abs(transition[1, :29] - start[:29]))) < 5e-5
assert float(np.max(np.abs(end[:29] - transition[-2, :29]))) < 5e-5
np.testing.assert_allclose(
np.linalg.norm(transition[:, 32:36], axis=-1),
1.0,
atol=1e-6,
)
def test_multiturn_task_entry_uses_relaxed_quintic_transition():
dataset = FullDuplexAudioCommandDataset.__new__(
FullDuplexAudioCommandDataset
)
dataset.multi_turn_sessions = True
dataset.stationary_blend_frames = 20
anchor = np.zeros(36, dtype=np.float32)
anchor[31] = 0.82
anchor[32] = 1.0
task = np.repeat(anchor[None, :], 80, axis=0)
task[:, 0] = 0.6
sequence, transition_frames = dataset._build_task_sequence(
task,
stationary_anchor=anchor,
transition_frames=50,
)
assert transition_frames == 50
np.testing.assert_allclose(sequence[0], anchor, atol=1e-7)
np.testing.assert_allclose(sequence[49], task[0], atol=1e-7)
assert abs(float(sequence[1, 0] - anchor[0])) < 5e-5
assert abs(float(task[0, 0] - sequence[48, 0])) < 5e-5
def test_source_sequence_uses_30_20_10_rtc_and_phase_labels(monkeypatch):
dataset = FullDuplexAudioCommandDataset.__new__(FullDuplexAudioCommandDataset)
dataset.records = [
{
"row_id": 7,
"filename": "test_motion",
"instruction": "test",
"audio_dir": "seed_audio/000007",
"motion_dir": Path("/unused/test_motion"),
"declared_frames": 45,
}
]
slices = tuple(
HiddenSlice(frame_id, Path(f"/unused/{frame_id}"), 0, 1)
for frame_id in (0, 20, 40)
)
dataset.audio_catalog = {
"seed_audio/000007": (
AudioSource("seed_audio/000007/voice", slices),
)
}
dataset.white_noise_hidden = tuple(
torch.full((4, 8), float(value)) for value in (9, 10, 11)
)
dataset.fixed_voice_per_record = False
dataset.action_chunk_size = 30
dataset.action_exec_horizon = 20
dataset.prefix_frames = 10
dataset.max_motion_steps = 0
dataset.initial_idle_max_steps = 2
dataset.stationary_blend_frames = 20
motion = np.arange(45 * 36, dtype=np.float32).reshape(45, 36)
monkeypatch.setattr(dataset, "_load_motion", lambda path: motion)
monkeypatch.setattr(
FullDuplexAudioCommandDataset,
"_load_slice",
staticmethod(lambda item: torch.full((4, 8), float(item.frame_id))),
)
monkeypatch.setattr(
dataset,
"_sample_stationary_sequence",
lambda phase, frame_count, anchor_pose: (
np.repeat(
np.asarray(anchor_pose, dtype=np.float32)[None, :],
frame_count,
axis=0,
),
f"test-{phase}",
),
)
monkeypatch.setattr(
dataset,
"_sample_stationary_anchor",
lambda task_start_pose: np.asarray(task_start_pose, dtype=np.float32),
)
monkeypatch.setattr(
dataset,
"_build_task_sequence",
lambda task_motion, stationary_anchor: (
np.asarray(task_motion, dtype=np.float32),
1,
),
)
monkeypatch.setattr("random.randint", lambda low, high: 2)
monkeypatch.setattr("random.randrange", lambda stop: 1)
source = dataset[0]
assert source["white_noise_start_index"] == 1
assert source["idle_steps"] == 2
assert source["listen_steps"] == 3
assert source["motion_steps"] == 2
assert source["action_idle_steps"] == 40
assert source["action_listen_steps"] == 60
assert source["action_motion_steps"] == 34
assert [step["phase"] for step in source["steps"]].count("idle") == 40
assert [step["phase"] for step in source["steps"]].count("listen") == 60
assert [step["phase"] for step in source["steps"]].count("motion") == 34
assert [
index
for index, step in enumerate(source["steps"])
if step["update_semantic_cache"]
] == [0, 20, 40, 60, 80, 100, 120]
assert all(
step["control_label"] == IDLE_TOKEN_ID
for step in source["steps"][:40]
)
assert all(
step["control_label"] == LISTEN_TOKEN_ID
for step in source["steps"][40:100]
)
assert all(
step["control_label"] == MOTION_TOKEN_ID
for step in source["steps"][100:]
)
# A 30-frame target crossing a state boundary now keeps one control label
# per future frame. It is no longer falsely described as all-idle or
# all-listen by the scalar current-window token.
np.testing.assert_array_equal(
source["steps"][20]["action_control_labels"],
np.asarray(
[IDLE_TOKEN_ID] * 19 + [LISTEN_TOKEN_ID] * 11,
dtype=np.int64,
),
)
np.testing.assert_array_equal(
source["steps"][80]["action_control_labels"],
np.asarray(
[LISTEN_TOKEN_ID] * 19 + [MOTION_TOKEN_ID] * 11,
dtype=np.int64,
),
)
assert source["steps"][40]["control_transition"] is True
assert source["steps"][100]["control_transition"] is True
assert source["steps"][41]["control_transition"] is False
assert [
float(source["steps"][index]["vlm_hidden_states"][0, 0])
for index in (0, 20, 40, 60, 80, 100, 120)
] == [10.0, 11.0, 0.0, 20.0, 40.0, 9.0, 10.0]
assert "vlm_hidden_states" not in source["steps"][1]
first_motion = source["steps"][100]
np.testing.assert_array_equal(first_motion["actions"], motion[1:31])
np.testing.assert_array_equal(first_motion["states"][0], motion[0])
second_motion = source["steps"][101]
np.testing.assert_array_equal(second_motion["actions"], motion[2:32])
np.testing.assert_array_equal(second_motion["states"][0], motion[1])
np.testing.assert_array_equal(
second_motion["action_reference_states"][0],
motion[1],
)
final_motion = source["steps"][-1]
np.testing.assert_array_equal(final_motion["actions"][:11], motion[34:45])
np.testing.assert_array_equal(
final_motion["actions"][11:],
np.repeat(motion[44:45], 19, axis=0),
)
assert final_motion["action_mask"].sum() == 11
def test_textop_rtc_first_delta_uses_current_state_before_future_horizon():
actions = np.zeros((30, 36), dtype=np.float32)
actions[:, 29] = np.arange(10, 40, dtype=np.float32)
actions[:, 32] = 1.0
observation_state = np.zeros((1, 36), dtype=np.float32)
observation_state[0, 29] = 29.0
observation_state[0, 32] = 1.0
action_reference_state = np.zeros((1, 36), dtype=np.float32)
action_reference_state[0, 29] = 9.0
action_reference_state[0, 32] = 1.0
transformed = SeedPosttrainRepackTransform(
action_chunk_size=30,
rtc_prefix_frames=10,
textop_actions=True,
)(
{
"actions": actions,
"action_mask": np.ones(30, dtype=np.float32),
"states": observation_state,
"action_reference_states": action_reference_state,
}
)
assert transformed["actions"].shape == (30, 39)
assert transformed["actions"][0, 29] == pytest.approx(1.0)
def test_dense_action_only_steps_skip_qwen_audio_transform():
class Stage:
def __init__(self, name, calls):
self.name = name
self.calls = calls
def __call__(self, data, **kwargs):
del kwargs
self.calls.append(self.name)
output = dict(data)
if self.name == "field":
output["actions_mask"] = output.pop("action_mask")
if self.name == "model":
assert "audio_waveform" in output
output["stream_input_ids"] = torch.ones(3, dtype=torch.long)
return output
class Transform:
def __init__(self):
self.calls = []
self.repack = Stage("repack", self.calls)
self.field = Stage("field", self.calls)
self.model = Stage("model", self.calls)
def __call__(self, data, **kwargs):
data = self.repack(data, **kwargs)
data = self.field(data, **kwargs)
return self.model(data, **kwargs)
base = {
"source_id": "source-a",
"source_audio_key": "audio-a",
"phase": "listen",
"control_label": LISTEN_TOKEN_ID,
"control_token": "</listen>",
"frame_id": 0,
"semantic_frame_id": 0,
"semantic_window_index": 0,
"actions": np.zeros((30, 39), dtype=np.float32),
"action_mask": np.ones((30, 39), dtype=np.float32),
"states": np.zeros((1, 39), dtype=np.float32),
"progress": np.zeros(5, dtype=np.float32),
"future_progress": np.zeros((30, 5), dtype=np.float32),
"motion_duration_target_seconds": 4.25,
"motion_duration_target_mask": False,
}
update = {
**base,
"step_index": 0,
"action_substep": 0,
"update_semantic_cache": True,
"audio_waveform": np.zeros(9_600, dtype=np.float32),
"motion_duration_target_mask": True,
}
reuse = {
**base,
"step_index": 1,
"action_substep": 1,
"update_semantic_cache": False,
}
class RawDataset:
def __len__(self):
return 1
def __getitem__(self, index):
assert index == 0
return {"source_id": "source-a", "steps": [update, reuse]}
def estimated_stream_steps(self, index):
assert index == 0
return 2
transform = Transform()
dataset = FullDuplexTransformedDataset(RawDataset(), transform)
source = dataset[0]
assert transform.calls == ["repack", "field", "model", "repack", "field"]
assert "stream_input_ids" in source["steps"][0]
assert "stream_input_ids" not in source["steps"][1]
assert source["steps"][0]["motion_duration_target_seconds"] == 4.25
assert source["steps"][0]["motion_duration_target_mask"] is True
assert source["steps"][1]["motion_duration_target_mask"] is False
def test_real_seed_repack_preserves_motion_duration_supervision():
class IdentityStage:
def __call__(self, data, **kwargs):
del kwargs
return data
class Transform:
def __init__(self):
self.repack = SeedPosttrainRepackTransform(
action_chunk_size=30,
rtc_prefix_frames=10,
textop_actions=False,
pad_action_dim=39,
pad_state_dim=39,
)
self.field = IdentityStage()
self.model = IdentityStage()
def __call__(self, data, **kwargs):
data = self.repack(data, **kwargs)
data = self.field(data, **kwargs)
return self.model(data, **kwargs)
raw_step = {
"source_id": "source-duration",
"source_audio_key": "audio-duration",
"phase": "motion",
"control_label": MOTION_TOKEN_ID,
"control_token": "</motion>",
"control_transition": True,
"action_control_labels": np.full(30, MOTION_TOKEN_ID),
"update_semantic_cache": True,
"step_index": 0,
"frame_id": 0,
"semantic_frame_id": 0,
"semantic_window_index": 0,
"action_substep": 0,
"turn_index": 0,
"actions": np.zeros((30, 39), dtype=np.float32),
"action_mask": np.ones((30, 39), dtype=np.float32),
"states": np.zeros((1, 39), dtype=np.float32),
"progress": np.zeros(5, dtype=np.float32),
"future_progress": np.zeros((30, 5), dtype=np.float32),
"instruction": "walk",
"motion_duration_target_seconds": 6.5,
"motion_duration_target_mask": True,
"latest_instruction_text": "walk",
"latest_instruction_target_mask": True,
}
class RawDataset:
def __len__(self):
return 1
def __getitem__(self, index):
assert index == 0
return {"source_id": "source-duration", "steps": [raw_step]}
def estimated_stream_steps(self, index):
assert index == 0
return 1
source = FullDuplexTransformedDataset(RawDataset(), Transform())[0]
assert source["steps"][0]["motion_duration_target_seconds"] == 6.5
assert source["steps"][0]["motion_duration_target_mask"] is True
assert source["steps"][0]["latest_instruction_text"] == "walk"
assert source["steps"][0]["latest_instruction_target_mask"] is True
assert source["steps"][0]["instruction_memory_text"] == ""
assert source["steps"][0]["instruction_memory_slot"] == ""
assert source["steps"][0]["instruction_memory_target_mask"] is False
def test_continuous_background_indices_are_adjacent_without_wrap(monkeypatch):
dataset = FullDuplexAudioCommandDataset.__new__(
FullDuplexAudioCommandDataset
)
dataset.records = [
{
"row_id": 7,
"filename": "test_motion",
"instruction": "test",
"audio_dir": "seed_audio/000007",
"motion_dir": Path("/unused/test_motion"),
"declared_frames": 45,
}
]
slices = tuple(
HiddenSlice(frame_id, Path(f"/unused/{frame_id}"), 0, 1)
for frame_id in (0, 20, 40)
)
voice = AudioSource("seed_audio/000007/voice", slices)
dataset.audio_catalog = {"seed_audio/000007": (voice,)}
dataset.continuous_background = True
dataset.background_assignments = {
voice.source_key: SourceBackgroundAssignment(
source_key=voice.source_key,
background_key="sessions/office",
listen_start_index=100,
listen_window_count=3,
background_window_count=300,
initial_idle_max_steps=2,
max_motion_steps=3,
safety_steps=10,
)
}
dataset.white_noise_hidden = ()
dataset.fixed_voice_per_record = True
dataset.action_chunk_size = 30
dataset.action_exec_horizon = 20
dataset.prefix_frames = 10
dataset.max_motion_steps = 0
dataset.initial_idle_max_steps = 2
dataset.stationary_blend_frames = 20
motion = np.arange(45 * 36, dtype=np.float32).reshape(45, 36)
monkeypatch.setattr(dataset, "_load_motion", lambda path: motion)
monkeypatch.setattr(
FullDuplexAudioCommandDataset,
"_load_slice",
staticmethod(lambda item: torch.full((4, 8), float(item.frame_id))),
)
loaded_background_blocks = []
def load_background_block(source_key, start, count):
assert source_key == "sessions/office"
loaded_background_blocks.append((start, count))
return tuple(
torch.full((4, 8), float(index))
for index in range(start, start + count)
)
monkeypatch.setattr(
dataset,
"_load_background_window_block",
load_background_block,
)
monkeypatch.setattr(
dataset,
"_sample_stationary_sequence",
lambda phase, frame_count, anchor_pose: (
np.repeat(
np.asarray(anchor_pose, dtype=np.float32)[None, :],
frame_count,
axis=0,
),
f"test-{phase}",
),
)
monkeypatch.setattr(
dataset,
"_sample_stationary_anchor",
lambda task_start_pose: np.asarray(task_start_pose, dtype=np.float32),
)
monkeypatch.setattr(
dataset,
"_build_task_sequence",
lambda task_motion, stationary_anchor: (
np.asarray(task_motion, dtype=np.float32),
1,
),
)
monkeypatch.setattr("random.randint", lambda low, high: 2)
source = dataset[0]
assert loaded_background_blocks == [(98, 2), (103, 2)]
assert source["background_key"] == "sessions/office"
assert source["background_listen_start_index"] == 100
assert source["listen_steps"] == 3
assert source["motion_steps"] == 2
def test_overfit_voice_selection_is_stable_per_record(monkeypatch):
voices = tuple(
AudioSource(f"seed_audio/voice_{index}", ()) for index in range(4)
)
dataset = FullDuplexAudioCommandDataset.__new__(FullDuplexAudioCommandDataset)
dataset.fixed_voice_per_record = True
selected = [dataset._select_voice(voices, row_id=7) for _ in range(5)]
assert selected == [voices[3]] * 5
dataset.fixed_voice_per_record = False
monkeypatch.setattr("random.choice", lambda values: values[1])
assert dataset._select_voice(voices, row_id=7) is voices[1]
def test_broken_tts_duration_outliers_are_filtered_before_sampling():
def voice(name: str, windows: int) -> AudioSource:
return AudioSource(
name,
(),
waveform_path=Path(f"/unused/{name}.wav"),
waveform_count=windows,
)
dataset = FullDuplexAudioCommandDataset.__new__(
FullDuplexAudioCommandDataset
)
dataset.max_voice_windows = 30
dataset.max_voice_median_ratio = 3.0
catalog = {
"normal-with-corruption": (
voice("short-a", 7),
voice("short-b", 8),
voice("broken", 196),
),
# Defensive fallback keeps a custom long-command directory usable.
"all-long": (voice("long-a", 35), voice("long-b", 40)),
}
filtered, removed = dataset._filter_voice_duration_outliers(catalog)
assert [item.source_key for item in filtered["normal-with-corruption"]] == [
"short-a",
"short-b",
]
assert [item.source_key for item in filtered["all-long"]] == ["long-a"]
assert removed == 2
def test_length_estimate_uses_fixed_voice_and_converts_metadata_fps():
dataset = FullDuplexAudioCommandDataset.__new__(
FullDuplexAudioCommandDataset
)
dataset.records = [
{
"row_id": 1,
"audio_dir": "seed_audio/one",
"declared_frames": 580,
}
]
dataset.audio_catalog = {
"seed_audio/one": (
AudioSource("voice-a", (), packed_path=Path("a"), packed_count=7),
AudioSource("voice-b", (), packed_path=Path("b"), packed_count=10),
)
}
dataset.fixed_voice_per_record = True
dataset.motion_fps = 50
dataset.motion_metadata_fps = 120
dataset.stationary_blend_frames = 20
dataset.prefix_frames = 10
dataset.action_exec_horizon = 20
dataset.initial_idle_max_steps = 5
dataset.max_motion_steps = 0
dataset.max_stream_steps = 1923
# 580 metadata frames -> 242 exported 50 Hz frames. With the 19-frame
# transition there are 250 valid dense motion starts.
assert dataset._estimated_runtime_motion_frames(dataset.records[0]) == 242
assert dataset.estimated_stream_steps(0) == (2 + 10) * 20 + 250
assert dataset._record_max_stream_steps(dataset.records[0]) == 5 + 10 + 13
def test_validation_source_augmentation_is_repeatable_and_restores_rng():
dataset = FullDuplexAudioCommandDataset.__new__(
FullDuplexAudioCommandDataset
)
dataset.records = [{"row_id": 17}]
dataset.split = "val"
dataset.split_seed = 42
dataset.deterministic_eval = True
dataset._build_source = lambda index: {
"index": index,
"voice": random.randrange(8),
"idle": random.randint(0, 5),
"anchor": random.random(),
}
random.seed(1234)
state_before = random.getstate()
first = dataset[0]
assert random.getstate() == state_before
second = dataset[0]
assert first == second
assert random.getstate() == state_before
def test_dataset_prefers_compact_source_indexes_and_reads_one_wav(
tmp_path,
):
source_dir = (
tmp_path
/ "audio"
/ "qwen3_tts_seed_audio_command"
/ "audio_command_text"
/ "sources"
)
source_path = (
source_dir
/ "seed_audio"
/ "000007"
/ "voice.safetensors"
)
source_path.parent.mkdir(parents=True)
hidden = torch.arange(32, dtype=torch.bfloat16).reshape(8, 4)
save_file(
{
"hidden_states": hidden,
"offsets": torch.tensor([0, 3, 8], dtype=torch.int64),
"frame_ids": torch.tensor([0, 20], dtype=torch.int64),
},
source_path,
metadata={
"format": HFD_AUDIO_SOURCE_FORMAT,
"encoder_contract": HFD_ONLINE_ENCODER_CONTRACT,
"encoder_batch_size": str(HFD_ONLINE_ENCODER_BATCH_SIZE),
"encoder_dtype": HFD_ONLINE_ENCODER_DTYPE,
"attention_implementation": HFD_ONLINE_ENCODER_ATTENTION,
"instruction_sha1": hashlib.sha1(
AUDIO_COMMAND_STREAMING_PROMPT.encode("utf-8")
).hexdigest(),
"count": "2",
"window_seconds": "0.600000000",
"step_frames": "20",
"motion_fps": "50",
"sampling_rate": "16000",
},
)
index_path = source_dir / "index.ws1.rank00.sqlite"
with sqlite3.connect(index_path) as conn:
conn.execute(
"CREATE TABLE sources ("
"key TEXT PRIMARY KEY, path TEXT NOT NULL, count INTEGER NOT NULL, "
"window_seconds REAL NOT NULL, step_frames INTEGER NOT NULL, "
"motion_fps INTEGER NOT NULL, sampling_rate INTEGER NOT NULL)"
)
conn.execute(
"INSERT INTO sources VALUES (?, ?, ?, ?, ?, ?, ?)",
(
"seed_audio/000007/voice",
"seed_audio/000007/voice.safetensors",
2,
0.6,
20,
50,
16000,
),
)
dataset = FullDuplexAudioCommandDataset.__new__(
FullDuplexAudioCommandDataset
)
dataset.hidden_root = tmp_path
dataset.source_name = "qwen3_tts_seed_audio_command"
dataset.task_name = "audio_command_text"
dataset.audio_window_seconds = 0.6
dataset.audio_sample_step_frames = 20
dataset.motion_fps = 50
dataset.audio_sampling_rate = 16000
catalog = dataset._load_audio_catalog()
voice = catalog["seed_audio/000007"][0]
windows = dataset._load_voice_hidden(voice)
assert dataset.hidden_layout == "source"
assert voice.packed_path == source_path.resolve()
assert voice.clips == ()
assert voice.window_count == 2
assert torch.equal(windows[0], hidden[:3])
assert torch.equal(windows[1], hidden[3:])
assert windows[0].untyped_storage().data_ptr() == (
windows[1].untyped_storage().data_ptr()
)
def test_stationary_motion_pool_is_kinematically_gated_and_blended(monkeypatch):
anchor = np.zeros(36, dtype=np.float32)
anchor[29:32] = [1.0, 2.0, 0.8]
anchor[32] = 1.0
natural = np.repeat(anchor[None, :], 80, axis=0)
natural[:, 0] = np.sin(np.linspace(0.0, 2.0 * np.pi, 80)) * 0.2
natural[:, 29] = 4.0 + np.sin(np.linspace(0.0, 2.0 * np.pi, 80)) * 0.02
natural[:, 30] = -3.0
natural[:, 31] = 0.92 + np.sin(np.linspace(0.0, np.pi, 80)) * 0.01
dataset = FullDuplexAudioCommandDataset.__new__(
FullDuplexAudioCommandDataset
)
candidate = StationaryMotionSource(
"safe-neutral",
Path("/unused/safe-neutral"),
len(natural),
"neutral_idle",
)
dataset.stationary_motion_pools = {
"idle": (candidate,),
"listen": (candidate,),
}
dataset.stationary_blend_frames = 10
monkeypatch.setattr(dataset, "_load_motion", lambda path: natural)
monkeypatch.setattr("random.randint", lambda low, high: 0)
sequence, name = dataset._sample_stationary_sequence(
"idle",
frame_count=40,
anchor_pose=anchor,
)
assert name == "safe-neutral"
np.testing.assert_allclose(sequence[0], anchor, atol=1e-6)
np.testing.assert_allclose(sequence[-1], anchor, atol=1e-6)
assert np.max(np.abs(sequence[10:-10, 0] - anchor[0])) > 0.05
np.testing.assert_allclose(
sequence[10, 29:31],
anchor[29:31] + natural[10, 29:31] - natural[0, 29:31],
atol=1e-6,
)
np.testing.assert_allclose(sequence[10, 31], natural[10, 31], atol=1e-6)
np.testing.assert_allclose(
np.linalg.norm(sequence[:, 32:36], axis=-1),
np.ones(40),
atol=1e-6,
)
task_start = anchor.copy()
task_start[29:32] = [7.0, 8.0, 0.35]
sampled_anchor = dataset._sample_stationary_anchor(task_start)
np.testing.assert_allclose(sampled_anchor[29:31], task_start[29:31], atol=1e-6)
np.testing.assert_allclose(sampled_anchor[31], natural[0, 31], atol=1e-6)
np.testing.assert_allclose(sampled_anchor[:29], natural[0, :29], atol=1e-6)
locomoting = natural.copy()
locomoting[:, 29] += np.linspace(0.0, 1.0, len(locomoting))
assert dataset._stationary_motion_is_safe(natural)
assert not dataset._stationary_motion_is_safe(locomoting)
def test_v24_idle_uses_motion_residual_instead_of_unrelated_absolute_pose(
monkeypatch,
):
anchor = np.zeros(36, dtype=np.float32)
anchor[0] = 0.20
anchor[31] = 0.82
anchor[32] = 1.0
recorded = np.repeat(anchor[None, :], 60, axis=0)
# The recorded clip is quiet, but its absolute arm pose is unrelated to
# this session's current pose. Only its subtle joint-1 residual is useful.
recorded[:, 0] = 1.50
recorded[:, 1] = 0.05 * np.sin(
np.linspace(0.0, 2.0 * np.pi, len(recorded), dtype=np.float32)
)
source = StationaryMotionSource(
"quiet-different-arm-pose",
Path("/unused/quiet-different-arm-pose"),
len(recorded),
"neutral_idle",
)
dataset = FullDuplexAudioCommandDataset.__new__(
FullDuplexAudioCommandDataset
)
dataset.stationary_motion_pools = {
"idle": (source,),
"listen": (source,),
}
dataset.stationary_blend_frames = 5
dataset.stationary_heading_alignment = False
dataset.safe_stationary_anchors = False
monkeypatch.setattr(dataset, "_load_motion", lambda path: recorded)
monkeypatch.setattr("random.randint", lambda low, high: 0)
dataset.stationary_idle_residual_retargeting = False
legacy, _ = dataset._sample_stationary_sequence(
"idle",
frame_count=60,
anchor_pose=anchor,
)
assert float(np.max(np.abs(legacy[:, 0] - anchor[0]))) > 1.0
dataset.stationary_idle_residual_retargeting = True
retargeted, _ = dataset._sample_stationary_sequence(
"idle",
frame_count=60,
anchor_pose=anchor,
)
np.testing.assert_allclose(retargeted[:, 0], anchor[0], atol=1e-6)
assert float(np.max(np.abs(retargeted[:, 1] - anchor[1]))) > 0.04
np.testing.assert_allclose(retargeted[0], anchor, atol=1e-6)
np.testing.assert_allclose(retargeted[-1], anchor, atol=1e-6)
def test_v24_idle_gate_rejects_jump_cuts_and_large_gestures():
clean = np.zeros((100, 36), dtype=np.float32)
clean[:, 31] = 0.82
clean[:, 32] = 1.0
clean[:, 0] = 0.08 * np.sin(
np.linspace(0.0, 2.0 * np.pi, len(clean), dtype=np.float32)
)
jump_cut = clean.copy()
jump_cut[50:, 3] += 0.20
gesture = clean.copy()
gesture[:, 4] = np.linspace(0.0, 0.60, len(gesture), dtype=np.float32)
assert FullDuplexAudioCommandDataset._idle_motion_is_quiet_and_continuous(
clean
)
assert not FullDuplexAudioCommandDataset._idle_motion_is_quiet_and_continuous(
jump_cut
)
assert not FullDuplexAudioCommandDataset._idle_motion_is_quiet_and_continuous(
gesture
)
def test_safe_stationary_audit_rejects_static_whole_body_outlier(monkeypatch):
dataset = FullDuplexAudioCommandDataset.__new__(
FullDuplexAudioCommandDataset
)
dataset.motion_fps = 50
dataset.stationary_motion_pools = {"idle": (), "listen": ()}
sources = tuple(
StationaryMotionSource(
name,
Path(f"/unused/{name}"),
100,
"neutral_idle",
)
for name in ("neutral-a", "neutral-b", "corrupt-static")
)
dataset.stationary_motion_pools["idle"] = sources
motions = {}
for source in sources:
motion = np.zeros((100, 36), dtype=np.float32)
motion[:, 31] = 0.82
motion[:, 32] = 1.0
motions[str(source.motion_dir)] = motion
motions["/unused/neutral-b"][:, 0] = 0.03
# Static alone is not sufficient: this mimics the real malformed idle clip
# whose arm joints are many robust deviations away from the neutral pool.
motions["/unused/corrupt-static"][:, 22:29] = 2.0
monkeypatch.setattr(
dataset,
"_load_motion",
lambda path: motions[str(path)],
)
dataset._initialise_safe_stationary_anchors()
assert "/unused/corrupt-static" not in dataset._safe_idle_motion_paths
assert dataset.stationary_audit_summary["accepted_clips"] == 2
assert dataset.stationary_audit_summary["rejected_clips"] == 1
assert dataset.stationary_audit_summary["anchor_frames"] > 0
query = motions["/unused/neutral-b"][50].copy()
query[29:31] = [4.0, -2.0]
anchor, _ = dataset._nearest_safe_stationary_anchor(query)
np.testing.assert_allclose(anchor[:29], query[:29], atol=1e-6)
np.testing.assert_allclose(anchor[29:31], query[29:31], atol=1e-6)
def test_stationary_entry_blends_from_executed_pose_to_safe_anchor(monkeypatch):
anchor = np.zeros(36, dtype=np.float32)
anchor[31] = 0.82
anchor[32] = 1.0
entry = anchor.copy()
entry[0] = 0.8
natural = np.repeat(anchor[None, :], 80, axis=0)
natural[:, 1] = 0.2
source = StationaryMotionSource(
"listen",
Path("/unused/listen"),
len(natural),
"listening",
)
dataset = FullDuplexAudioCommandDataset.__new__(
FullDuplexAudioCommandDataset
)
dataset.stationary_motion_pools = {"idle": (source,), "listen": (source,)}
dataset.stationary_blend_frames = 10
dataset.turn_transition_frames = 50
dataset.stationary_heading_alignment = True
dataset.safe_stationary_anchors = False
monkeypatch.setattr(dataset, "_load_motion", lambda path: natural)
monkeypatch.setattr("random.randint", lambda low, high: 0)
sequence, _ = dataset._sample_stationary_sequence(
"listen",
frame_count=80,
anchor_pose=anchor,
blend_frames=50,
entry_pose=entry,
)
np.testing.assert_allclose(sequence[0], entry, atol=1e-6)
np.testing.assert_allclose(sequence[-1], anchor, atol=1e-6)
assert float(sequence[25, 0]) < float(entry[0])
def test_stationary_motion_heading_is_reexpressed_in_current_anchor(monkeypatch):
anchor = np.zeros(36, dtype=np.float32)
anchor[29:32] = [2.0, -1.0, 0.8]
anchor_yaw = 0.5 * np.pi
anchor[32:36] = rpy_to_quat_wxyz(0.0, 0.0, anchor_yaw)
candidate_motion = np.repeat(anchor[None, :], 40, axis=0)
candidate_motion[:, 29:31] = [8.0, 4.0]
recorded_yaw = -0.5 * np.pi
relative_yaw = np.linspace(0.0, 0.1, 40, dtype=np.float32)
candidate_motion[:, 32:36] = np.stack(
[
rpy_to_quat_wxyz(0.0, 0.0, recorded_yaw + float(offset))
for offset in relative_yaw
]
)
dataset = FullDuplexAudioCommandDataset.__new__(
FullDuplexAudioCommandDataset
)
source = StationaryMotionSource(
"listen-facing-another-way",
Path("/unused/listen-facing-another-way"),
len(candidate_motion),
"listening",
)
dataset.stationary_motion_pools = {
"idle": (source,),
"listen": (source,),
}
dataset.stationary_blend_frames = 5
dataset.stationary_heading_alignment = True
monkeypatch.setattr(dataset, "_load_motion", lambda path: candidate_motion)
monkeypatch.setattr("random.randint", lambda low, high: 0)
sequence, _ = dataset._sample_stationary_sequence(
"listen",
frame_count=40,
anchor_pose=anchor,
)
output_yaw = np.unwrap(
np.asarray(
[quat_to_yaw_wxyz(frame[32:36]) for frame in sequence]
)
)
assert abs(float(wrap_angle(output_yaw[0] - anchor_yaw))) < 1e-6
assert abs(float(wrap_angle(output_yaw[-1] - anchor_yaw))) < 1e-6
# The candidate's small relative turn is preserved, while its unrelated
# 180-degree recording orientation is removed.
assert np.max(np.abs(output_yaw - anchor_yaw)) < 0.11
def test_dense_stream_rows_carry_two_source_local_context_tokens():
dataset = FullDuplexAudioCommandDataset.__new__(
FullDuplexAudioCommandDataset
)
dataset.action_chunk_size = 3
dataset.prefix_frames = 1
sequence = np.zeros((8, 36), dtype=np.float32)
sequence[:, 0] = np.arange(8, dtype=np.float32)
sequence[:, 29] = np.arange(8, dtype=np.float32) * 0.1
sequence[:, 32] = 1.0
rows = dataset._stream_action_rows(sequence, start=4)
actions, mask, state, reference, context, context_reference = rows
np.testing.assert_allclose(actions, sequence[5:8])
np.testing.assert_array_equal(mask, np.ones(3, dtype=np.float32))
np.testing.assert_allclose(state, sequence[4])
np.testing.assert_allclose(reference, sequence[4])
np.testing.assert_allclose(context, sequence[2:4 + 1][1:])
np.testing.assert_allclose(context_reference, sequence[2])
start_rows = dataset._stream_action_rows(sequence, start=0)
np.testing.assert_allclose(start_rows[4], np.repeat(sequence[0:1], 2, axis=0))
np.testing.assert_allclose(start_rows[5], sequence[0])
def test_task_motion_transition_starts_from_upright_anchor_without_jump():
dataset = FullDuplexAudioCommandDataset.__new__(
FullDuplexAudioCommandDataset
)
dataset.stationary_blend_frames = 20
anchor = np.zeros(36, dtype=np.float32)
anchor[0] = -0.5
anchor[29:32] = [1.0, 2.0, 0.8]
anchor[32] = 1.0
task = np.repeat(anchor[None, :], 60, axis=0)
task[:, 0] = 0.75
task[:, 31] = 0.5
sequence, transition_frames = dataset._build_task_sequence(
task,
stationary_anchor=anchor,
)
assert transition_frames == 20
assert len(sequence) == len(task) + transition_frames - 1
np.testing.assert_allclose(sequence[0], anchor, atol=1e-6)
np.testing.assert_allclose(sequence[transition_frames - 1], task[0], atol=1e-6)
np.testing.assert_allclose(sequence[transition_frames], task[1], atol=1e-6)
assert np.max(np.linalg.norm(np.diff(sequence[:, :29], axis=0), axis=1)) < 0.11
def test_dynamic_task_entry_blends_into_moving_prefix_without_phase_restart():
dataset = FullDuplexAudioCommandDataset.__new__(
FullDuplexAudioCommandDataset
)
dataset.stationary_blend_frames = 20
dataset.dynamic_task_entry_blend = True
anchor = np.zeros(36, dtype=np.float32)
anchor[31] = 0.82
anchor[32] = 1.0
task = np.repeat(anchor[None, :], 80, axis=0)
task[:, 0] = np.linspace(0.6, 1.4, len(task), dtype=np.float32)
sequence, transition_frames = dataset._build_task_sequence(
task,
stationary_anchor=anchor,
)
assert transition_frames == 20
assert len(sequence) == len(task)
np.testing.assert_allclose(sequence[0], anchor, atol=1e-6)
np.testing.assert_allclose(sequence[19], task[19], atol=1e-6)
# At the crossfade boundary the real trajectory simply continues; it never
# jumps back to primitive frame zero or restarts its phase.
np.testing.assert_allclose(sequence[20], task[20], atol=1e-6)
np.testing.assert_allclose(
sequence[20, :29] - sequence[19, :29],
task[20, :29] - task[19, :29],
atol=1e-6,
)
np.testing.assert_array_equal(
dataset._task_original_frame_indices(
sequence_frames=len(sequence),
motion_frames=len(task),
transition_frames=transition_frames,
),
np.arange(len(task)),
)
def test_task_motion_is_rigidly_aligned_to_stationary_anchor_heading():
dataset = FullDuplexAudioCommandDataset.__new__(
FullDuplexAudioCommandDataset
)
dataset.stationary_blend_frames = 5
anchor = np.zeros(36, dtype=np.float32)
anchor[29:32] = [10.0, -3.0, 0.8]
anchor_yaw = -0.5 * np.pi
anchor[32:36] = rpy_to_quat_wxyz(0.0, 0.0, anchor_yaw)
task = np.zeros((4, 36), dtype=np.float32)
task[:, :29] = np.arange(4, dtype=np.float32)[:, None]
task[:, 29] = 1.0
task[:, 30] = 2.0 + np.arange(4, dtype=np.float32)
task[:, 31] = 0.8 + 0.05 * np.arange(4, dtype=np.float32)
task_yaw = 0.5 * np.pi
yaw_progress = 0.1 * np.arange(4, dtype=np.float32)
task[:, 32:36] = np.stack(
[
rpy_to_quat_wxyz(0.0, 0.0, float(task_yaw + offset))
for offset in yaw_progress
]
)
sequence, transition_frames = dataset._build_task_sequence(
task,
stationary_anchor=anchor,
)
aligned = sequence[transition_frames - 1 :]
np.testing.assert_allclose(sequence[0], anchor, atol=1e-6)
np.testing.assert_allclose(aligned[0, 29:32], anchor[29:32], atol=1e-6)
np.testing.assert_allclose(aligned[:, :29], task[:, :29], atol=1e-6)
np.testing.assert_allclose(
aligned[:, 31] - aligned[0, 31],
task[:, 31] - task[0, 31],
atol=1e-6,
)
aligned_yaw = quat_to_yaw_wxyz(aligned[:, 32:36])
np.testing.assert_allclose(
wrap_angle(aligned_yaw - anchor_yaw),
yaw_progress,
atol=1e-6,
)
original_local_delta = local_delta_from_world(
task[:-1, 29:32],
task[1:, 29:32],
task[:-1, 32:36],
)
aligned_local_delta = local_delta_from_world(
aligned[:-1, 29:32],
aligned[1:, 29:32],
aligned[:-1, 32:36],
)
np.testing.assert_allclose(
aligned_local_delta,
original_local_delta,
atol=1e-6,
)
np.testing.assert_allclose(
np.linalg.norm(aligned[:, 32:36], axis=-1),
np.ones(len(aligned)),
atol=1e-6,
)
def test_kv_cache_is_shared_only_inside_one_source():
memory = StreamingCausalMemory(
input_dim=8,
hidden_dim=16,
num_layers=2,
num_heads=4,
dropout=0.0,
max_cache_tokens=32,
)
cache_a = memory.begin_source(["source-a"])
pooled = torch.randn(1, 8)
_, cache_a, _ = memory(
pooled, cache_a, source_ids=["source-a"], update_cache=True
)
_, cache_a, _ = memory(
pooled, cache_a, source_ids=["source-a"], update_cache=True
)
assert cache_a.sequence_length == 2
# False is reserved for a trainer-inserted padding row after the source has
# ended; it must not insert the placeholder token into semantic history.
_, padded_cache, _ = memory(
torch.randn(1, 8), cache_a, source_ids=["source-a"], update_cache=False
)
assert padded_cache.sequence_length == 2
for before, after in zip(cache_a.keys, padded_cache.keys):
assert before is after
with pytest.raises(RuntimeError, match="across sources"):
memory(pooled, cache_a, source_ids=["source-b"], update_cache=True)
cache_b = memory.begin_source(["source-b"])
assert cache_b.sequence_length == 0
def test_batched_kv_cache_tracks_mixed_active_and_padded_sources():
memory = StreamingCausalMemory(
input_dim=8,
hidden_dim=16,
num_layers=2,
num_heads=4,
dropout=0.0,
max_cache_tokens=32,
).eval()
cache = memory.begin_source(["source-a", "source-b"])
_, cache, _ = memory(
torch.randn(2, 8),
cache,
source_ids=["source-a", "source-b"],
update_cache=[True, True],
)
assert cache.sequence_lengths == (1, 1)
# A has ended while B still has an active audio window. The shared physical
# cache grows once, but the new position is invalid for A and valid for B.
_, cache, _ = memory(
torch.randn(2, 8),
cache,
source_ids=["source-a", "source-b"],
update_cache=[False, True],
)
assert cache.sequence_lengths == (1, 2)
assert cache.valid_mask is not None
assert cache.valid_mask.tolist() == [[True, False], [True, True]]
with pytest.raises(RuntimeError, match="different source lengths"):
_ = cache.sequence_length
before_keys = cache.keys
_, padded_cache, _ = memory(
torch.randn(2, 8),
cache,
source_ids=["source-a", "source-b"],
update_cache=[False, False],
)
assert padded_cache.sequence_lengths == (1, 2)
assert all(before is after for before, after in zip(before_keys, padded_cache.keys))
def test_trainer_stream_forward_is_a_real_two_source_batch():
config = FullDuplexPsi0ModelConfig(
streaming_qwen_enabled=False,
action_dim=3,
action_chunk_size=30,
action_exec_horizon=20,
observation_horizon=1,
odim=3,
n_conditions=0,
hidden_dim=48,
num_blocks=1,
dim_feedforward=64,
nhead=4,
view_feature_dim=8,
use_future_progress_embedding=False,
rtc=True,
max_delay=11,
streaming_memory_dim=16,
streaming_memory_layers=1,
streaming_memory_heads=4,
streaming_max_cache_tokens=8,
)
model = FullDuplexPsi0Model(config, vlm_model=None).eval()
captured_timesteps = []
original_forward = model.forward
def capture_forward(*args, **kwargs):
captured_timesteps.append(kwargs["timestep"].detach().clone())
return original_forward(*args, **kwargs)
model.forward = capture_forward
trainer = PosttrainFullDuplexTrainer.__new__(PosttrainFullDuplexTrainer)
trainer.device = torch.device("cpu")
trainer.rtc_train_overlap = 10
trainer.train_diffusion_steps = 1_000
trainer.loss_w = torch.ones(3)
trainer.loss_group_w = torch.ones(3)
class FixedSampler:
def sample_like(self, actions):
return SimpleNamespace(
sigmas=torch.tensor([0.25, 0.75], dtype=torch.float32),
noise=torch.zeros_like(actions),
)
def make_step(
source_id: str,
*,
listen: bool,
value: float,
step_index: int,
):
return {
"source_id": source_id,
"step_index": step_index,
"vlm_hidden_states": torch.full((5, 8), value),
"attention_mask": torch.ones(5, dtype=torch.long),
"actions": torch.full((30, 3), value),
"actions_mask": torch.ones(30, 3),
"states": torch.zeros(1, 3),
"progress": torch.zeros(5),
"future_progress": torch.zeros(30, 5),
"update_semantic_cache": listen,
"control_label": LISTEN_TOKEN_ID if listen else MOTION_TOKEN_ID,
}
source_ids = ["source-a", "source-b"]
cache = model.begin_source(source_ids)
losses, cache = trainer._forward_stream_batch(
model,
source_ids,
[
make_step("source-a", listen=True, value=1.0, step_index=0),
make_step("source-b", listen=True, value=2.0, step_index=0),
],
cache,
FixedSampler(),
)
assert losses["action_loss"].shape == (2,)
assert losses["control_loss"].shape == (2,)
assert torch.isfinite(losses["action_loss"]).all()
assert cache.sequence_lengths == (1, 1)
# The first source window has no previous prediction, hence no clean RTC
# prefix is leaked even though the sampler fallback requests delay 10.
assert torch.all(captured_timesteps[0][0] == 250.0)
assert torch.all(captured_timesteps[0][1] == 750.0)
losses, cache = trainer._forward_stream_batch(
model,
source_ids,
[
make_step("source-a", listen=False, value=3.0, step_index=1),
make_step("source-b", listen=True, value=4.0, step_index=1),
],
cache,
FixedSampler(),
)
assert losses["action_loss"].shape == (2,)
assert cache.sequence_lengths == (1, 2)
assert torch.all(captured_timesteps[1][:, :10] == 0.0)
@pytest.mark.parametrize("semantics_version", [12, 15, 19, 20])
def test_framewise_control_uses_versioned_future_mask_and_rtc_prefix(
semantics_version,
):
trainer = PosttrainFullDuplexTrainer.__new__(PosttrainFullDuplexTrainer)
trainer.device = torch.device("cpu")
trainer.rtc_train_overlap = 10
trainer.train_diffusion_steps = 1_000
trainer.loss_w = torch.ones(39)
trainer.loss_group_w = torch.ones(39)
trainer.cfg = SimpleNamespace(
train=SimpleNamespace(
full_duplex_semantics_version=semantics_version,
joint_velocity_loss_weight=0.10,
joint_acceleration_loss_weight=0.05,
root_continuity_loss_weight=0.10,
stationary_root_height_loss_weight=1.0,
),
model=SimpleNamespace(max_delay=11),
)
class FixedSampler:
def sample_like(self, actions):
return SimpleNamespace(
sigmas=torch.tensor([0.5], dtype=torch.float32),
noise=torch.zeros_like(actions),
delay=torch.tensor([10], dtype=torch.long),
)
class FakeModel:
def __init__(self):
self.captured_timestep = None
self._latest_reusable_condition = None
def __call__(self, **kwargs):
self.captured_timestep = kwargs["timestep"].detach().clone()
batch = kwargs["action_samples"].shape[0]
logits = torch.zeros((batch, 3), dtype=torch.float32)
probabilities = torch.softmax(logits, dim=-1)
condition = torch.zeros((batch, 1, 8), dtype=torch.float32)
mask = torch.ones((batch, 1), dtype=torch.long)
self._latest_reusable_condition = {
"condition_hidden_states": condition,
"condition_attention_mask": mask,
"control_logits": logits,
"control_probabilities": probabilities,
"control_condition_weights": probabilities,
"control_entropy": torch.full((batch,), float(np.log(3.0))),
"memory_token": condition,
"motion_duration_log1p_seconds": torch.zeros(
batch, dtype=torch.float32
),
}
return SimpleNamespace(
action=torch.zeros_like(kwargs["action_samples"]),
control_logits=logits,
control_probabilities=probabilities,
control_entropy=torch.full((batch,), float(np.log(3.0))),
motion_duration_log1p_seconds=torch.zeros(
batch, dtype=torch.float32
),
latest_instruction_loss=None,
latest_instruction_token_correct=None,
latest_instruction_token_count=None,
kv_cache=kwargs["kv_cache"],
)
actions = torch.zeros((30, 39), dtype=torch.float32)
actions[:, 36] = 2.0
step = {
"source_id": "source-a",
"step_index": 20,
"vlm_hidden_states": torch.zeros((5, 8)),
"attention_mask": torch.ones(5, dtype=torch.long),
"actions": actions,
"actions_mask": torch.ones((30, 39)),
"states": torch.zeros((1, 39)),
"smoothness_context_actions": torch.zeros((2, 39)),
"progress": torch.zeros(5),
"future_progress": torch.zeros((30, 5)),
"update_semantic_cache": True,
"control_label": LISTEN_TOKEN_ID,
"action_control_labels": torch.tensor(
[LISTEN_TOKEN_ID] * 15 + [MOTION_TOKEN_ID] * 15,
dtype=torch.long,
),
"control_transition": True,
"motion_duration_target_seconds": 0.0,
"motion_duration_target_mask": False,
"latest_instruction_text": "",
"latest_instruction_target_mask": False,
}
model = FakeModel()
cache = SimpleNamespace()
losses, _ = trainer._forward_stream_batch(
model,
["source-a"],
[step],
cache,
FixedSampler(),
)
# Historical v12/v15 regenerate all 30 frames and mask the cross-phase
# tail. v19 preserves the previous ten-frame RTC prefix and supervises the
# complete valid future horizon so that preceding windows learn that tail.
if semantics_version in {19, 20}:
assert torch.all(model.captured_timestep[:, :10] == 0.0)
assert torch.all(model.captured_timestep[:, 10:] == 500.0)
else:
assert torch.all(model.captured_timestep == 500.0)
assert losses["cross_phase_frames"].tolist() == [15.0]
assert losses["supervised_phase_frames"].tolist() == [
30.0 if semantics_version in {19, 20} else 15.0
]
assert losses["control_transition"].tolist() == [1.0]
assert float(losses["stationary_root_height_loss"].item()) > 0.0
@pytest.mark.parametrize("semantics_version", [18, 19, 20])
def test_completed_motion_progress_survives_cross_control_rtc_tail(
semantics_version,
):
trainer = PosttrainFullDuplexTrainer.__new__(PosttrainFullDuplexTrainer)
trainer.device = torch.device("cpu")
trainer.rtc_train_overlap = 10
trainer.train_diffusion_steps = 1_000
trainer.loss_w = torch.ones(39)
trainer.loss_group_w = torch.ones(39)
trainer.cfg = SimpleNamespace(
train=SimpleNamespace(
full_duplex_semantics_version=semantics_version,
joint_velocity_loss_weight=0.10,
joint_acceleration_loss_weight=0.05,
root_continuity_loss_weight=0.10,
stationary_root_height_loss_weight=1.0,
),
model=SimpleNamespace(max_delay=11),
)
class FixedSampler:
def sample_like(self, actions):
return SimpleNamespace(
sigmas=torch.tensor([0.5], dtype=torch.float32),
noise=torch.zeros_like(actions),
delay=torch.tensor([10], dtype=torch.long),
)
class FakeModel:
def __init__(self):
self.future_progress = None
self._latest_reusable_condition = None
def __call__(self, **kwargs):
self.future_progress = kwargs["future_progress"].detach().clone()
batch = kwargs["action_samples"].shape[0]
logits = torch.zeros((batch, 3), dtype=torch.float32)
probabilities = torch.softmax(logits, dim=-1)
condition = torch.zeros((batch, 1, 8), dtype=torch.float32)
mask = torch.ones((batch, 1), dtype=torch.long)
self._latest_reusable_condition = {
"condition_hidden_states": condition,
"condition_attention_mask": mask,
"control_logits": logits,
"control_probabilities": probabilities,
"control_condition_weights": probabilities,
"control_entropy": torch.full((batch,), float(np.log(3.0))),
"memory_token": condition,
"motion_duration_log1p_seconds": torch.zeros(
batch, dtype=torch.float32
),
}
return SimpleNamespace(
action=torch.zeros_like(kwargs["action_samples"]),
control_logits=logits,
control_probabilities=probabilities,
control_entropy=torch.full((batch,), float(np.log(3.0))),
motion_duration_log1p_seconds=torch.zeros(
batch, dtype=torch.float32
),
latest_instruction_loss=None,
latest_instruction_token_correct=None,
latest_instruction_token_count=None,
kv_cache=kwargs["kv_cache"],
)
future_progress = torch.zeros((30, 5), dtype=torch.float32)
future_progress[:15] = torch.tensor(
[0.8, 0.2, 0.8, -0.5, 0.5], dtype=torch.float32
)
step = {
"source_id": "source-a",
"step_index": 100,
"vlm_hidden_states": torch.zeros((5, 8)),
"attention_mask": torch.ones(5, dtype=torch.long),
"actions": torch.zeros((30, 39), dtype=torch.float32),
"actions_mask": torch.ones((30, 39), dtype=torch.float32),
"states": torch.zeros((1, 39), dtype=torch.float32),
"smoothness_context_actions": torch.zeros((2, 39), dtype=torch.float32),
"progress": torch.tensor([0.8, 0.2, 0.8, -0.5, 0.5]),
# The raw GT timeline changes to stationary p=0 in the tail.
"future_progress": future_progress,
"update_semantic_cache": True,
"control_label": MOTION_TOKEN_ID,
"action_control_labels": torch.tensor(
[MOTION_TOKEN_ID] * 15 + [IDLE_TOKEN_ID] * 15,
dtype=torch.long,
),
# This is the final motion window, not the next idle window.
"control_transition": False,
"motion_duration_target_seconds": 0.0,
"motion_duration_target_mask": False,
"latest_instruction_text": "",
"latest_instruction_target_mask": False,
}
model = FakeModel()
trainer._forward_stream_batch(
model,
["source-a"],
[step],
SimpleNamespace(),
FixedSampler(),
)
torch.testing.assert_close(
model.future_progress[0, :15],
future_progress[:15],
)
expected_tail = (
torch.tensor([1.0, 0.0, 1.0, 0.0, 1.0]).expand(15, -1)
if semantics_version in {19, 20}
else future_progress[15:]
)
torch.testing.assert_close(model.future_progress[0, 15:], expected_tail)
@pytest.mark.skip(
reason=(
"Legacy cached-hidden optimizer test; HFDv2 native-Qwen accumulation "
"is covered by test_hfdv2_qwen_streaming.py"
)
)
def test_training_step_calls_one_parallel_forward_per_stream_position():
config = FullDuplexPsi0ModelConfig(
streaming_qwen_enabled=False,
action_dim=3,
action_chunk_size=30,
action_exec_horizon=20,
observation_horizon=1,
odim=3,
n_conditions=0,
hidden_dim=48,
num_blocks=1,
dim_feedforward=64,
nhead=4,
view_feature_dim=8,
use_future_progress_embedding=False,
rtc=True,
max_delay=11,
streaming_memory_dim=16,
streaming_memory_layers=1,
streaming_memory_heads=4,
streaming_max_cache_tokens=8,
)
model = FullDuplexPsi0Model(config, vlm_model=None)
model.enable_full_source_gradient_checkpointing()
class FakeAccelerator:
num_processes = 1
sync_gradients = True
@staticmethod
def accumulate(model):
del model
return contextlib.nullcontext()
@staticmethod
def no_sync(model):
del model
return contextlib.nullcontext()
@staticmethod
def autocast():
return contextlib.nullcontext()
@staticmethod
def backward(loss):
loss.backward()
@staticmethod
def unwrap_model(model):
return model
@staticmethod
def gather(value):
return value
class FixedSampler:
@staticmethod
def sample_like(actions):
return SimpleNamespace(
sigmas=torch.full((actions.shape[0],), 0.5),
noise=torch.zeros_like(actions),
)
class Scheduler:
def __init__(self):
self.step_calls = 0
def step(self):
self.step_calls += 1
def get_last_lr(self):
return [1e-4]
trainer = PosttrainFullDuplexTrainer.__new__(PosttrainFullDuplexTrainer)
trainer.device = torch.device("cpu")
trainer.cfg = SimpleNamespace(
train=SimpleNamespace(
tbptt_unroll_steps=0,
control_loss_weight=1.0,
data_parallel="ddp",
gradient_accumulation_steps=1,
skip_resumed_steps=False,
)
)
trainer.model = model
trainer.accelerator = FakeAccelerator()
trainer.flow_random_sampler = FixedSampler()
trainer.rtc_train_overlap = 10
trainer.train_diffusion_steps = 1_000
trainer.loss_w = torch.ones(3)
trainer.loss_group_w = torch.ones(3)
class CountingSgd(torch.optim.SGD):
def __init__(self, parameters):
super().__init__(parameters, lr=1e-3)
self.step_calls = 0
def step(self, closure=None):
self.step_calls += 1
return super().step(closure)
trainer.optimizer = CountingSgd(model.parameters())
trainer.lr_scheduler = Scheduler()
def make_step(source_id: str, phase: str, value: float):
labels = {
"idle": IDLE_TOKEN_ID,
"listen": LISTEN_TOKEN_ID,
"motion": MOTION_TOKEN_ID,
}
return {
"source_id": source_id,
"vlm_hidden_states": torch.full((5, 8), value),
"attention_mask": torch.ones(5, dtype=torch.long),
"actions": torch.full((30, 3), value),
"actions_mask": torch.ones(30, 3),
"states": torch.zeros(1, 3),
"progress": torch.zeros(5),
"future_progress": torch.zeros(30, 5),
"update_semantic_cache": True,
"control_label": labels[phase],
"phase": phase,
}
batch = {
"sources": [
{
"source_id": "source-a",
"idle_steps": 1,
"listen_steps": 1,
"motion_steps": 1,
"steps": [
make_step("source-a", "idle", 0.5),
make_step("source-a", "listen", 1.0),
make_step("source-a", "motion", 2.0),
],
},
{
"source_id": "source-b",
"idle_steps": 0,
"listen_steps": 2,
"motion_steps": 1,
"steps": [
make_step("source-b", "listen", 3.0),
make_step("source-b", "motion", 4.0),
make_step("source-b", "listen", 5.0),
],
},
]
}
seen_batch_sizes: list[int] = []
seen_cache_lengths: list[tuple[int, ...]] = []
def record_batch_size(module, args, kwargs):
del module, args
seen_batch_sizes.append(int(kwargs["vlm_hidden_states"].shape[0]))
original_forward_stream_batch = trainer._forward_stream_batch
def record_cache_lengths(model, source_ids, steps, cache, sampler):
seen_cache_lengths.append(cache.sequence_lengths)
return original_forward_stream_batch(
model, source_ids, steps, cache, sampler
)
trainer._forward_stream_batch = record_cache_lengths
hook = model.register_forward_pre_hook(record_batch_size, with_kwargs=True)
try:
events = list(
trainer.iter_optimizer_steps(
batch,
global_step_start=0,
local_step=0,
)
)
finally:
hook.remove()
# Three parallel time positions, not five serial source forwards.
assert seen_batch_sizes == [2, 2, 2]
assert seen_cache_lengths == [(0, 0), (1, 1), (2, 2)]
assert len(events) == 1
assert trainer.optimizer.step_calls == 1
assert trainer.lr_scheduler.step_calls == 1
sync, metrics, source_group_end = events[0]
assert sync is True
assert source_group_end is True
assert metrics["source_step_seconds"] > 0.0
assert metrics["active_windows_per_second"] > 0.0
assert np.isfinite(metrics["loss"])
assert metrics["loss"] == pytest.approx(
metrics["loss_action"]
+ trainer.train_cfg.control_loss_weight * metrics["loss_control"]
)
assert {
"loss_objective",
"loss_window_mean",
"sources_per_second",
"optimizer_source_windows",
"parallel_source_batch",
"peak_memory_allocated_gib",
"peak_memory_reserved_gib",
"idle_steps",
"listen_steps",
"motion_steps",
"full_source_bptt",
"source_group_complete",
"chunks_in_source_group",
"chunk_index",
"optimizer_chunk_steps",
}.isdisjoint(metrics)
# A max-step stop never starts and truncates a source graph.
seen_cache_lengths.clear()
stopped_events = list(
trainer.iter_optimizer_steps(
batch,
global_step_start=1,
local_step=1,
max_optimizer_events=0,
)
)
assert stopped_events == []
assert trainer.optimizer.step_calls == 1
assert trainer.lr_scheduler.step_calls == 1
assert seen_cache_lengths == []
assert trainer._last_cache_state == "empty_source_boundary"
def test_fully_padded_rank_still_advances_one_flow_draw():
trainer = PosttrainFullDuplexTrainer.__new__(PosttrainFullDuplexTrainer)
trainer.device = torch.device("cpu")
class CountingSampler:
def __init__(self):
self.shapes = []
def sample_like(self, actions):
self.shapes.append(tuple(actions.shape))
return None
trainer.flow_random_sampler = CountingSampler()
trainer._advance_padded_flow_rng(
SimpleNamespace(action_exec_horizon=20, action_dim=3),
source_batch_size=2,
)
assert trainer.flow_random_sampler.shapes == [(2, 30, 3)]
def test_cache_free_checkpoint_marker_records_window_step_training(
tmp_path,
monkeypatch,
):
contract = {
"format": "hfd_qwen_serial_train_infer_contract_v6",
"identity": "test",
}
data_contract = {
"format": "hfd_continuous_office_production_v1",
"source_count": 2,
"mixed_window_count": 9,
"background_source_count": 1,
"background_window_count": 20,
"initial_idle_max_steps": 5,
"stationary_blend_frames": 20,
"safety_steps": 10,
"assignment_seed": 260730,
"assignment_sha256": "assignment",
}
monkeypatch.setattr(
"psi.trainers.posttrain_full_duplex."
"_active_continuous_data_contract_identity",
lambda: data_contract,
)
trainer = PosttrainFullDuplexTrainer.__new__(PosttrainFullDuplexTrainer)
trainer.cfg = SimpleNamespace(
train=SimpleNamespace(
full_duplex_semantics_version=11,
joint_velocity_loss_weight=0.10,
joint_acceleration_loss_weight=0.05,
root_continuity_loss_weight=0.10,
tbptt_unroll_steps=0,
full_source_gradient_checkpointing=True,
qwen_kv_gradient_horizon_windows=1,
source_accumulation_groups=1,
train_batch_size=8,
),
model=SimpleNamespace(
streaming_qwen_trainable_last_layers=2,
streaming_qwen_train_final_norm=True,
control_condition_train_mode="hard",
control_condition_inference_mode="hard",
),
)
trainer._hfdv2_checkpoint_contract = lambda: contract
class FakeAccelerator:
is_main_process = True
@staticmethod
def wait_for_everyone():
return None
trainer.accelerator = FakeAccelerator()
trainer._last_cache_state = "empty_source_boundary"
trainer._last_source_group_complete = True
trainer._write_full_duplex_boundary_marker(
str(tmp_path),
global_step=7,
)
marker = json.loads(
(tmp_path / "full_duplex_source_boundary.json").read_text()
)
assert marker["global_step"] == 7
assert marker["full_duplex_semantics_version"] == 11
assert marker["architecture"] == "hfd_qwen_native_kv_serial_control"
assert marker["online_raw_audio"] is True
assert marker["semantic_memory"] == "qwen_dynamic_cache"
assert marker["action_streaming_memory"] is False
assert marker["serial_control_conditioning"] is True
assert marker["control_condition_train_mode"] == "hard"
assert marker["control_condition_inference_mode"] == "hard"
assert marker["control_condition_source"] == "predicted_no_ground_truth"
assert marker["joint_velocity_loss_weight"] == 0.10
assert marker["joint_acceleration_loss_weight"] == 0.05
assert marker["root_continuity_loss_weight"] == 0.10
assert marker["optimizer_scope"] == "one_aligned_stream_window"
assert marker["optimizer_chunk_steps"] == 1
assert marker["source_tail_loss_normalization"] == (
"fixed_global_source_capacity"
)
assert marker["source_slot_policy"] == (
"qwen_boundary_dynamic_refill_v1"
)
assert marker["qwen_kv_gradient_horizon_windows"] == 1
assert marker["source_accumulation_groups"] == 1
assert marker["effective_sources_per_gpu"] == 8
assert marker["action_gradient_checkpointing"] is True
assert marker["cache_state"] == "empty_source_boundary"
assert marker["source_group_complete"] is True
assert marker["train_infer_contract"] == contract
assert marker["continuous_audio_data_contract"] == data_contract
def test_small_full_duplex_model_has_finite_action_and_control_gradients():
config = FullDuplexPsi0ModelConfig(
streaming_qwen_enabled=False,
action_dim=3,
action_chunk_size=30,
action_exec_horizon=20,
observation_horizon=1,
odim=3,
n_conditions=0,
hidden_dim=48,
num_blocks=1,
dim_feedforward=64,
nhead=4,
view_feature_dim=8,
use_future_progress_embedding=False,
rtc=True,
max_delay=11,
streaming_memory_dim=16,
streaming_memory_layers=1,
streaming_memory_heads=4,
streaming_max_cache_tokens=8,
)
model = FullDuplexPsi0Model(config, vlm_model=None)
cache = model.begin_source(["source-a"])
output = model(
source_ids=["source-a"],
kv_cache=cache,
update_semantic_cache=True,
vlm_hidden_states=torch.randn(1, 5, 8),
attention_mask=torch.ones(1, 5, dtype=torch.long),
action_samples=torch.randn(1, 20, 3),
known_action_prefix=torch.randn(1, 10, 3),
states=torch.randn(1, 1, 3),
timestep=torch.tensor([500.0]),
)
assert output.action.shape == (1, 20, 3)
assert output.control_logits.shape == (1, 3)
loss = output.action.float().square().mean() + F.cross_entropy(
output.control_logits.float(), torch.tensor([LISTEN_TOKEN_ID])
)
assert torch.isfinite(loss)
loss.backward()
assert model.control_head[-1].weight.grad is not None
assert torch.isfinite(model.control_head[-1].weight.grad).all()
@pytest.mark.parametrize("action_recompute", [False, True])
def test_late_window_loss_reaches_early_source_input_through_kv_graph(
action_recompute,
):
config = FullDuplexPsi0ModelConfig(
streaming_qwen_enabled=False,
action_dim=3,
action_chunk_size=30,
action_exec_horizon=20,
observation_horizon=1,
odim=3,
n_conditions=0,
hidden_dim=48,
num_blocks=1,
dim_feedforward=64,
nhead=4,
view_feature_dim=8,
use_future_progress_embedding=False,
rtc=True,
max_delay=11,
streaming_memory_dim=16,
streaming_memory_layers=1,
streaming_memory_heads=4,
streaming_max_cache_tokens=8,
)
model = FullDuplexPsi0Model(config, vlm_model=None)
if action_recompute:
model.enable_full_source_gradient_checkpointing()
cache = model.begin_source(["source-a"])
early_hidden = torch.randn(1, 5, 8, requires_grad=True)
early = model(
source_ids=["source-a"],
kv_cache=cache,
update_semantic_cache=True,
vlm_hidden_states=early_hidden,
attention_mask=torch.ones(1, 5, dtype=torch.long),
action_samples=torch.randn(1, 20, 3),
known_action_prefix=torch.randn(1, 10, 3),
states=torch.randn(1, 1, 3),
timestep=torch.tensor([500.0]),
)
late = model(
source_ids=["source-a"],
kv_cache=early.kv_cache,
update_semantic_cache=True,
vlm_hidden_states=torch.randn(1, 5, 8),
attention_mask=torch.ones(1, 5, dtype=torch.long),
action_samples=torch.randn(1, 20, 3),
known_action_prefix=torch.randn(1, 10, 3),
states=torch.randn(1, 1, 3),
timestep=torch.tensor([500.0]),
)
late_loss = late.action.float().square().mean() + F.cross_entropy(
late.control_logits.float(),
torch.tensor([MOTION_TOKEN_ID]),
)
late_loss.backward()
assert early_hidden.grad is not None
assert torch.isfinite(early_hidden.grad).all()
assert float(early_hidden.grad.abs().sum()) > 0.0
def test_stream_condition_is_encoded_once_across_flow_solver_steps():
config = FullDuplexPsi0ModelConfig(
streaming_qwen_enabled=False,
action_dim=3,
action_chunk_size=30,
action_exec_horizon=20,
observation_horizon=1,
odim=3,
n_conditions=0,
hidden_dim=48,
num_blocks=1,
dim_feedforward=64,
nhead=4,
view_feature_dim=8,
use_future_progress_embedding=False,
rtc=True,
max_delay=11,
streaming_memory_dim=16,
streaming_memory_layers=1,
streaming_memory_heads=4,
streaming_max_cache_tokens=8,
)
model = FullDuplexPsi0Model(config, vlm_model=None).eval()
cache = model.begin_source(["source-a"])
speech = model.encode_stream_input(
source_ids=["source-a"],
kv_cache=cache,
update_semantic_cache=True,
vlm_hidden_states=torch.randn(1, 5, 8),
attention_mask=torch.ones(1, 5, dtype=torch.long),
)
assert speech.kv_cache.sequence_length == 1
for timestep in (900.0, 100.0):
action = model.forward_action_conditioned(
condition=speech,
action_samples=torch.randn(1, 20, 3),
known_action_prefix=torch.randn(1, 10, 3),
states=torch.randn(1, 1, 3),
timestep=torch.tensor([timestep]),
)
assert action.shape == (1, 20, 3)
assert speech.kv_cache.sequence_length == 1
padded = model.encode_stream_input(
source_ids=["source-a"],
kv_cache=speech.kv_cache,
update_semantic_cache=False,
vlm_hidden_states=torch.randn(1, 4, 8),
attention_mask=torch.ones(1, 4, dtype=torch.long),
)
assert padded.kv_cache.sequence_length == 1
def test_length_group_sampler_balances_each_global_rank_group():
class Dataset:
lengths = [100, 1, 98, 2, 96, 3, 94, 4]
def __len__(self):
return len(self.lengths)
def estimated_stream_steps(self, index):
return self.lengths[index]
dataset = Dataset()
sampler = SourceLengthGroupedSampler(
dataset, world_size=2, batch_size=1, seed=17
)
assert len(sampler) // sampler.group_size == 4
first = list(sampler)
sampler.set_epoch(1)
second = list(sampler)
assert sorted(first) == list(range(len(dataset)))
assert sorted(second) == list(range(len(dataset)))
assert first != second
for offset in range(0, len(first), 2):
pair = [dataset.lengths[index] for index in first[offset : offset + 2]]
assert max(pair) - min(pair) <= 3
def test_length_group_sampler_balances_long_source_work_across_ranks():
class Dataset:
lengths = [1_000, 900, 800, 700, 600, 500, 400, 300]
def __len__(self):
return len(self.lengths)
def estimated_stream_steps(self, index):
return self.lengths[index]
dataset = Dataset()
sampler = SourceLengthGroupedSampler(
dataset, world_size=2, batch_size=4, seed=23
)
order = list(sampler)
rank_rows = [order[rank::2] for rank in range(2)]
assert all(len(rows) == 4 for rows in rank_rows)
assert {
max(dataset.lengths[index] for index in rows)
for rows in rank_rows
} == {900, 1_000}
for offset in range(0, len(order), 2):
aligned = [
dataset.lengths[index]
for index in order[offset : offset + 2]
]
assert max(aligned) - min(aligned) <= 100
def test_source_batching_loader_groups_independently_loaded_sources():
singleton_loader = [
{"sources": [{"source_id": f"source-{index}"}]}
for index in range(6)
]
loader = SourceBatchingLoader(singleton_loader, batch_size=3)
assert len(loader) == 2
batches = list(loader)
assert [
[source["source_id"] for source in batch["sources"]]
for batch in batches
] == [
["source-0", "source-1", "source-2"],
["source-3", "source-4", "source-5"],
]
def test_source_batching_loader_rebuckets_materialized_exact_lengths():
lengths = [8, 1, 7, 2]
singleton_loader = [
{
"sources": [
{
"source_id": f"source-{index}",
"steps": [{} for _ in range(length)],
}
]
}
for index, length in enumerate(lengths)
]
loader = SourceBatchingLoader(
singleton_loader,
batch_size=2,
length_lookahead_batches=2,
)
batches = list(loader)
assert [
[len(source["steps"]) for source in batch["sources"]]
for batch in batches
] == [[1, 2], [7, 8]]
assert sorted(
source["source_id"]
for batch in batches
for source in batch["sources"]
) == [f"source-{index}" for index in range(4)]
def test_source_batching_loader_flushes_short_final_lookahead_block():
singleton_loader = [
{
"sources": [
{
"source_id": f"source-{index}",
"steps": [{} for _ in range(length)],
}
]
}
for index, length in enumerate([6, 1, 5, 2, 4, 3])
]
loader = SourceBatchingLoader(
singleton_loader,
batch_size=2,
length_lookahead_batches=2,
)
assert [
[len(source["steps"]) for source in batch["sources"]]
for batch in loader
] == [[1, 2], [5, 6], [3, 4]]
def test_source_prefetch_defaults_to_two_complete_local_batches(monkeypatch):
monkeypatch.delenv("PSI_DATALOADER_PREFETCH_FACTOR", raising=False)
assert _source_prefetch_factor(workers=16, local_source_batch=64) == 8
monkeypatch.setenv("PSI_DATALOADER_PREFETCH_FACTOR", "3")
assert _source_prefetch_factor(workers=16, local_source_batch=64) == 3
def test_source_prefetch_keeps_two_batches_ready_for_exact_lookahead(monkeypatch):
monkeypatch.delenv("PSI_DATALOADER_PREFETCH_FACTOR", raising=False)
assert (
_source_prefetch_factor(
workers=16,
local_source_batch=128,
)
== 16
)
def test_session_voice_lookup_index_preserves_exact_audio_source():
dataset = FullDuplexAudioCommandDataset.__new__(
FullDuplexAudioCommandDataset
)
voices = (
AudioSource("seed_audio/000001/female_01_en", ()),
AudioSource("seed_audio/000001/female_01_zh", ()),
)
dataset.audio_catalog = {"seed_audio/000001": voices}
dataset._audio_catalog_by_voice_name = {
"seed_audio/000001": {
dataset._voice_name(voice): voice for voice in voices
}
}
record = {
"row_id": 1,
"audio_dir": "seed_audio/000001",
"_session_voice_name": "female_01_zh",
}
assert dataset._voice_for_record(record) is voices[1]
record["_exclude_zh_voices"] = True
with pytest.raises(RuntimeError, match="unavailable or ambiguous"):
dataset._voice_for_record(record)
def test_motion_cache_is_reserved_for_reused_stationary_sources(
monkeypatch, tmp_path
):
dataset = FullDuplexAudioCommandDataset.__new__(
FullDuplexAudioCommandDataset
)
stationary = tmp_path / "stationary"
task = tmp_path / "task"
dataset.motion_cache_size = 256
dataset._motion_cache = OrderedDict()
dataset._stationary_motion_paths = frozenset({str(stationary)})
reads = {str(stationary): 0, str(task): 0}
class FakeFrame:
def __init__(self, values):
self.values = values
def to_numpy(self, dtype):
return self.values.astype(dtype)
def fake_read_csv(path):
motion_dir = str(Path(path).parent)
reads[motion_dir] += 1
columns = {
"joint_pos.csv": 29,
"body_pos.csv": 3,
"body_quat.csv": 4,
}[Path(path).name]
return FakeFrame(np.zeros((2, columns), dtype=np.float32))
monkeypatch.setattr(
"psi.data.seed.full_duplex_audio_command_dataset.pd.read_csv",
fake_read_csv,
)
dataset._load_motion(stationary)
dataset._load_motion(stationary)
dataset._load_motion(task)
dataset._load_motion(task)
assert reads[str(stationary)] == 3
assert reads[str(task)] == 6
assert list(dataset._motion_cache) == [str(stationary)]
def test_manual_gradient_average_includes_remote_missing_grad(monkeypatch):
first = torch.nn.Parameter(torch.zeros(2))
second = torch.nn.Parameter(torch.zeros(1))
first.grad = torch.tensor([2.0, 4.0])
second.grad = None
monkeypatch.setattr(torch.distributed, "is_initialized", lambda: True)
def fake_all_reduce(flat, op):
del op
flat.add_(torch.tensor([4.0, 8.0, 6.0]))
monkeypatch.setattr(torch.distributed, "all_reduce", fake_all_reduce)
average_full_duplex_gradients([first, second], world_size=2, bucket_cap_mb=1)
torch.testing.assert_close(first.grad, torch.tensor([3.0, 6.0]))
torch.testing.assert_close(second.grad, torch.tensor([3.0]))
def test_audio_windows_and_motion_start_use_integer_sample_timeline():
assert complete_audio_clip_starts(9_599) == []
assert complete_audio_clip_starts(16_100) == [0, 6_400]
assert aligned_motion_start_frame(16_000) == 50
assert aligned_motion_start_frame(16_100) == 51
from __future__ import annotations
import hashlib
import contextlib
import importlib.util
import json
import random
import sqlite3
import sys
import wave
from collections import OrderedDict
from pathlib import Path
from types import SimpleNamespace
import numpy as np
import pytest
import torch
import torch.nn.functional as F
from torch import nn
from safetensors.torch import save_file
from psi.config.transform import SeedPosttrainRepackTransform
from psi.config.model_full_duplex_psi0 import FullDuplexPsi0ModelConfig
from psi.data.seed.full_duplex_audio_command_dataset import (
AudioSource,
BackgroundWaveformSource,
CANONICAL_MOTION_MANIFEST_FORMAT,
DISFLUENCY_UNCERTAINTY_ROLES,
DisfluencyAudioSource,
FullDuplexAudioCommandDataset,
FullDuplexTransformedDataset,
HiddenSlice,
IDLE_TOKEN_ID,
LISTEN_TOKEN_ID,
MOTION_TOKEN_ID,
SourceBackgroundAssignment,
StationaryMotionSource,
detect_clean_speech_onset_sample,
has_known_tts_translation_mismatch,
instruction_text_for_voice,
load_canonical_motion_selection,
)
from psi.data.seed.full_duplex_audio_contract import (
AUDIO_COMMAND_STREAMING_PROMPT,
HFD_AUDIO_SOURCE_FORMAT,
HFD_ONLINE_ENCODER_ATTENTION,
HFD_ONLINE_ENCODER_BATCH_SIZE,
HFD_ONLINE_ENCODER_CONTRACT,
HFD_ONLINE_ENCODER_DTYPE,
)
from psi.models.full_duplex_psi0 import FullDuplexPsi0Model, StreamingCausalMemory
from psi.trainers.posttrain_full_duplex import (
PosttrainFullDuplexTrainer,
SourceBatchingLoader,
SourceLengthGroupedSampler,
_source_prefetch_factor,
average_full_duplex_gradients,
)
from psi.data.seed.full_duplex_timing import (
aligned_motion_start_frame,
complete_audio_clip_starts,
)
from psi.data.seed.motion_repr import (
local_delta_from_world,
quat_to_yaw_wxyz,
rpy_to_quat_wxyz,
wrap_angle,
)
from psi.data.seed.source_split import select_source_row_ids
def test_train_and_eval_are_disjoint_at_complete_source_level():
source_ids = ["source-a", "source-a", "source-b", "source-c", "source-c"]
train_rows = select_source_row_ids(
source_ids, split="train", val_ratio=0.25, split_seed=42
)
eval_rows = select_source_row_ids(
source_ids, split="val", val_ratio=0.25, split_seed=42
)
train_sources = {source_ids[index] for index in train_rows}
eval_sources = {source_ids[index] for index in eval_rows}
assert train_sources.isdisjoint(eval_sources)
def test_clean_speech_onset_ignores_leading_silence_and_pcm_click():
rate = 16_000
waveform = np.zeros(rate, dtype=np.float32)
waveform[800] = 0.8
start = 4_000
time_axis = np.arange(rate - start, dtype=np.float32) / rate
waveform[start:] = 0.1 * np.sin(2.0 * np.pi * 220.0 * time_axis)
onset = detect_clean_speech_onset_sample(waveform, rate)
assert start - 320 <= onset <= start + 160
def test_latest_instruction_text_matches_selected_tts_language():
record = {
"row_id": 7,