code-HFD / scripts /data /cache_omni_hidden_states_test.py
Tinghong-Ye22's picture
Add files using upload-large-folder tool
9fc9873 verified
Raw History Blame Contribute Delete
10.9 kB
from __future__ import annotations
import json
import tempfile
import unittest
import wave
import sqlite3
from pathlib import Path
from types import SimpleNamespace
from unittest import mock
import torch
from safetensors import safe_open
from scripts.data.cache_omni_hidden_states import (
SourcePreset,
assign_source_to_least_loaded_rank,
build_rank_source_sqlite_index,
complete_audio_window_count,
iter_audio_items,
iter_complete_audio_window_frame_ids,
instruction_for_item,
save_hidden_state_source_cpu,
source_hidden_file_is_complete,
source_index_row,
)
from psi.data.seed.full_duplex_audio_contract import AUDIO_COMMAND_STREAMING_PROMPT
def _write_pcm16_wav(path: Path, *, duration_seconds: float, sampling_rate: int = 24_000) -> None:
path.parent.mkdir(parents=True, exist_ok=True)
sample_count = int(round(duration_seconds * sampling_rate))
with wave.open(str(path), "wb") as wav_file:
wav_file.setnchannels(1)
wav_file.setsampwidth(2)
wav_file.setframerate(sampling_rate)
wav_file.writeframes(b"\0\0" * sample_count)
def _window_args(
*,
full_audio: bool,
storage_format: str = "file",
limit_sources: int = 0,
) -> SimpleNamespace:
return SimpleNamespace(
audio_sampling_rate=16_000,
audio_command_full_audio=full_audio,
audio_command_window_seconds=0.6,
audio_command_sample_step_frames=20,
motion_fps=50,
storage_format=storage_format,
limit_sources=limit_sources,
)
class AudioCommandWindowTest(unittest.TestCase):
def test_complete_sources_are_balanced_by_window_count(self) -> None:
loads = [0, 0]
owners = [
assign_source_to_least_loaded_rank(loads, count)
for count in (100, 1, 100, 1)
]
self.assertEqual(owners, [0, 1, 1, 0])
self.assertEqual(loads, [101, 101])
def test_audio_command_prompt_is_identical_for_cache_and_live_inference(self) -> None:
first = instruction_for_item(
"qwen3_tts_seed_audio_command",
"audio_command_text",
{"filename": "walk", "content_short_description": "walk forward"},
)
second = instruction_for_item(
"hfd_white_noise",
"audio_command_text",
{"filename": "background"},
)
self.assertEqual(first, AUDIO_COMMAND_STREAMING_PROMPT)
self.assertEqual(second, AUDIO_COMMAND_STREAMING_PROMPT)
def test_complete_audio_windows_drop_incomplete_tail(self) -> None:
with tempfile.TemporaryDirectory() as tmp_dir:
tmp_path = Path(tmp_dir)
wav_path = tmp_path / "one_second.wav"
_write_pcm16_wav(wav_path, duration_seconds=1.0)
kwargs = {
"sampling_rate": 16_000,
"window_seconds": 0.6,
"step_frames": 20,
"motion_fps": 50,
}
self.assertEqual(complete_audio_window_count(wav_path, **kwargs), 2)
self.assertEqual(list(iter_complete_audio_window_frame_ids(wav_path, **kwargs)), [0, 20])
short_path = tmp_path / "short.wav"
_write_pcm16_wav(short_path, duration_seconds=0.4)
self.assertEqual(complete_audio_window_count(short_path, **kwargs), 0)
def test_audio_command_manifest_expands_unique_wavs_into_windows(self) -> None:
with tempfile.TemporaryDirectory() as tmp_dir:
tmp_path = Path(tmp_dir)
audio_dir = tmp_path / "seed_audio" / "000000"
_write_pcm16_wav(audio_dir / "long.wav", duration_seconds=1.0)
_write_pcm16_wav(audio_dir / "short.wav", duration_seconds=0.4)
manifest_path = tmp_path / "metadata.jsonl"
rows = [
{
"filename": "motion_a",
"content_short_description": "walk forward",
"audio_path": "seed_audio/000000",
},
{
"filename": "motion_b",
"content_short_description": "walk forward",
"audio_path": "seed_audio/000000",
},
]
manifest_path.write_text("".join(json.dumps(row) + "\n" for row in rows), encoding="utf-8")
preset = SourcePreset(
name="qwen3_tts_seed_audio_command",
modality="audio",
dataset_root=tmp_path,
manifest_path=manifest_path,
allowed_tasks=("audio_command_text",),
instruction_key="content_short_description",
is_audio_dir_manifest=True,
audio_file_extensions=(".wav",),
)
items = list(
iter_audio_items(
preset,
"audio_command_text",
tmp_path / "hidden",
_window_args(full_audio=False),
)
)
self.assertEqual([item.audio_start_sec for item in items], [0.0, 0.4])
self.assertEqual([item.audio_duration_sec for item in items], [0.6, 0.6])
self.assertEqual(
[item.output_path.name for item in items],
["long.f00000000.safetensors", "long.f00000020.safetensors"],
)
self.assertTrue(all(not item.full_audio for item in items))
legacy_items = list(
iter_audio_items(
preset,
"audio_command_text",
tmp_path / "hidden",
_window_args(full_audio=True),
)
)
self.assertEqual(
[item.output_path.name for item in legacy_items],
["long.safetensors", "short.safetensors"],
)
self.assertTrue(all(item.full_audio for item in legacy_items))
def test_source_layout_keeps_one_complete_wav_in_one_file(self) -> None:
with tempfile.TemporaryDirectory() as tmp_dir:
tmp_path = Path(tmp_dir)
audio_dir = tmp_path / "seed_audio" / "000000"
_write_pcm16_wav(audio_dir / "first.wav", duration_seconds=1.0)
_write_pcm16_wav(audio_dir / "second.wav", duration_seconds=1.0)
manifest_path = tmp_path / "metadata.jsonl"
manifest_path.write_text(
json.dumps(
{
"filename": "motion_a",
"content_short_description": "walk forward",
"audio_path": "seed_audio/000000",
}
)
+ "\n",
encoding="utf-8",
)
preset = SourcePreset(
name="qwen3_tts_seed_audio_command",
modality="audio",
dataset_root=tmp_path,
manifest_path=manifest_path,
allowed_tasks=("audio_command_text",),
instruction_key="content_short_description",
is_audio_dir_manifest=True,
audio_file_extensions=(".wav",),
)
output_root = tmp_path / "hidden"
items = list(
iter_audio_items(
preset,
"audio_command_text",
output_root,
_window_args(
full_audio=False,
storage_format="source",
limit_sources=1,
),
)
)
self.assertEqual(len(items), 2)
self.assertEqual([item.frame_id for item in items], [0, 20])
self.assertEqual(
[item.source_item_index for item in items],
[0, 1],
)
self.assertTrue(
all(item.source_item_count == 2 for item in items)
)
self.assertEqual(
{item.source_key for item in items},
{"seed_audio/000000/first"},
)
self.assertEqual(
len({item.source_output_path for item in items}),
1,
)
source_path = items[0].source_output_path
assert source_path is not None
self.assertEqual(
source_path.relative_to(output_root).as_posix(),
(
"audio/qwen3_tts_seed_audio_command/audio_command_text/"
"sources/seed_audio/000000/first.safetensors"
),
)
hiddens = [
torch.arange(12, dtype=torch.bfloat16).reshape(3, 4),
torch.arange(20, dtype=torch.bfloat16).reshape(5, 4),
]
with mock.patch(
"scripts.data.cache_omni_hidden_states.ACTIVE_MODEL_FINGERPRINT",
"unit-test-model",
):
stats = save_hidden_state_source_cpu(
items,
hiddens,
overwrite=False,
)
self.assertEqual(stats["written"], 2)
self.assertTrue(source_hidden_file_is_complete(items[0]))
with safe_open(
str(source_path),
framework="pt",
device="cpu",
) as reader:
self.assertEqual(
reader.get_tensor("offsets").tolist(),
[0, 3, 8],
)
self.assertEqual(
reader.get_tensor("frame_ids").tolist(),
[0, 20],
)
self.assertEqual(
tuple(reader.get_tensor("hidden_states").shape),
(8, 4),
)
row = source_index_row(items[0], output_root)
index_path = build_rank_source_sqlite_index(
output_root,
preset.name,
"audio_command_text",
"audio",
[row],
shard_rank=0,
)
with sqlite3.connect(
f"file:{index_path}?mode=ro",
uri=True,
) as conn:
indexed = conn.execute(
"SELECT key, path, count, window_seconds, step_frames "
"FROM sources"
).fetchone()
self.assertEqual(
indexed,
(
"seed_audio/000000/first",
"seed_audio/000000/first.safetensors",
2,
0.6,
20,
),
)
self.assertEqual(list(source_path.parent.glob("*.jsonl")), [])
if __name__ == "__main__":
unittest.main()