from __future__ import annotations import hashlib import shutil import threading import time from pathlib import Path import torch from safetensors.torch import save_file from psi.data.seed.full_duplex_audio_contract import ( AUDIO_COMMAND_STREAMING_PROMPT, HFD_AUDIO_SOURCE_FORMAT, HFD_ONLINE_ENCODER_ATTENTION, HFD_ONLINE_ENCODER_BATCH_SIZE, HFD_ONLINE_ENCODER_CONTRACT, HFD_ONLINE_ENCODER_DTYPE, ) from psi.data.seed import omni_hidden_cache def test_direct_oss_uri_keeps_path_relative_to_mount(tmp_path, monkeypatch): mount_root = tmp_path / "mounted" shard = mount_root / "tasks" / "text" / "shards" / "one.safetensors" monkeypatch.setenv("PSI_OMNI_DIRECT_OSS_MOUNT_ROOT", str(mount_root)) monkeypatch.setenv("PSI_OMNI_DIRECT_OSS_URI_ROOT", "oss://bucket/dataset/") assert omni_hidden_cache._direct_oss_uri(shard) == ( "oss://bucket/dataset/tasks/text/shards/one.safetensors" ) def test_hidden_slice_does_not_retain_complete_safetensors_storage( tmp_path, monkeypatch ): omni_hidden_cache.close_cached_omni_shards() shard = tmp_path / "large-parent.safetensors" parent = torch.arange( 2_048 * 128, dtype=torch.bfloat16, ).reshape(2_048, 128) save_file({"hidden_states": parent}, shard) monkeypatch.setenv("PSI_OMNI_SAFE_OPEN_CACHE_SIZE", "0") monkeypatch.setenv("PSI_OMNI_FULL_SHARD_CACHE_SIZE", "0") monkeypatch.setenv("PSI_OMNI_HIDDEN_TENSOR_CACHE_SIZE", "0") sliced = omni_hidden_cache.load_hidden_states_slice( shard, offset=100, length=3, ) assert torch.equal(sliced, parent[100:103]) assert sliced.is_contiguous() assert sliced.untyped_storage().nbytes() == ( sliced.numel() * sliced.element_size() ) assert sliced.untyped_storage().nbytes() < ( parent.numel() * parent.element_size() ) omni_hidden_cache.close_cached_omni_shards() def test_compact_source_loads_all_windows_with_one_shared_storage(tmp_path): source_path = tmp_path / "voice.safetensors" hidden = torch.arange(32, dtype=torch.bfloat16).reshape(8, 4) save_file( { "hidden_states": hidden, "offsets": torch.tensor([0, 3, 8], dtype=torch.int64), "frame_ids": torch.tensor([0, 20], dtype=torch.int64), }, source_path, metadata={ "format": HFD_AUDIO_SOURCE_FORMAT, "encoder_contract": HFD_ONLINE_ENCODER_CONTRACT, "encoder_batch_size": str(HFD_ONLINE_ENCODER_BATCH_SIZE), "encoder_dtype": HFD_ONLINE_ENCODER_DTYPE, "attention_implementation": HFD_ONLINE_ENCODER_ATTENTION, "instruction_sha1": hashlib.sha1( AUDIO_COMMAND_STREAMING_PROMPT.encode("utf-8") ).hexdigest(), "count": "2", "window_seconds": "0.600000000", "step_frames": "20", "motion_fps": "50", "sampling_rate": "16000", }, ) frame_ids, windows = omni_hidden_cache.load_hidden_states_source( source_path ) assert frame_ids == (0, 20) assert len(windows) == 2 assert torch.equal(windows[0], hidden[:3]) assert torch.equal(windows[1], hidden[3:]) assert windows[0].untyped_storage().data_ptr() == ( windows[1].untyped_storage().data_ptr() ) assert windows[0].untyped_storage().nbytes() == ( hidden.numel() * hidden.element_size() ) def test_direct_oss_stage_materializes_and_removes_scratch(tmp_path, monkeypatch): omni_hidden_cache.close_cached_omni_shards() mount_root = tmp_path / "mounted" shard = mount_root / "tasks" / "text" / "shards" / "one.safetensors" shard.parent.mkdir(parents=True) expected = torch.arange(24, dtype=torch.bfloat16).reshape(6, 4) save_file({"hidden_states": expected}, shard) stage_root = tmp_path / "stage" monkeypatch.setenv("PSI_OMNI_DIRECT_OSS_STAGE", "1") monkeypatch.setenv("PSI_OMNI_DIRECT_OSS_MOUNT_ROOT", str(mount_root)) monkeypatch.setenv("PSI_OMNI_DIRECT_OSS_URI_ROOT", "oss://bucket/dataset") monkeypatch.setenv("PSI_OMNI_DIRECT_OSS_STAGE_DIR", str(stage_root)) monkeypatch.setenv("PSI_OMNI_OSSUTIL_BIN", "/fake/ossutil") monkeypatch.setenv("PSI_OMNI_OSSUTIL_PARALLEL", "17") monkeypatch.setenv("PSI_OMNI_FULL_SHARD_CACHE_SIZE", "1") monkeypatch.setenv("PSI_OMNI_FULL_SHARD_CACHE_MAX_GIB", "1") monkeypatch.setenv("PSI_OMNI_FULL_SHARD_IO_SLOTS", "0") observed = {} def fake_run(command, **kwargs): del kwargs observed["command"] = list(command) shutil.copyfile(shard, Path(command[-1])) class Result: returncode = 0 stdout = "" return Result() monkeypatch.setattr(omni_hidden_cache.subprocess, "run", fake_run) resident = omni_hidden_cache._get_full_shard_tensor(shard) assert resident is not None assert torch.equal(resident, expected) assert "--parallel" in observed["command"] parallel_index = observed["command"].index("--parallel") assert observed["command"][parallel_index + 1] == "17" assert observed["command"][-2] == ( "oss://bucket/dataset/tasks/text/shards/one.safetensors" ) assert list(stage_root.iterdir()) == [] omni_hidden_cache.close_cached_omni_shards() def test_concurrent_full_shard_requests_share_one_download(tmp_path, monkeypatch): omni_hidden_cache.close_cached_omni_shards() mount_root = tmp_path / "mounted" shard = mount_root / "tasks" / "video_clip_text" / "shards" / "one.safetensors" shard.parent.mkdir(parents=True) expected = torch.arange(48, dtype=torch.bfloat16).reshape(12, 4) save_file({"hidden_states": expected}, shard) stage_root = tmp_path / "stage" monkeypatch.setenv("PSI_OMNI_DIRECT_OSS_STAGE", "1") monkeypatch.setenv("PSI_OMNI_DIRECT_OSS_MOUNT_ROOT", str(mount_root)) monkeypatch.setenv("PSI_OMNI_DIRECT_OSS_URI_ROOT", "oss://bucket/dataset") monkeypatch.setenv("PSI_OMNI_DIRECT_OSS_STAGE_DIR", str(stage_root)) monkeypatch.setenv("PSI_OMNI_OSSUTIL_BIN", "/fake/ossutil") monkeypatch.setenv("PSI_OMNI_FULL_SHARD_CACHE_SIZE", "1") monkeypatch.setenv("PSI_OMNI_FULL_SHARD_CACHE_MAX_GIB", "1") monkeypatch.setenv("PSI_OMNI_FULL_SHARD_IO_SLOTS", "0") download_started = threading.Event() allow_download = threading.Event() run_count = 0 run_count_lock = threading.Lock() def fake_run(command, **kwargs): nonlocal run_count del kwargs with run_count_lock: run_count += 1 download_started.set() assert allow_download.wait(timeout=5) shutil.copyfile(shard, Path(command[-1])) class Result: returncode = 0 stdout = "" return Result() monkeypatch.setattr(omni_hidden_cache.subprocess, "run", fake_run) results = [] errors = [] def load(): try: results.append(omni_hidden_cache._get_full_shard_tensor(shard)) except BaseException as exc: errors.append(exc) first = threading.Thread(target=load) second = threading.Thread(target=load) first.start() assert download_started.wait(timeout=5) second.start() time.sleep(0.05) allow_download.set() first.join(timeout=5) second.join(timeout=5) assert not first.is_alive() assert not second.is_alive() assert errors == [] assert run_count == 1 assert len(results) == 2 assert all(result is not None and torch.equal(result, expected) for result in results) assert results[0].data_ptr() == results[1].data_ptr() assert list(stage_root.iterdir()) == [] omni_hidden_cache.close_cached_omni_shards()