Download src/psi/data/seed/omni_hidden_cache_test.py from Tinghong-Ye22/code-HFD: direct link, hf CLI and curl.
- Browser
- Download file 7.78 kB
-
https://huggingface.co/Tinghong-Ye22/code-HFD/resolve/main/src/psi/data/seed/omni_hidden_cache_test.py
- Command line
-
hf download hf://Tinghong-Ye22/code-HFD/src/psi/data/seed/omni_hidden_cache_test.py
-
curl -L -o omni_hidden_cache_test.py https://huggingface.co/Tinghong-Ye22/code-HFD/resolve/main/src/psi/data/seed/omni_hidden_cache_test.py
7.78 kB
| from __future__ import annotations | |
| import hashlib | |
| import shutil | |
| import threading | |
| import time | |
| from pathlib import Path | |
| import torch | |
| from safetensors.torch import save_file | |
| from psi.data.seed.full_duplex_audio_contract import ( | |
| AUDIO_COMMAND_STREAMING_PROMPT, | |
| HFD_AUDIO_SOURCE_FORMAT, | |
| HFD_ONLINE_ENCODER_ATTENTION, | |
| HFD_ONLINE_ENCODER_BATCH_SIZE, | |
| HFD_ONLINE_ENCODER_CONTRACT, | |
| HFD_ONLINE_ENCODER_DTYPE, | |
| ) | |
| from psi.data.seed import omni_hidden_cache | |
| def test_direct_oss_uri_keeps_path_relative_to_mount(tmp_path, monkeypatch): | |
| mount_root = tmp_path / "mounted" | |
| shard = mount_root / "tasks" / "text" / "shards" / "one.safetensors" | |
| monkeypatch.setenv("PSI_OMNI_DIRECT_OSS_MOUNT_ROOT", str(mount_root)) | |
| monkeypatch.setenv("PSI_OMNI_DIRECT_OSS_URI_ROOT", "oss://bucket/dataset/") | |
| assert omni_hidden_cache._direct_oss_uri(shard) == ( | |
| "oss://bucket/dataset/tasks/text/shards/one.safetensors" | |
| ) | |
| def test_hidden_slice_does_not_retain_complete_safetensors_storage( | |
| tmp_path, monkeypatch | |
| ): | |
| omni_hidden_cache.close_cached_omni_shards() | |
| shard = tmp_path / "large-parent.safetensors" | |
| parent = torch.arange( | |
| 2_048 * 128, | |
| dtype=torch.bfloat16, | |
| ).reshape(2_048, 128) | |
| save_file({"hidden_states": parent}, shard) | |
| monkeypatch.setenv("PSI_OMNI_SAFE_OPEN_CACHE_SIZE", "0") | |
| monkeypatch.setenv("PSI_OMNI_FULL_SHARD_CACHE_SIZE", "0") | |
| monkeypatch.setenv("PSI_OMNI_HIDDEN_TENSOR_CACHE_SIZE", "0") | |
| sliced = omni_hidden_cache.load_hidden_states_slice( | |
| shard, | |
| offset=100, | |
| length=3, | |
| ) | |
| assert torch.equal(sliced, parent[100:103]) | |
| assert sliced.is_contiguous() | |
| assert sliced.untyped_storage().nbytes() == ( | |
| sliced.numel() * sliced.element_size() | |
| ) | |
| assert sliced.untyped_storage().nbytes() < ( | |
| parent.numel() * parent.element_size() | |
| ) | |
| omni_hidden_cache.close_cached_omni_shards() | |
| def test_compact_source_loads_all_windows_with_one_shared_storage(tmp_path): | |
| source_path = tmp_path / "voice.safetensors" | |
| hidden = torch.arange(32, dtype=torch.bfloat16).reshape(8, 4) | |
| save_file( | |
| { | |
| "hidden_states": hidden, | |
| "offsets": torch.tensor([0, 3, 8], dtype=torch.int64), | |
| "frame_ids": torch.tensor([0, 20], dtype=torch.int64), | |
| }, | |
| source_path, | |
| metadata={ | |
| "format": HFD_AUDIO_SOURCE_FORMAT, | |
| "encoder_contract": HFD_ONLINE_ENCODER_CONTRACT, | |
| "encoder_batch_size": str(HFD_ONLINE_ENCODER_BATCH_SIZE), | |
| "encoder_dtype": HFD_ONLINE_ENCODER_DTYPE, | |
| "attention_implementation": HFD_ONLINE_ENCODER_ATTENTION, | |
| "instruction_sha1": hashlib.sha1( | |
| AUDIO_COMMAND_STREAMING_PROMPT.encode("utf-8") | |
| ).hexdigest(), | |
| "count": "2", | |
| "window_seconds": "0.600000000", | |
| "step_frames": "20", | |
| "motion_fps": "50", | |
| "sampling_rate": "16000", | |
| }, | |
| ) | |
| frame_ids, windows = omni_hidden_cache.load_hidden_states_source( | |
| source_path | |
| ) | |
| assert frame_ids == (0, 20) | |
| assert len(windows) == 2 | |
| assert torch.equal(windows[0], hidden[:3]) | |
| assert torch.equal(windows[1], hidden[3:]) | |
| assert windows[0].untyped_storage().data_ptr() == ( | |
| windows[1].untyped_storage().data_ptr() | |
| ) | |
| assert windows[0].untyped_storage().nbytes() == ( | |
| hidden.numel() * hidden.element_size() | |
| ) | |
| def test_direct_oss_stage_materializes_and_removes_scratch(tmp_path, monkeypatch): | |
| omni_hidden_cache.close_cached_omni_shards() | |
| mount_root = tmp_path / "mounted" | |
| shard = mount_root / "tasks" / "text" / "shards" / "one.safetensors" | |
| shard.parent.mkdir(parents=True) | |
| expected = torch.arange(24, dtype=torch.bfloat16).reshape(6, 4) | |
| save_file({"hidden_states": expected}, shard) | |
| stage_root = tmp_path / "stage" | |
| monkeypatch.setenv("PSI_OMNI_DIRECT_OSS_STAGE", "1") | |
| monkeypatch.setenv("PSI_OMNI_DIRECT_OSS_MOUNT_ROOT", str(mount_root)) | |
| monkeypatch.setenv("PSI_OMNI_DIRECT_OSS_URI_ROOT", "oss://bucket/dataset") | |
| monkeypatch.setenv("PSI_OMNI_DIRECT_OSS_STAGE_DIR", str(stage_root)) | |
| monkeypatch.setenv("PSI_OMNI_OSSUTIL_BIN", "/fake/ossutil") | |
| monkeypatch.setenv("PSI_OMNI_OSSUTIL_PARALLEL", "17") | |
| monkeypatch.setenv("PSI_OMNI_FULL_SHARD_CACHE_SIZE", "1") | |
| monkeypatch.setenv("PSI_OMNI_FULL_SHARD_CACHE_MAX_GIB", "1") | |
| monkeypatch.setenv("PSI_OMNI_FULL_SHARD_IO_SLOTS", "0") | |
| observed = {} | |
| def fake_run(command, **kwargs): | |
| del kwargs | |
| observed["command"] = list(command) | |
| shutil.copyfile(shard, Path(command[-1])) | |
| class Result: | |
| returncode = 0 | |
| stdout = "" | |
| return Result() | |
| monkeypatch.setattr(omni_hidden_cache.subprocess, "run", fake_run) | |
| resident = omni_hidden_cache._get_full_shard_tensor(shard) | |
| assert resident is not None | |
| assert torch.equal(resident, expected) | |
| assert "--parallel" in observed["command"] | |
| parallel_index = observed["command"].index("--parallel") | |
| assert observed["command"][parallel_index + 1] == "17" | |
| assert observed["command"][-2] == ( | |
| "oss://bucket/dataset/tasks/text/shards/one.safetensors" | |
| ) | |
| assert list(stage_root.iterdir()) == [] | |
| omni_hidden_cache.close_cached_omni_shards() | |
| def test_concurrent_full_shard_requests_share_one_download(tmp_path, monkeypatch): | |
| omni_hidden_cache.close_cached_omni_shards() | |
| mount_root = tmp_path / "mounted" | |
| shard = mount_root / "tasks" / "video_clip_text" / "shards" / "one.safetensors" | |
| shard.parent.mkdir(parents=True) | |
| expected = torch.arange(48, dtype=torch.bfloat16).reshape(12, 4) | |
| save_file({"hidden_states": expected}, shard) | |
| stage_root = tmp_path / "stage" | |
| monkeypatch.setenv("PSI_OMNI_DIRECT_OSS_STAGE", "1") | |
| monkeypatch.setenv("PSI_OMNI_DIRECT_OSS_MOUNT_ROOT", str(mount_root)) | |
| monkeypatch.setenv("PSI_OMNI_DIRECT_OSS_URI_ROOT", "oss://bucket/dataset") | |
| monkeypatch.setenv("PSI_OMNI_DIRECT_OSS_STAGE_DIR", str(stage_root)) | |
| monkeypatch.setenv("PSI_OMNI_OSSUTIL_BIN", "/fake/ossutil") | |
| monkeypatch.setenv("PSI_OMNI_FULL_SHARD_CACHE_SIZE", "1") | |
| monkeypatch.setenv("PSI_OMNI_FULL_SHARD_CACHE_MAX_GIB", "1") | |
| monkeypatch.setenv("PSI_OMNI_FULL_SHARD_IO_SLOTS", "0") | |
| download_started = threading.Event() | |
| allow_download = threading.Event() | |
| run_count = 0 | |
| run_count_lock = threading.Lock() | |
| def fake_run(command, **kwargs): | |
| nonlocal run_count | |
| del kwargs | |
| with run_count_lock: | |
| run_count += 1 | |
| download_started.set() | |
| assert allow_download.wait(timeout=5) | |
| shutil.copyfile(shard, Path(command[-1])) | |
| class Result: | |
| returncode = 0 | |
| stdout = "" | |
| return Result() | |
| monkeypatch.setattr(omni_hidden_cache.subprocess, "run", fake_run) | |
| results = [] | |
| errors = [] | |
| def load(): | |
| try: | |
| results.append(omni_hidden_cache._get_full_shard_tensor(shard)) | |
| except BaseException as exc: | |
| errors.append(exc) | |
| first = threading.Thread(target=load) | |
| second = threading.Thread(target=load) | |
| first.start() | |
| assert download_started.wait(timeout=5) | |
| second.start() | |
| time.sleep(0.05) | |
| allow_download.set() | |
| first.join(timeout=5) | |
| second.join(timeout=5) | |
| assert not first.is_alive() | |
| assert not second.is_alive() | |
| assert errors == [] | |
| assert run_count == 1 | |
| assert len(results) == 2 | |
| assert all(result is not None and torch.equal(result, expected) for result in results) | |
| assert results[0].data_ptr() == results[1].data_ptr() | |
| assert list(stage_root.iterdir()) == [] | |
| omni_hidden_cache.close_cached_omni_shards() | |