code-HFD / src /psi /data /seed /omni_hidden_cache_test.py
Tinghong-Ye22's picture
Add files using upload-large-folder tool
06f9561 verified
Raw History Blame Contribute Delete
7.78 kB
from __future__ import annotations
import hashlib
import shutil
import threading
import time
from pathlib import Path
import torch
from safetensors.torch import save_file
from psi.data.seed.full_duplex_audio_contract import (
AUDIO_COMMAND_STREAMING_PROMPT,
HFD_AUDIO_SOURCE_FORMAT,
HFD_ONLINE_ENCODER_ATTENTION,
HFD_ONLINE_ENCODER_BATCH_SIZE,
HFD_ONLINE_ENCODER_CONTRACT,
HFD_ONLINE_ENCODER_DTYPE,
)
from psi.data.seed import omni_hidden_cache
def test_direct_oss_uri_keeps_path_relative_to_mount(tmp_path, monkeypatch):
mount_root = tmp_path / "mounted"
shard = mount_root / "tasks" / "text" / "shards" / "one.safetensors"
monkeypatch.setenv("PSI_OMNI_DIRECT_OSS_MOUNT_ROOT", str(mount_root))
monkeypatch.setenv("PSI_OMNI_DIRECT_OSS_URI_ROOT", "oss://bucket/dataset/")
assert omni_hidden_cache._direct_oss_uri(shard) == (
"oss://bucket/dataset/tasks/text/shards/one.safetensors"
)
def test_hidden_slice_does_not_retain_complete_safetensors_storage(
tmp_path, monkeypatch
):
omni_hidden_cache.close_cached_omni_shards()
shard = tmp_path / "large-parent.safetensors"
parent = torch.arange(
2_048 * 128,
dtype=torch.bfloat16,
).reshape(2_048, 128)
save_file({"hidden_states": parent}, shard)
monkeypatch.setenv("PSI_OMNI_SAFE_OPEN_CACHE_SIZE", "0")
monkeypatch.setenv("PSI_OMNI_FULL_SHARD_CACHE_SIZE", "0")
monkeypatch.setenv("PSI_OMNI_HIDDEN_TENSOR_CACHE_SIZE", "0")
sliced = omni_hidden_cache.load_hidden_states_slice(
shard,
offset=100,
length=3,
)
assert torch.equal(sliced, parent[100:103])
assert sliced.is_contiguous()
assert sliced.untyped_storage().nbytes() == (
sliced.numel() * sliced.element_size()
)
assert sliced.untyped_storage().nbytes() < (
parent.numel() * parent.element_size()
)
omni_hidden_cache.close_cached_omni_shards()
def test_compact_source_loads_all_windows_with_one_shared_storage(tmp_path):
source_path = tmp_path / "voice.safetensors"
hidden = torch.arange(32, dtype=torch.bfloat16).reshape(8, 4)
save_file(
{
"hidden_states": hidden,
"offsets": torch.tensor([0, 3, 8], dtype=torch.int64),
"frame_ids": torch.tensor([0, 20], dtype=torch.int64),
},
source_path,
metadata={
"format": HFD_AUDIO_SOURCE_FORMAT,
"encoder_contract": HFD_ONLINE_ENCODER_CONTRACT,
"encoder_batch_size": str(HFD_ONLINE_ENCODER_BATCH_SIZE),
"encoder_dtype": HFD_ONLINE_ENCODER_DTYPE,
"attention_implementation": HFD_ONLINE_ENCODER_ATTENTION,
"instruction_sha1": hashlib.sha1(
AUDIO_COMMAND_STREAMING_PROMPT.encode("utf-8")
).hexdigest(),
"count": "2",
"window_seconds": "0.600000000",
"step_frames": "20",
"motion_fps": "50",
"sampling_rate": "16000",
},
)
frame_ids, windows = omni_hidden_cache.load_hidden_states_source(
source_path
)
assert frame_ids == (0, 20)
assert len(windows) == 2
assert torch.equal(windows[0], hidden[:3])
assert torch.equal(windows[1], hidden[3:])
assert windows[0].untyped_storage().data_ptr() == (
windows[1].untyped_storage().data_ptr()
)
assert windows[0].untyped_storage().nbytes() == (
hidden.numel() * hidden.element_size()
)
def test_direct_oss_stage_materializes_and_removes_scratch(tmp_path, monkeypatch):
omni_hidden_cache.close_cached_omni_shards()
mount_root = tmp_path / "mounted"
shard = mount_root / "tasks" / "text" / "shards" / "one.safetensors"
shard.parent.mkdir(parents=True)
expected = torch.arange(24, dtype=torch.bfloat16).reshape(6, 4)
save_file({"hidden_states": expected}, shard)
stage_root = tmp_path / "stage"
monkeypatch.setenv("PSI_OMNI_DIRECT_OSS_STAGE", "1")
monkeypatch.setenv("PSI_OMNI_DIRECT_OSS_MOUNT_ROOT", str(mount_root))
monkeypatch.setenv("PSI_OMNI_DIRECT_OSS_URI_ROOT", "oss://bucket/dataset")
monkeypatch.setenv("PSI_OMNI_DIRECT_OSS_STAGE_DIR", str(stage_root))
monkeypatch.setenv("PSI_OMNI_OSSUTIL_BIN", "/fake/ossutil")
monkeypatch.setenv("PSI_OMNI_OSSUTIL_PARALLEL", "17")
monkeypatch.setenv("PSI_OMNI_FULL_SHARD_CACHE_SIZE", "1")
monkeypatch.setenv("PSI_OMNI_FULL_SHARD_CACHE_MAX_GIB", "1")
monkeypatch.setenv("PSI_OMNI_FULL_SHARD_IO_SLOTS", "0")
observed = {}
def fake_run(command, **kwargs):
del kwargs
observed["command"] = list(command)
shutil.copyfile(shard, Path(command[-1]))
class Result:
returncode = 0
stdout = ""
return Result()
monkeypatch.setattr(omni_hidden_cache.subprocess, "run", fake_run)
resident = omni_hidden_cache._get_full_shard_tensor(shard)
assert resident is not None
assert torch.equal(resident, expected)
assert "--parallel" in observed["command"]
parallel_index = observed["command"].index("--parallel")
assert observed["command"][parallel_index + 1] == "17"
assert observed["command"][-2] == (
"oss://bucket/dataset/tasks/text/shards/one.safetensors"
)
assert list(stage_root.iterdir()) == []
omni_hidden_cache.close_cached_omni_shards()
def test_concurrent_full_shard_requests_share_one_download(tmp_path, monkeypatch):
omni_hidden_cache.close_cached_omni_shards()
mount_root = tmp_path / "mounted"
shard = mount_root / "tasks" / "video_clip_text" / "shards" / "one.safetensors"
shard.parent.mkdir(parents=True)
expected = torch.arange(48, dtype=torch.bfloat16).reshape(12, 4)
save_file({"hidden_states": expected}, shard)
stage_root = tmp_path / "stage"
monkeypatch.setenv("PSI_OMNI_DIRECT_OSS_STAGE", "1")
monkeypatch.setenv("PSI_OMNI_DIRECT_OSS_MOUNT_ROOT", str(mount_root))
monkeypatch.setenv("PSI_OMNI_DIRECT_OSS_URI_ROOT", "oss://bucket/dataset")
monkeypatch.setenv("PSI_OMNI_DIRECT_OSS_STAGE_DIR", str(stage_root))
monkeypatch.setenv("PSI_OMNI_OSSUTIL_BIN", "/fake/ossutil")
monkeypatch.setenv("PSI_OMNI_FULL_SHARD_CACHE_SIZE", "1")
monkeypatch.setenv("PSI_OMNI_FULL_SHARD_CACHE_MAX_GIB", "1")
monkeypatch.setenv("PSI_OMNI_FULL_SHARD_IO_SLOTS", "0")
download_started = threading.Event()
allow_download = threading.Event()
run_count = 0
run_count_lock = threading.Lock()
def fake_run(command, **kwargs):
nonlocal run_count
del kwargs
with run_count_lock:
run_count += 1
download_started.set()
assert allow_download.wait(timeout=5)
shutil.copyfile(shard, Path(command[-1]))
class Result:
returncode = 0
stdout = ""
return Result()
monkeypatch.setattr(omni_hidden_cache.subprocess, "run", fake_run)
results = []
errors = []
def load():
try:
results.append(omni_hidden_cache._get_full_shard_tensor(shard))
except BaseException as exc:
errors.append(exc)
first = threading.Thread(target=load)
second = threading.Thread(target=load)
first.start()
assert download_started.wait(timeout=5)
second.start()
time.sleep(0.05)
allow_download.set()
first.join(timeout=5)
second.join(timeout=5)
assert not first.is_alive()
assert not second.is_alive()
assert errors == []
assert run_count == 1
assert len(results) == 2
assert all(result is not None and torch.equal(result, expected) for result in results)
assert results[0].data_ptr() == results[1].data_ptr()
assert list(stage_root.iterdir()) == []
omni_hidden_cache.close_cached_omni_shards()