code-HFD / tests /test_startup_index_cache.py
Tinghong-Ye22's picture
Add files using upload-large-folder tool
5d9f970 verified
Raw History Blame Contribute Delete
5.09 kB
from __future__ import annotations
import multiprocessing as mp
import os
from pathlib import Path
import random
import time
import numpy as np
import pytest
import torch
from psi.data.seed.startup_index_cache import cache_key, cached_index
from psi.data.seed.full_duplex_audio_command_dataset import FullDuplexAudioCommandDataset
from psi.data.seed.swa_visual_qa_dataset import SwaVisualQaDataset, _load_body
@pytest.fixture(autouse=True)
def private_cache(tmp_path, monkeypatch):
monkeypatch.setenv("HFD_STARTUP_INDEX_CACHE_DIR", str(tmp_path / "cache"))
monkeypatch.setenv("HFD_STARTUP_INDEX_CACHE", "1")
def test_key_covers_contents_order_independent_and_code(tmp_path):
code = tmp_path / "code.py"
code.write_text("original")
a = {"settings": {"a": 1, "b": 2}, "array": np.arange(5),
"set": frozenset({"listen", "idle"}), "path": Path("source")}
b = dict(reversed(list(a.items())))
b["settings"] = {"b": 2, "a": 1}
assert cache_key(a, (code,)) == cache_key(b, (code,))
key = cache_key(a, (code,))
a["array"][0] = 99
assert cache_key(a, (code,)) != key
code.write_text("changed")
assert cache_key(b, (code,)) != key
def test_hit_corruption_and_opt_out(tmp_path, monkeypatch):
calls = []
def build():
calls.append(1)
return {"rows": (1, 2, 3)}
assert cached_index("test", {"seed": 42}, build) == build_value()
assert cached_index("test", {"seed": 42}, build) == build_value()
assert len(calls) == 1
path = next((tmp_path / "cache").glob("test-*.pkl"))
path.write_bytes(b"incomplete pickle")
assert cached_index("test", {"seed": 42}, build) == build_value()
assert len(calls) == 2
monkeypatch.setenv("HFD_STARTUP_INDEX_CACHE", "0")
cached_index("test", {"seed": 42}, build)
assert len(calls) == 3
def build_value():
return {"rows": (1, 2, 3)}
def _concurrent_builder(queue, counter):
def build():
with open(counter, "a") as file:
file.write(f"{os.getpid()}\n")
time.sleep(.2)
return os.getpid()
queue.put(cached_index("concurrent", {"records": [1, 2]}, build))
def test_concurrent_ranks_only_build_once(tmp_path):
context = mp.get_context("fork")
queue = context.Queue()
counter = tmp_path / "builds"
workers = [context.Process(target=_concurrent_builder, args=(queue, counter))
for _ in range(4)]
for worker in workers:
worker.start()
results = [queue.get(timeout=15) for _ in workers]
for worker in workers:
worker.join(timeout=15)
assert worker.exitcode == 0
assert len(set(results)) == 1
assert len(counter.read_text().splitlines()) == 1
assert not list((tmp_path / "cache").glob(".concurrent-*"))
class TinySessionDataset(FullDuplexAudioCommandDataset):
builds = 0
def __init__(self, seed=42):
self.split = "train"
self.split_seed = seed
self.multi_turn_sessions = True
self.records = [{"row_id": i, "frames": i + 30} for i in range(6)]
self._motion_cache = {}
self._prepare_session_index()
def _build_session_index(self):
type(self).builds += 1
rows = list(self.records)
random.Random(self.split_seed).shuffle(rows)
self.sessions = tuple(tuple(rows[i:i+2]) for i in range(0, 6, 2))
self.long_session_filter_summary = {"turn-taking": {"retained": 3}}
self._session_length_cache = {id(row): 10 for row in self.sessions}
return 0, 0
def _estimate_stream_steps(self, index):
return sum(row["frames"] for row in self.sessions[index])
def test_session_cache_preserves_order_lengths_summaries_and_rng():
TinySessionDataset.builds = 0
rng = random.getstate()
cold = TinySessionDataset()
warm = TinySessionDataset()
assert TinySessionDataset.builds == 1
assert random.getstate() == rng
assert warm.sessions == cold.sessions
assert warm.long_session_filter_summary == cold.long_session_filter_summary
assert warm._session_length_cache == {}
assert [warm.estimated_stream_steps(i) for i in range(len(warm))] == [
cold._estimate_stream_steps(i) for i in range(len(cold))]
TinySessionDataset(seed=43)
assert TinySessionDataset.builds == 2
def test_swa_length_index_avoids_reloading_bodies_and_invalidates(tmp_path, monkeypatch):
import json
body = tmp_path / "body.pt"
torch.save({"source": {"steps": [{}, {}, {}]}}, body)
plan = tmp_path / "plan.json"
plan.write_text(json.dumps([{"split": "train", "path": str(body)}]))
first = SwaVisualQaDataset(str(plan), "train", repeat=2)
assert [first.estimated_stream_steps(i) for i in range(2)] == [3, 3]
_load_body.cache_clear()
second = SwaVisualQaDataset(str(plan), "train")
assert second.estimated_stream_steps(0) == 3
assert _load_body.cache_info().misses == 0
_load_body(str(body))
torch.save({"source": {"steps": [{}] * 7}}, body)
third = SwaVisualQaDataset(str(plan), "train")
assert third.estimated_stream_steps(0) == 7