from __future__ import annotations import multiprocessing as mp import os from pathlib import Path import random import time import numpy as np import pytest import torch from psi.data.seed.startup_index_cache import cache_key, cached_index from psi.data.seed.full_duplex_audio_command_dataset import FullDuplexAudioCommandDataset from psi.data.seed.swa_visual_qa_dataset import SwaVisualQaDataset, _load_body @pytest.fixture(autouse=True) def private_cache(tmp_path, monkeypatch): monkeypatch.setenv("HFD_STARTUP_INDEX_CACHE_DIR", str(tmp_path / "cache")) monkeypatch.setenv("HFD_STARTUP_INDEX_CACHE", "1") def test_key_covers_contents_order_independent_and_code(tmp_path): code = tmp_path / "code.py" code.write_text("original") a = {"settings": {"a": 1, "b": 2}, "array": np.arange(5), "set": frozenset({"listen", "idle"}), "path": Path("source")} b = dict(reversed(list(a.items()))) b["settings"] = {"b": 2, "a": 1} assert cache_key(a, (code,)) == cache_key(b, (code,)) key = cache_key(a, (code,)) a["array"][0] = 99 assert cache_key(a, (code,)) != key code.write_text("changed") assert cache_key(b, (code,)) != key def test_hit_corruption_and_opt_out(tmp_path, monkeypatch): calls = [] def build(): calls.append(1) return {"rows": (1, 2, 3)} assert cached_index("test", {"seed": 42}, build) == build_value() assert cached_index("test", {"seed": 42}, build) == build_value() assert len(calls) == 1 path = next((tmp_path / "cache").glob("test-*.pkl")) path.write_bytes(b"incomplete pickle") assert cached_index("test", {"seed": 42}, build) == build_value() assert len(calls) == 2 monkeypatch.setenv("HFD_STARTUP_INDEX_CACHE", "0") cached_index("test", {"seed": 42}, build) assert len(calls) == 3 def build_value(): return {"rows": (1, 2, 3)} def _concurrent_builder(queue, counter): def build(): with open(counter, "a") as file: file.write(f"{os.getpid()}\n") time.sleep(.2) return os.getpid() queue.put(cached_index("concurrent", {"records": [1, 2]}, build)) def test_concurrent_ranks_only_build_once(tmp_path): context = mp.get_context("fork") queue = context.Queue() counter = tmp_path / "builds" workers = [context.Process(target=_concurrent_builder, args=(queue, counter)) for _ in range(4)] for worker in workers: worker.start() results = [queue.get(timeout=15) for _ in workers] for worker in workers: worker.join(timeout=15) assert worker.exitcode == 0 assert len(set(results)) == 1 assert len(counter.read_text().splitlines()) == 1 assert not list((tmp_path / "cache").glob(".concurrent-*")) class TinySessionDataset(FullDuplexAudioCommandDataset): builds = 0 def __init__(self, seed=42): self.split = "train" self.split_seed = seed self.multi_turn_sessions = True self.records = [{"row_id": i, "frames": i + 30} for i in range(6)] self._motion_cache = {} self._prepare_session_index() def _build_session_index(self): type(self).builds += 1 rows = list(self.records) random.Random(self.split_seed).shuffle(rows) self.sessions = tuple(tuple(rows[i:i+2]) for i in range(0, 6, 2)) self.long_session_filter_summary = {"turn-taking": {"retained": 3}} self._session_length_cache = {id(row): 10 for row in self.sessions} return 0, 0 def _estimate_stream_steps(self, index): return sum(row["frames"] for row in self.sessions[index]) def test_session_cache_preserves_order_lengths_summaries_and_rng(): TinySessionDataset.builds = 0 rng = random.getstate() cold = TinySessionDataset() warm = TinySessionDataset() assert TinySessionDataset.builds == 1 assert random.getstate() == rng assert warm.sessions == cold.sessions assert warm.long_session_filter_summary == cold.long_session_filter_summary assert warm._session_length_cache == {} assert [warm.estimated_stream_steps(i) for i in range(len(warm))] == [ cold._estimate_stream_steps(i) for i in range(len(cold))] TinySessionDataset(seed=43) assert TinySessionDataset.builds == 2 def test_swa_length_index_avoids_reloading_bodies_and_invalidates(tmp_path, monkeypatch): import json body = tmp_path / "body.pt" torch.save({"source": {"steps": [{}, {}, {}]}}, body) plan = tmp_path / "plan.json" plan.write_text(json.dumps([{"split": "train", "path": str(body)}])) first = SwaVisualQaDataset(str(plan), "train", repeat=2) assert [first.estimated_stream_steps(i) for i in range(2)] == [3, 3] _load_body.cache_clear() second = SwaVisualQaDataset(str(plan), "train") assert second.estimated_stream_steps(0) == 3 assert _load_body.cache_info().misses == 0 _load_body(str(body)) torch.save({"source": {"steps": [{}] * 7}}, body) third = SwaVisualQaDataset(str(plan), "train") assert third.estimated_stream_steps(0) == 7