File size: 5,087 Bytes
5d9f970 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 | from __future__ import annotations
import multiprocessing as mp
import os
from pathlib import Path
import random
import time
import numpy as np
import pytest
import torch
from psi.data.seed.startup_index_cache import cache_key, cached_index
from psi.data.seed.full_duplex_audio_command_dataset import FullDuplexAudioCommandDataset
from psi.data.seed.swa_visual_qa_dataset import SwaVisualQaDataset, _load_body
@pytest.fixture(autouse=True)
def private_cache(tmp_path, monkeypatch):
monkeypatch.setenv("HFD_STARTUP_INDEX_CACHE_DIR", str(tmp_path / "cache"))
monkeypatch.setenv("HFD_STARTUP_INDEX_CACHE", "1")
def test_key_covers_contents_order_independent_and_code(tmp_path):
code = tmp_path / "code.py"
code.write_text("original")
a = {"settings": {"a": 1, "b": 2}, "array": np.arange(5),
"set": frozenset({"listen", "idle"}), "path": Path("source")}
b = dict(reversed(list(a.items())))
b["settings"] = {"b": 2, "a": 1}
assert cache_key(a, (code,)) == cache_key(b, (code,))
key = cache_key(a, (code,))
a["array"][0] = 99
assert cache_key(a, (code,)) != key
code.write_text("changed")
assert cache_key(b, (code,)) != key
def test_hit_corruption_and_opt_out(tmp_path, monkeypatch):
calls = []
def build():
calls.append(1)
return {"rows": (1, 2, 3)}
assert cached_index("test", {"seed": 42}, build) == build_value()
assert cached_index("test", {"seed": 42}, build) == build_value()
assert len(calls) == 1
path = next((tmp_path / "cache").glob("test-*.pkl"))
path.write_bytes(b"incomplete pickle")
assert cached_index("test", {"seed": 42}, build) == build_value()
assert len(calls) == 2
monkeypatch.setenv("HFD_STARTUP_INDEX_CACHE", "0")
cached_index("test", {"seed": 42}, build)
assert len(calls) == 3
def build_value():
return {"rows": (1, 2, 3)}
def _concurrent_builder(queue, counter):
def build():
with open(counter, "a") as file:
file.write(f"{os.getpid()}\n")
time.sleep(.2)
return os.getpid()
queue.put(cached_index("concurrent", {"records": [1, 2]}, build))
def test_concurrent_ranks_only_build_once(tmp_path):
context = mp.get_context("fork")
queue = context.Queue()
counter = tmp_path / "builds"
workers = [context.Process(target=_concurrent_builder, args=(queue, counter))
for _ in range(4)]
for worker in workers:
worker.start()
results = [queue.get(timeout=15) for _ in workers]
for worker in workers:
worker.join(timeout=15)
assert worker.exitcode == 0
assert len(set(results)) == 1
assert len(counter.read_text().splitlines()) == 1
assert not list((tmp_path / "cache").glob(".concurrent-*"))
class TinySessionDataset(FullDuplexAudioCommandDataset):
builds = 0
def __init__(self, seed=42):
self.split = "train"
self.split_seed = seed
self.multi_turn_sessions = True
self.records = [{"row_id": i, "frames": i + 30} for i in range(6)]
self._motion_cache = {}
self._prepare_session_index()
def _build_session_index(self):
type(self).builds += 1
rows = list(self.records)
random.Random(self.split_seed).shuffle(rows)
self.sessions = tuple(tuple(rows[i:i+2]) for i in range(0, 6, 2))
self.long_session_filter_summary = {"turn-taking": {"retained": 3}}
self._session_length_cache = {id(row): 10 for row in self.sessions}
return 0, 0
def _estimate_stream_steps(self, index):
return sum(row["frames"] for row in self.sessions[index])
def test_session_cache_preserves_order_lengths_summaries_and_rng():
TinySessionDataset.builds = 0
rng = random.getstate()
cold = TinySessionDataset()
warm = TinySessionDataset()
assert TinySessionDataset.builds == 1
assert random.getstate() == rng
assert warm.sessions == cold.sessions
assert warm.long_session_filter_summary == cold.long_session_filter_summary
assert warm._session_length_cache == {}
assert [warm.estimated_stream_steps(i) for i in range(len(warm))] == [
cold._estimate_stream_steps(i) for i in range(len(cold))]
TinySessionDataset(seed=43)
assert TinySessionDataset.builds == 2
def test_swa_length_index_avoids_reloading_bodies_and_invalidates(tmp_path, monkeypatch):
import json
body = tmp_path / "body.pt"
torch.save({"source": {"steps": [{}, {}, {}]}}, body)
plan = tmp_path / "plan.json"
plan.write_text(json.dumps([{"split": "train", "path": str(body)}]))
first = SwaVisualQaDataset(str(plan), "train", repeat=2)
assert [first.estimated_stream_steps(i) for i in range(2)] == [3, 3]
_load_body.cache_clear()
second = SwaVisualQaDataset(str(plan), "train")
assert second.estimated_stream_steps(0) == 3
assert _load_body.cache_info().misses == 0
_load_body(str(body))
torch.save({"source": {"steps": [{}] * 7}}, body)
third = SwaVisualQaDataset(str(plan), "train")
assert third.estimated_stream_steps(0) == 7
|