Download tests/test_startup_index_cache.py from Tinghong-Ye22/code-HFD: direct link, hf CLI and curl.
- Browser
- Download file 5.09 kB
-
https://huggingface.co/Tinghong-Ye22/code-HFD/resolve/main/tests/test_startup_index_cache.py
- Command line
-
hf download hf://Tinghong-Ye22/code-HFD/tests/test_startup_index_cache.py
-
curl -L -o test_startup_index_cache.py https://huggingface.co/Tinghong-Ye22/code-HFD/resolve/main/tests/test_startup_index_cache.py
5.09 kB
| from __future__ import annotations | |
| import multiprocessing as mp | |
| import os | |
| from pathlib import Path | |
| import random | |
| import time | |
| import numpy as np | |
| import pytest | |
| import torch | |
| from psi.data.seed.startup_index_cache import cache_key, cached_index | |
| from psi.data.seed.full_duplex_audio_command_dataset import FullDuplexAudioCommandDataset | |
| from psi.data.seed.swa_visual_qa_dataset import SwaVisualQaDataset, _load_body | |
| def private_cache(tmp_path, monkeypatch): | |
| monkeypatch.setenv("HFD_STARTUP_INDEX_CACHE_DIR", str(tmp_path / "cache")) | |
| monkeypatch.setenv("HFD_STARTUP_INDEX_CACHE", "1") | |
| def test_key_covers_contents_order_independent_and_code(tmp_path): | |
| code = tmp_path / "code.py" | |
| code.write_text("original") | |
| a = {"settings": {"a": 1, "b": 2}, "array": np.arange(5), | |
| "set": frozenset({"listen", "idle"}), "path": Path("source")} | |
| b = dict(reversed(list(a.items()))) | |
| b["settings"] = {"b": 2, "a": 1} | |
| assert cache_key(a, (code,)) == cache_key(b, (code,)) | |
| key = cache_key(a, (code,)) | |
| a["array"][0] = 99 | |
| assert cache_key(a, (code,)) != key | |
| code.write_text("changed") | |
| assert cache_key(b, (code,)) != key | |
| def test_hit_corruption_and_opt_out(tmp_path, monkeypatch): | |
| calls = [] | |
| def build(): | |
| calls.append(1) | |
| return {"rows": (1, 2, 3)} | |
| assert cached_index("test", {"seed": 42}, build) == build_value() | |
| assert cached_index("test", {"seed": 42}, build) == build_value() | |
| assert len(calls) == 1 | |
| path = next((tmp_path / "cache").glob("test-*.pkl")) | |
| path.write_bytes(b"incomplete pickle") | |
| assert cached_index("test", {"seed": 42}, build) == build_value() | |
| assert len(calls) == 2 | |
| monkeypatch.setenv("HFD_STARTUP_INDEX_CACHE", "0") | |
| cached_index("test", {"seed": 42}, build) | |
| assert len(calls) == 3 | |
| def build_value(): | |
| return {"rows": (1, 2, 3)} | |
| def _concurrent_builder(queue, counter): | |
| def build(): | |
| with open(counter, "a") as file: | |
| file.write(f"{os.getpid()}\n") | |
| time.sleep(.2) | |
| return os.getpid() | |
| queue.put(cached_index("concurrent", {"records": [1, 2]}, build)) | |
| def test_concurrent_ranks_only_build_once(tmp_path): | |
| context = mp.get_context("fork") | |
| queue = context.Queue() | |
| counter = tmp_path / "builds" | |
| workers = [context.Process(target=_concurrent_builder, args=(queue, counter)) | |
| for _ in range(4)] | |
| for worker in workers: | |
| worker.start() | |
| results = [queue.get(timeout=15) for _ in workers] | |
| for worker in workers: | |
| worker.join(timeout=15) | |
| assert worker.exitcode == 0 | |
| assert len(set(results)) == 1 | |
| assert len(counter.read_text().splitlines()) == 1 | |
| assert not list((tmp_path / "cache").glob(".concurrent-*")) | |
| class TinySessionDataset(FullDuplexAudioCommandDataset): | |
| builds = 0 | |
| def __init__(self, seed=42): | |
| self.split = "train" | |
| self.split_seed = seed | |
| self.multi_turn_sessions = True | |
| self.records = [{"row_id": i, "frames": i + 30} for i in range(6)] | |
| self._motion_cache = {} | |
| self._prepare_session_index() | |
| def _build_session_index(self): | |
| type(self).builds += 1 | |
| rows = list(self.records) | |
| random.Random(self.split_seed).shuffle(rows) | |
| self.sessions = tuple(tuple(rows[i:i+2]) for i in range(0, 6, 2)) | |
| self.long_session_filter_summary = {"turn-taking": {"retained": 3}} | |
| self._session_length_cache = {id(row): 10 for row in self.sessions} | |
| return 0, 0 | |
| def _estimate_stream_steps(self, index): | |
| return sum(row["frames"] for row in self.sessions[index]) | |
| def test_session_cache_preserves_order_lengths_summaries_and_rng(): | |
| TinySessionDataset.builds = 0 | |
| rng = random.getstate() | |
| cold = TinySessionDataset() | |
| warm = TinySessionDataset() | |
| assert TinySessionDataset.builds == 1 | |
| assert random.getstate() == rng | |
| assert warm.sessions == cold.sessions | |
| assert warm.long_session_filter_summary == cold.long_session_filter_summary | |
| assert warm._session_length_cache == {} | |
| assert [warm.estimated_stream_steps(i) for i in range(len(warm))] == [ | |
| cold._estimate_stream_steps(i) for i in range(len(cold))] | |
| TinySessionDataset(seed=43) | |
| assert TinySessionDataset.builds == 2 | |
| def test_swa_length_index_avoids_reloading_bodies_and_invalidates(tmp_path, monkeypatch): | |
| import json | |
| body = tmp_path / "body.pt" | |
| torch.save({"source": {"steps": [{}, {}, {}]}}, body) | |
| plan = tmp_path / "plan.json" | |
| plan.write_text(json.dumps([{"split": "train", "path": str(body)}])) | |
| first = SwaVisualQaDataset(str(plan), "train", repeat=2) | |
| assert [first.estimated_stream_steps(i) for i in range(2)] == [3, 3] | |
| _load_body.cache_clear() | |
| second = SwaVisualQaDataset(str(plan), "train") | |
| assert second.estimated_stream_steps(0) == 3 | |
| assert _load_body.cache_info().misses == 0 | |
| _load_body(str(body)) | |
| torch.save({"source": {"steps": [{}] * 7}}, body) | |
| third = SwaVisualQaDataset(str(plan), "train") | |
| assert third.estimated_stream_steps(0) == 7 | |