Download code/tests/test_lazy_fork_safety.py from maxpicy/mamba-cot-tts: direct link, hf CLI and curl.
- Browser
- Download file 3.8 kB
-
https://huggingface.co/maxpicy/mamba-cot-tts/resolve/main/code/tests/test_lazy_fork_safety.py
- Command line
-
hf download hf://maxpicy/mamba-cot-tts/code/tests/test_lazy_fork_safety.py
-
curl -L -o test_lazy_fork_safety.py https://huggingface.co/maxpicy/mamba-cot-tts/resolve/main/code/tests/test_lazy_fork_safety.py
3.8 kB
| # test_lazy_fork_safety.py: The lazy corpus reader is safe under forked workers. | |
| import json | |
| import os | |
| import sys | |
| from pathlib import Path | |
| import pytest | |
| REPO = Path(__file__).resolve().parent.parent | |
| sys.path.insert(0, str(REPO)) | |
| dd = pytest.importorskip("delay_dataset") | |
| pytestmark = pytest.mark.skipif( | |
| not hasattr(os, "fork"), reason="fork-specific failure mode (POSIX only)") | |
| class _FakeLazy: | |
| def __init__(self, path, pid_guard): | |
| self.path = str(path) | |
| self.pid_guard = pid_guard | |
| self._fh = None | |
| self._fh_pid = None | |
| self.offsets = [] | |
| with open(self.path, "rb") as f: | |
| pos = f.tell() | |
| for line in f: | |
| if line.strip(): | |
| self.offsets.append(pos) | |
| pos = f.tell() | |
| def row(self, idx): | |
| if self._fh is None or (self.pid_guard and self._fh_pid != os.getpid()): | |
| self._fh = open(self.path, "rb") | |
| self._fh_pid = os.getpid() | |
| self._fh.seek(self.offsets[idx]) | |
| return json.loads(self._fh.readline()) | |
| def _corpus(tmp_path, n=400): | |
| p = tmp_path / "corpus.jsonl" | |
| with open(p, "w", encoding="utf-8") as f: | |
| for i in range(n): | |
| f.write(json.dumps({"row": i, "pad": "x" * (7 + (i * 13) % 400)}) + "\n") | |
| return p | |
| def _read_in_forks(ds, n_children, idxs): | |
| ds.row(0) | |
| pipes = [os.pipe() for _ in range(n_children)] | |
| pids = [] | |
| for c, (r, w) in enumerate(pipes): | |
| pid = os.fork() | |
| if pid == 0: | |
| code = 0 | |
| try: | |
| os.close(r) | |
| for k, idx in enumerate(idxs): | |
| if k % n_children != c: | |
| continue | |
| got = ds.row(idx) | |
| if got["row"] != idx: | |
| code = 2 | |
| break | |
| except Exception: | |
| code = 1 | |
| finally: | |
| try: | |
| os.write(w, bytes([code])) | |
| os.close(w) | |
| except Exception: | |
| pass | |
| os._exit(0) | |
| os.close(w) | |
| pids.append(pid) | |
| codes = [] | |
| for r, _ in pipes: | |
| b = os.read(r, 1) | |
| os.close(r) | |
| codes.append(b[0] if b else 3) | |
| for pid in pids: | |
| os.waitpid(pid, 0) | |
| return codes | |
| def test_offset_index_is_correct(tmp_path): | |
| p = _corpus(tmp_path, n=50) | |
| ds = _FakeLazy(p, pid_guard=True) | |
| assert len(ds.offsets) == 50 | |
| for i in (0, 1, 17, 49): | |
| assert ds.row(i)["row"] == i | |
| def test_shared_handle_across_forks_corrupts_reads(tmp_path): | |
| p = _corpus(tmp_path, n=400) | |
| ds = _FakeLazy(p, pid_guard=False) | |
| codes = _read_in_forks(ds, n_children=4, idxs=list(range(400))) | |
| assert any(c != 0 for c in codes), ( | |
| "expected corruption from a shared file descriptor; if this passes, " | |
| "the reproduction no longer exercises the mechanism and the guard " | |
| "below is no longer proven to be load-bearing") | |
| def test_pid_guard_makes_forked_reads_correct(tmp_path): | |
| p = _corpus(tmp_path, n=400) | |
| ds = _FakeLazy(p, pid_guard=True) | |
| codes = _read_in_forks(ds, n_children=4, idxs=list(range(400))) | |
| assert all(c == 0 for c in codes), f"child exit codes {codes}" | |
| def test_dataset_declares_fh_pid(): | |
| src = (REPO / "delay_dataset.py").read_text(encoding="utf-8") | |
| assert "_fh_pid" in src | |
| assert "self._fh_pid != os.getpid()" in src, \ | |
| "the reopen condition must compare the OWNING pid, not just None" | |
| def test_getstate_drops_both_handle_and_pid(): | |
| src = (REPO / "delay_dataset.py").read_text(encoding="utf-8") | |
| i = src.index("def __getstate__") | |
| body = src[i:i + 400] | |
| assert 'state["_fh"] = None' in body | |
| assert 'state["_fh_pid"] = None' in body | |