mamba-cot-tts / code /tests /test_lazy_fork_safety.py
maxpicy's picture
Add files using upload-large-folder tool
6e802a4 verified
Raw History Blame Contribute Delete
3.8 kB
# test_lazy_fork_safety.py: The lazy corpus reader is safe under forked workers.
import json
import os
import sys
from pathlib import Path
import pytest
REPO = Path(__file__).resolve().parent.parent
sys.path.insert(0, str(REPO))
dd = pytest.importorskip("delay_dataset")
pytestmark = pytest.mark.skipif(
not hasattr(os, "fork"), reason="fork-specific failure mode (POSIX only)")
class _FakeLazy:
def __init__(self, path, pid_guard):
self.path = str(path)
self.pid_guard = pid_guard
self._fh = None
self._fh_pid = None
self.offsets = []
with open(self.path, "rb") as f:
pos = f.tell()
for line in f:
if line.strip():
self.offsets.append(pos)
pos = f.tell()
def row(self, idx):
if self._fh is None or (self.pid_guard and self._fh_pid != os.getpid()):
self._fh = open(self.path, "rb")
self._fh_pid = os.getpid()
self._fh.seek(self.offsets[idx])
return json.loads(self._fh.readline())
def _corpus(tmp_path, n=400):
p = tmp_path / "corpus.jsonl"
with open(p, "w", encoding="utf-8") as f:
for i in range(n):
f.write(json.dumps({"row": i, "pad": "x" * (7 + (i * 13) % 400)}) + "\n")
return p
def _read_in_forks(ds, n_children, idxs):
ds.row(0)
pipes = [os.pipe() for _ in range(n_children)]
pids = []
for c, (r, w) in enumerate(pipes):
pid = os.fork()
if pid == 0:
code = 0
try:
os.close(r)
for k, idx in enumerate(idxs):
if k % n_children != c:
continue
got = ds.row(idx)
if got["row"] != idx:
code = 2
break
except Exception:
code = 1
finally:
try:
os.write(w, bytes([code]))
os.close(w)
except Exception:
pass
os._exit(0)
os.close(w)
pids.append(pid)
codes = []
for r, _ in pipes:
b = os.read(r, 1)
os.close(r)
codes.append(b[0] if b else 3)
for pid in pids:
os.waitpid(pid, 0)
return codes
def test_offset_index_is_correct(tmp_path):
p = _corpus(tmp_path, n=50)
ds = _FakeLazy(p, pid_guard=True)
assert len(ds.offsets) == 50
for i in (0, 1, 17, 49):
assert ds.row(i)["row"] == i
def test_shared_handle_across_forks_corrupts_reads(tmp_path):
p = _corpus(tmp_path, n=400)
ds = _FakeLazy(p, pid_guard=False)
codes = _read_in_forks(ds, n_children=4, idxs=list(range(400)))
assert any(c != 0 for c in codes), (
"expected corruption from a shared file descriptor; if this passes, "
"the reproduction no longer exercises the mechanism and the guard "
"below is no longer proven to be load-bearing")
def test_pid_guard_makes_forked_reads_correct(tmp_path):
p = _corpus(tmp_path, n=400)
ds = _FakeLazy(p, pid_guard=True)
codes = _read_in_forks(ds, n_children=4, idxs=list(range(400)))
assert all(c == 0 for c in codes), f"child exit codes {codes}"
def test_dataset_declares_fh_pid():
src = (REPO / "delay_dataset.py").read_text(encoding="utf-8")
assert "_fh_pid" in src
assert "self._fh_pid != os.getpid()" in src, \
"the reopen condition must compare the OWNING pid, not just None"
def test_getstate_drops_both_handle_and_pid():
src = (REPO / "delay_dataset.py").read_text(encoding="utf-8")
i = src.index("def __getstate__")
body = src[i:i + 400]
assert 'state["_fh"] = None' in body
assert 'state["_fh_pid"] = None' in body