File size: 5,450 Bytes
aa02c28 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 | import importlib.util
from pathlib import Path
import sys
import pytest
SCRIPTS = Path(__file__).resolve().parents[1] / "scripts/data"
def load(name):
sys.path.insert(0, str(SCRIPTS))
spec = importlib.util.spec_from_file_location(name, SCRIPTS / f"{name}.py")
module = importlib.util.module_from_spec(spec)
spec.loader.exec_module(module)
return module
def queue(tmp_path, rows, **kw):
return load("swa_pipeline_queue").ready_batches(rows,
output_path=lambda r: tmp_path / "outputs" / f"{r}.json",
input_paths=lambda r: [tmp_path / "inputs" / f"{r}.json"], batch_size=2, **kw)
def publish(tmp_path, folder, names):
path = tmp_path / folder
path.mkdir(exist_ok=True)
for name in names:
(path / f"{name}.json").write_text("{}")
def test_consume_ready_before_upstream_finishes_and_resume(tmp_path):
publish(tmp_path, "inputs", [0, 1])
publish(tmp_path, "outputs", [0])
gen = queue(tmp_path, [0, 1, 2], follow=True, upstream_done=[tmp_path / "done"])
assert next(gen) == [1]
publish(tmp_path, "inputs", [2])
(tmp_path / "done").touch()
assert next(gen) == [2]
with pytest.raises(StopIteration):
next(gen)
def test_partial_receipt_is_never_consumed(tmp_path):
publish(tmp_path, "inputs", [1])
(tmp_path / "inputs/2.json.partial").write_text("{")
gen = queue(tmp_path, [1, 2])
assert next(gen) == [1]
with pytest.raises(RuntimeError, match="Upstream inputs missing"):
next(gen)
def test_finished_upstream_missing_receipt_fails_closed(tmp_path):
(tmp_path / "done").touch()
with pytest.raises(RuntimeError, match="Upstream inputs missing"):
list(queue(tmp_path, [1], follow=True, upstream_done=[tmp_path / "done"]))
def test_corrupt_receipt_not_silently_skipped(tmp_path):
# Queue ensures atomic publication; consumers must still parse/hash-check.
publish(tmp_path, "inputs", [1])
(tmp_path / "inputs/1.json").write_text("broken")
assert list(queue(tmp_path, [1])) == [[1]]
def test_two_independent_reviews_required(tmp_path):
m = load("swa_pipeline_queue")
publish(tmp_path, "asr", [1, 2])
publish(tmp_path, "visual", [1])
gen = m.ready_batches([1, 2], output_path=lambda r: tmp_path / "out" / str(r),
input_paths=lambda r: [tmp_path / "asr" / f"{r}.json", tmp_path / "visual" / f"{r}.json"],
batch_size=2, follow=True, upstream_done=[tmp_path / "done"])
assert next(gen) == [1]
publish(tmp_path, "visual", [2])
assert next(gen) == [2]
assert list(gen) == []
def test_timeout_without_producer_progress(tmp_path, monkeypatch):
m = load("swa_pipeline_queue")
ticks = iter([0., 2.])
monkeypatch.setattr(m.time, "monotonic", lambda: next(ticks))
with pytest.raises(TimeoutError):
list(m.ready_batches([1], output_path=lambda r: tmp_path / "out", input_paths=lambda r: [tmp_path / "missing"],
batch_size=1, follow=True, upstream_done=[tmp_path / "done"], timeout_seconds=1))
def test_follow_requires_completion_markers(tmp_path):
with pytest.raises(ValueError, match="completion markers"):
list(queue(tmp_path, [1], follow=True))
def test_duplicate_outputs_fail_closed(tmp_path):
with pytest.raises(ValueError, match="Duplicate"):
list(queue(tmp_path, [1, 1]))
def test_no_per_row_stat_storm(tmp_path, monkeypatch):
publish(tmp_path, "inputs", range(100))
def forbidden(*args, **kwargs):
raise AssertionError("Per-row stat would overload shared filesystem")
monkeypatch.setattr(Path, "exists", forbidden)
monkeypatch.setattr(Path, "is_file", forbidden)
assert len(list(queue(tmp_path, list(range(100))))) == 50
def test_dag_overlaps_independent_stages_without_global_barrier():
m = load("run_swa_vqav2_pipeline")
assert m.launchable(set(), set(), 0, True) == ["voices", "asr", "review", "encode"]
assert m.launchable(set(), set(), 0, False) == ["voices", "asr"]
assert m.launchable(set(), set(), 0, True, overlap_all=False) == ["voices", "asr"]
started = {("voices", 0), ("asr", 0)}
# GPU0 starts large downstream models before other TTS shards finish.
assert m.launchable({("voices", 0)}, started, 0, True, overlap_all=False) == ["review", "encode"]
assert m.launchable(set(), {("voices", 0), ("asr", 0), ("review", 0), ("encode", 0)}, 0, True) == []
def test_memory_preflight_rejects_conflicting_jobs(monkeypatch):
m = load("run_swa_vqav2_pipeline")
fractions = dict(voices=.45, asr=.04, review=.20, encode=.15)
monkeypatch.setattr(m.subprocess, "check_output", lambda *a, **kw: "143771, 125939\n")
m.check_memory(["0"], fractions, True)
monkeypatch.setattr(m.subprocess, "check_output", lambda *a, **kw: "143771, 10000\n")
with pytest.raises(RuntimeError, match="free.*budget"):
m.check_memory(["0"], fractions, True)
def test_streaming_and_serial_cover_same_rows_exactly_once(tmp_path):
rows = list(range(9))
publish(tmp_path, "inputs", [8, 0, 3])
gen = queue(tmp_path, rows, follow=True, upstream_done=[tmp_path / "done"])
streamed = next(gen) + next(gen)
publish(tmp_path, "inputs", [1, 2, 4, 5, 6, 7])
(tmp_path / "done").touch()
streamed += [r for batch in gen for r in batch]
serial = [r for batch in queue(tmp_path, rows) for r in batch]
assert sorted(streamed) == serial == rows
assert len(set(streamed)) == len(rows)
|