code-HFD / tests /test_swa_pipeline_queue.py
Tinghong-Ye22's picture
Add files using upload-large-folder tool
aa02c28 verified
Raw History Blame Contribute Delete
5.45 kB
import importlib.util
from pathlib import Path
import sys
import pytest
SCRIPTS = Path(__file__).resolve().parents[1] / "scripts/data"
def load(name):
sys.path.insert(0, str(SCRIPTS))
spec = importlib.util.spec_from_file_location(name, SCRIPTS / f"{name}.py")
module = importlib.util.module_from_spec(spec)
spec.loader.exec_module(module)
return module
def queue(tmp_path, rows, **kw):
return load("swa_pipeline_queue").ready_batches(rows,
output_path=lambda r: tmp_path / "outputs" / f"{r}.json",
input_paths=lambda r: [tmp_path / "inputs" / f"{r}.json"], batch_size=2, **kw)
def publish(tmp_path, folder, names):
path = tmp_path / folder
path.mkdir(exist_ok=True)
for name in names:
(path / f"{name}.json").write_text("{}")
def test_consume_ready_before_upstream_finishes_and_resume(tmp_path):
publish(tmp_path, "inputs", [0, 1])
publish(tmp_path, "outputs", [0])
gen = queue(tmp_path, [0, 1, 2], follow=True, upstream_done=[tmp_path / "done"])
assert next(gen) == [1]
publish(tmp_path, "inputs", [2])
(tmp_path / "done").touch()
assert next(gen) == [2]
with pytest.raises(StopIteration):
next(gen)
def test_partial_receipt_is_never_consumed(tmp_path):
publish(tmp_path, "inputs", [1])
(tmp_path / "inputs/2.json.partial").write_text("{")
gen = queue(tmp_path, [1, 2])
assert next(gen) == [1]
with pytest.raises(RuntimeError, match="Upstream inputs missing"):
next(gen)
def test_finished_upstream_missing_receipt_fails_closed(tmp_path):
(tmp_path / "done").touch()
with pytest.raises(RuntimeError, match="Upstream inputs missing"):
list(queue(tmp_path, [1], follow=True, upstream_done=[tmp_path / "done"]))
def test_corrupt_receipt_not_silently_skipped(tmp_path):
# Queue ensures atomic publication; consumers must still parse/hash-check.
publish(tmp_path, "inputs", [1])
(tmp_path / "inputs/1.json").write_text("broken")
assert list(queue(tmp_path, [1])) == [[1]]
def test_two_independent_reviews_required(tmp_path):
m = load("swa_pipeline_queue")
publish(tmp_path, "asr", [1, 2])
publish(tmp_path, "visual", [1])
gen = m.ready_batches([1, 2], output_path=lambda r: tmp_path / "out" / str(r),
input_paths=lambda r: [tmp_path / "asr" / f"{r}.json", tmp_path / "visual" / f"{r}.json"],
batch_size=2, follow=True, upstream_done=[tmp_path / "done"])
assert next(gen) == [1]
publish(tmp_path, "visual", [2])
assert next(gen) == [2]
assert list(gen) == []
def test_timeout_without_producer_progress(tmp_path, monkeypatch):
m = load("swa_pipeline_queue")
ticks = iter([0., 2.])
monkeypatch.setattr(m.time, "monotonic", lambda: next(ticks))
with pytest.raises(TimeoutError):
list(m.ready_batches([1], output_path=lambda r: tmp_path / "out", input_paths=lambda r: [tmp_path / "missing"],
batch_size=1, follow=True, upstream_done=[tmp_path / "done"], timeout_seconds=1))
def test_follow_requires_completion_markers(tmp_path):
with pytest.raises(ValueError, match="completion markers"):
list(queue(tmp_path, [1], follow=True))
def test_duplicate_outputs_fail_closed(tmp_path):
with pytest.raises(ValueError, match="Duplicate"):
list(queue(tmp_path, [1, 1]))
def test_no_per_row_stat_storm(tmp_path, monkeypatch):
publish(tmp_path, "inputs", range(100))
def forbidden(*args, **kwargs):
raise AssertionError("Per-row stat would overload shared filesystem")
monkeypatch.setattr(Path, "exists", forbidden)
monkeypatch.setattr(Path, "is_file", forbidden)
assert len(list(queue(tmp_path, list(range(100))))) == 50
def test_dag_overlaps_independent_stages_without_global_barrier():
m = load("run_swa_vqav2_pipeline")
assert m.launchable(set(), set(), 0, True) == ["voices", "asr", "review", "encode"]
assert m.launchable(set(), set(), 0, False) == ["voices", "asr"]
assert m.launchable(set(), set(), 0, True, overlap_all=False) == ["voices", "asr"]
started = {("voices", 0), ("asr", 0)}
# GPU0 starts large downstream models before other TTS shards finish.
assert m.launchable({("voices", 0)}, started, 0, True, overlap_all=False) == ["review", "encode"]
assert m.launchable(set(), {("voices", 0), ("asr", 0), ("review", 0), ("encode", 0)}, 0, True) == []
def test_memory_preflight_rejects_conflicting_jobs(monkeypatch):
m = load("run_swa_vqav2_pipeline")
fractions = dict(voices=.45, asr=.04, review=.20, encode=.15)
monkeypatch.setattr(m.subprocess, "check_output", lambda *a, **kw: "143771, 125939\n")
m.check_memory(["0"], fractions, True)
monkeypatch.setattr(m.subprocess, "check_output", lambda *a, **kw: "143771, 10000\n")
with pytest.raises(RuntimeError, match="free.*budget"):
m.check_memory(["0"], fractions, True)
def test_streaming_and_serial_cover_same_rows_exactly_once(tmp_path):
rows = list(range(9))
publish(tmp_path, "inputs", [8, 0, 3])
gen = queue(tmp_path, rows, follow=True, upstream_done=[tmp_path / "done"])
streamed = next(gen) + next(gen)
publish(tmp_path, "inputs", [1, 2, 4, 5, 6, 7])
(tmp_path / "done").touch()
streamed += [r for batch in gen for r in batch]
serial = [r for batch in queue(tmp_path, rows) for r in batch]
assert sorted(streamed) == serial == rows
assert len(set(streamed)) == len(rows)