File size: 5,450 Bytes
aa02c28
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
import importlib.util
from pathlib import Path
import sys

import pytest

SCRIPTS = Path(__file__).resolve().parents[1] / "scripts/data"


def load(name):
    sys.path.insert(0, str(SCRIPTS))
    spec = importlib.util.spec_from_file_location(name, SCRIPTS / f"{name}.py")
    module = importlib.util.module_from_spec(spec)
    spec.loader.exec_module(module)
    return module


def queue(tmp_path, rows, **kw):
    return load("swa_pipeline_queue").ready_batches(rows,
        output_path=lambda r: tmp_path / "outputs" / f"{r}.json",
        input_paths=lambda r: [tmp_path / "inputs" / f"{r}.json"], batch_size=2, **kw)


def publish(tmp_path, folder, names):
    path = tmp_path / folder
    path.mkdir(exist_ok=True)
    for name in names:
        (path / f"{name}.json").write_text("{}")


def test_consume_ready_before_upstream_finishes_and_resume(tmp_path):
    publish(tmp_path, "inputs", [0, 1])
    publish(tmp_path, "outputs", [0])
    gen = queue(tmp_path, [0, 1, 2], follow=True, upstream_done=[tmp_path / "done"])
    assert next(gen) == [1]
    publish(tmp_path, "inputs", [2])
    (tmp_path / "done").touch()
    assert next(gen) == [2]
    with pytest.raises(StopIteration):
        next(gen)


def test_partial_receipt_is_never_consumed(tmp_path):
    publish(tmp_path, "inputs", [1])
    (tmp_path / "inputs/2.json.partial").write_text("{")
    gen = queue(tmp_path, [1, 2])
    assert next(gen) == [1]
    with pytest.raises(RuntimeError, match="Upstream inputs missing"):
        next(gen)


def test_finished_upstream_missing_receipt_fails_closed(tmp_path):
    (tmp_path / "done").touch()
    with pytest.raises(RuntimeError, match="Upstream inputs missing"):
        list(queue(tmp_path, [1], follow=True, upstream_done=[tmp_path / "done"]))


def test_corrupt_receipt_not_silently_skipped(tmp_path):
    # Queue ensures atomic publication; consumers must still parse/hash-check.
    publish(tmp_path, "inputs", [1])
    (tmp_path / "inputs/1.json").write_text("broken")
    assert list(queue(tmp_path, [1])) == [[1]]


def test_two_independent_reviews_required(tmp_path):
    m = load("swa_pipeline_queue")
    publish(tmp_path, "asr", [1, 2])
    publish(tmp_path, "visual", [1])
    gen = m.ready_batches([1, 2], output_path=lambda r: tmp_path / "out" / str(r),
        input_paths=lambda r: [tmp_path / "asr" / f"{r}.json", tmp_path / "visual" / f"{r}.json"],
        batch_size=2, follow=True, upstream_done=[tmp_path / "done"])
    assert next(gen) == [1]
    publish(tmp_path, "visual", [2])
    assert next(gen) == [2]
    assert list(gen) == []


def test_timeout_without_producer_progress(tmp_path, monkeypatch):
    m = load("swa_pipeline_queue")
    ticks = iter([0., 2.])
    monkeypatch.setattr(m.time, "monotonic", lambda: next(ticks))
    with pytest.raises(TimeoutError):
        list(m.ready_batches([1], output_path=lambda r: tmp_path / "out", input_paths=lambda r: [tmp_path / "missing"],
            batch_size=1, follow=True, upstream_done=[tmp_path / "done"], timeout_seconds=1))


def test_follow_requires_completion_markers(tmp_path):
    with pytest.raises(ValueError, match="completion markers"):
        list(queue(tmp_path, [1], follow=True))


def test_duplicate_outputs_fail_closed(tmp_path):
    with pytest.raises(ValueError, match="Duplicate"):
        list(queue(tmp_path, [1, 1]))


def test_no_per_row_stat_storm(tmp_path, monkeypatch):
    publish(tmp_path, "inputs", range(100))
    def forbidden(*args, **kwargs):
        raise AssertionError("Per-row stat would overload shared filesystem")
    monkeypatch.setattr(Path, "exists", forbidden)
    monkeypatch.setattr(Path, "is_file", forbidden)
    assert len(list(queue(tmp_path, list(range(100))))) == 50


def test_dag_overlaps_independent_stages_without_global_barrier():
    m = load("run_swa_vqav2_pipeline")
    assert m.launchable(set(), set(), 0, True) == ["voices", "asr", "review", "encode"]
    assert m.launchable(set(), set(), 0, False) == ["voices", "asr"]
    assert m.launchable(set(), set(), 0, True, overlap_all=False) == ["voices", "asr"]
    started = {("voices", 0), ("asr", 0)}
    # GPU0 starts large downstream models before other TTS shards finish.
    assert m.launchable({("voices", 0)}, started, 0, True, overlap_all=False) == ["review", "encode"]
    assert m.launchable(set(), {("voices", 0), ("asr", 0), ("review", 0), ("encode", 0)}, 0, True) == []


def test_memory_preflight_rejects_conflicting_jobs(monkeypatch):
    m = load("run_swa_vqav2_pipeline")
    fractions = dict(voices=.45, asr=.04, review=.20, encode=.15)
    monkeypatch.setattr(m.subprocess, "check_output", lambda *a, **kw: "143771, 125939\n")
    m.check_memory(["0"], fractions, True)
    monkeypatch.setattr(m.subprocess, "check_output", lambda *a, **kw: "143771, 10000\n")
    with pytest.raises(RuntimeError, match="free.*budget"):
        m.check_memory(["0"], fractions, True)


def test_streaming_and_serial_cover_same_rows_exactly_once(tmp_path):
    rows = list(range(9))
    publish(tmp_path, "inputs", [8, 0, 3])
    gen = queue(tmp_path, rows, follow=True, upstream_done=[tmp_path / "done"])
    streamed = next(gen) + next(gen)
    publish(tmp_path, "inputs", [1, 2, 4, 5, 6, 7])
    (tmp_path / "done").touch()
    streamed += [r for batch in gen for r in batch]
    serial = [r for batch in queue(tmp_path, rows) for r in batch]
    assert sorted(streamed) == serial == rows
    assert len(set(streamed)) == len(rows)