Download tests/test_swa_pipeline_queue.py from Tinghong-Ye22/code-HFD: direct link, hf CLI and curl.
- Browser
- Download file 5.45 kB
-
https://huggingface.co/Tinghong-Ye22/code-HFD/resolve/main/tests/test_swa_pipeline_queue.py
- Command line
-
hf download hf://Tinghong-Ye22/code-HFD/tests/test_swa_pipeline_queue.py
-
curl -L -o test_swa_pipeline_queue.py https://huggingface.co/Tinghong-Ye22/code-HFD/resolve/main/tests/test_swa_pipeline_queue.py
5.45 kB
| import importlib.util | |
| from pathlib import Path | |
| import sys | |
| import pytest | |
| SCRIPTS = Path(__file__).resolve().parents[1] / "scripts/data" | |
| def load(name): | |
| sys.path.insert(0, str(SCRIPTS)) | |
| spec = importlib.util.spec_from_file_location(name, SCRIPTS / f"{name}.py") | |
| module = importlib.util.module_from_spec(spec) | |
| spec.loader.exec_module(module) | |
| return module | |
| def queue(tmp_path, rows, **kw): | |
| return load("swa_pipeline_queue").ready_batches(rows, | |
| output_path=lambda r: tmp_path / "outputs" / f"{r}.json", | |
| input_paths=lambda r: [tmp_path / "inputs" / f"{r}.json"], batch_size=2, **kw) | |
| def publish(tmp_path, folder, names): | |
| path = tmp_path / folder | |
| path.mkdir(exist_ok=True) | |
| for name in names: | |
| (path / f"{name}.json").write_text("{}") | |
| def test_consume_ready_before_upstream_finishes_and_resume(tmp_path): | |
| publish(tmp_path, "inputs", [0, 1]) | |
| publish(tmp_path, "outputs", [0]) | |
| gen = queue(tmp_path, [0, 1, 2], follow=True, upstream_done=[tmp_path / "done"]) | |
| assert next(gen) == [1] | |
| publish(tmp_path, "inputs", [2]) | |
| (tmp_path / "done").touch() | |
| assert next(gen) == [2] | |
| with pytest.raises(StopIteration): | |
| next(gen) | |
| def test_partial_receipt_is_never_consumed(tmp_path): | |
| publish(tmp_path, "inputs", [1]) | |
| (tmp_path / "inputs/2.json.partial").write_text("{") | |
| gen = queue(tmp_path, [1, 2]) | |
| assert next(gen) == [1] | |
| with pytest.raises(RuntimeError, match="Upstream inputs missing"): | |
| next(gen) | |
| def test_finished_upstream_missing_receipt_fails_closed(tmp_path): | |
| (tmp_path / "done").touch() | |
| with pytest.raises(RuntimeError, match="Upstream inputs missing"): | |
| list(queue(tmp_path, [1], follow=True, upstream_done=[tmp_path / "done"])) | |
| def test_corrupt_receipt_not_silently_skipped(tmp_path): | |
| # Queue ensures atomic publication; consumers must still parse/hash-check. | |
| publish(tmp_path, "inputs", [1]) | |
| (tmp_path / "inputs/1.json").write_text("broken") | |
| assert list(queue(tmp_path, [1])) == [[1]] | |
| def test_two_independent_reviews_required(tmp_path): | |
| m = load("swa_pipeline_queue") | |
| publish(tmp_path, "asr", [1, 2]) | |
| publish(tmp_path, "visual", [1]) | |
| gen = m.ready_batches([1, 2], output_path=lambda r: tmp_path / "out" / str(r), | |
| input_paths=lambda r: [tmp_path / "asr" / f"{r}.json", tmp_path / "visual" / f"{r}.json"], | |
| batch_size=2, follow=True, upstream_done=[tmp_path / "done"]) | |
| assert next(gen) == [1] | |
| publish(tmp_path, "visual", [2]) | |
| assert next(gen) == [2] | |
| assert list(gen) == [] | |
| def test_timeout_without_producer_progress(tmp_path, monkeypatch): | |
| m = load("swa_pipeline_queue") | |
| ticks = iter([0., 2.]) | |
| monkeypatch.setattr(m.time, "monotonic", lambda: next(ticks)) | |
| with pytest.raises(TimeoutError): | |
| list(m.ready_batches([1], output_path=lambda r: tmp_path / "out", input_paths=lambda r: [tmp_path / "missing"], | |
| batch_size=1, follow=True, upstream_done=[tmp_path / "done"], timeout_seconds=1)) | |
| def test_follow_requires_completion_markers(tmp_path): | |
| with pytest.raises(ValueError, match="completion markers"): | |
| list(queue(tmp_path, [1], follow=True)) | |
| def test_duplicate_outputs_fail_closed(tmp_path): | |
| with pytest.raises(ValueError, match="Duplicate"): | |
| list(queue(tmp_path, [1, 1])) | |
| def test_no_per_row_stat_storm(tmp_path, monkeypatch): | |
| publish(tmp_path, "inputs", range(100)) | |
| def forbidden(*args, **kwargs): | |
| raise AssertionError("Per-row stat would overload shared filesystem") | |
| monkeypatch.setattr(Path, "exists", forbidden) | |
| monkeypatch.setattr(Path, "is_file", forbidden) | |
| assert len(list(queue(tmp_path, list(range(100))))) == 50 | |
| def test_dag_overlaps_independent_stages_without_global_barrier(): | |
| m = load("run_swa_vqav2_pipeline") | |
| assert m.launchable(set(), set(), 0, True) == ["voices", "asr", "review", "encode"] | |
| assert m.launchable(set(), set(), 0, False) == ["voices", "asr"] | |
| assert m.launchable(set(), set(), 0, True, overlap_all=False) == ["voices", "asr"] | |
| started = {("voices", 0), ("asr", 0)} | |
| # GPU0 starts large downstream models before other TTS shards finish. | |
| assert m.launchable({("voices", 0)}, started, 0, True, overlap_all=False) == ["review", "encode"] | |
| assert m.launchable(set(), {("voices", 0), ("asr", 0), ("review", 0), ("encode", 0)}, 0, True) == [] | |
| def test_memory_preflight_rejects_conflicting_jobs(monkeypatch): | |
| m = load("run_swa_vqav2_pipeline") | |
| fractions = dict(voices=.45, asr=.04, review=.20, encode=.15) | |
| monkeypatch.setattr(m.subprocess, "check_output", lambda *a, **kw: "143771, 125939\n") | |
| m.check_memory(["0"], fractions, True) | |
| monkeypatch.setattr(m.subprocess, "check_output", lambda *a, **kw: "143771, 10000\n") | |
| with pytest.raises(RuntimeError, match="free.*budget"): | |
| m.check_memory(["0"], fractions, True) | |
| def test_streaming_and_serial_cover_same_rows_exactly_once(tmp_path): | |
| rows = list(range(9)) | |
| publish(tmp_path, "inputs", [8, 0, 3]) | |
| gen = queue(tmp_path, rows, follow=True, upstream_done=[tmp_path / "done"]) | |
| streamed = next(gen) + next(gen) | |
| publish(tmp_path, "inputs", [1, 2, 4, 5, 6, 7]) | |
| (tmp_path / "done").touch() | |
| streamed += [r for batch in gen for r in batch] | |
| serial = [r for batch in queue(tmp_path, rows) for r in batch] | |
| assert sorted(streamed) == serial == rows | |
| assert len(set(streamed)) == len(rows) | |