File size: 4,137 Bytes
51e84d2 fc1dcbc | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 | # License: Apache-2.0 License
# Created by: Patrick Lumbantobing, VertoX-AI
# Copyright (c) 2026 VertoX-AI. All rights reserved.
#
# This work is licensed under the Apache-2.0 License.
# To view a copy of this license, visit
# https://www.apache.org/licenses/LICENSE-2.0
from __future__ import annotations
import argparse
from pathlib import Path
from types import SimpleNamespace
import pytest
from speech_text_data_aligners.benchmarking.reports import BenchmarkReport
from speech_text_data_aligners.cli import benchmark as cli
from speech_text_data_aligners.core.errors import BenchmarkError, ConfigurationError
class Registry:
def __init__(self) -> None:
self.calls: list[tuple[str, object]] = []
def create(self, name: str, **kwargs: object) -> object:
self.calls.append((name, kwargs["config"]))
return SimpleNamespace(name=name)
def parse_args(arguments: list[str]) -> argparse.Namespace:
parser = argparse.ArgumentParser()
cli.configure_parser(parser)
return parser.parse_args(arguments)
def base_arguments(tmp_path: Path) -> list[str]:
return [
str(tmp_path / "in.csv"),
str(tmp_path / "gold.csv"),
str(tmp_path / "wavs"),
"--backend",
"a",
]
def test_run_delegates_benchmark_and_prints_selected_format(
tmp_path: Path, monkeypatch: pytest.MonkeyPatch, capsys: pytest.CaptureFixture[str]
) -> None:
args = parse_args(
[
*base_arguments(tmp_path),
"--backend",
"b",
"--backend-config-json",
'{"a":{"x":1}}',
"--strict",
"--format",
"json",
]
)
report = BenchmarkReport(())
calls: list[dict[str, object]] = []
monkeypatch.setattr(
cli,
"benchmark_backends",
lambda **kwargs: (calls.append(kwargs), report)[1],
)
monkeypatch.setattr(cli, "render_report", lambda value, output_format: "rendered\n")
registry = Registry()
assert cli.run(args, registry=registry) == 0 # type: ignore[arg-type]
assert registry.calls == [("a", {"x": 1}), ("b", {})]
assert calls[0]["reference_csv"] == tmp_path / "gold.csv"
assert calls[0]["options"].strict # type: ignore[union-attr]
assert capsys.readouterr().out == "rendered\n"
def test_run_writes_bundle_and_uses_default_registry(
tmp_path: Path, monkeypatch: pytest.MonkeyPatch
) -> None:
args = parse_args(
[
*base_arguments(tmp_path),
"--output-base",
str(tmp_path / "report"),
"--overwrite",
]
)
registry = Registry()
report = BenchmarkReport(())
writes: list[tuple[object, Path, bool]] = []
monkeypatch.setattr(cli, "default_backend_registry", lambda: registry)
monkeypatch.setattr(cli, "benchmark_backends", lambda **kwargs: report)
monkeypatch.setattr(
cli,
"write_report_bundle",
lambda value, path, **kwargs: writes.append((value, path, kwargs["overwrite"])),
)
assert cli.run(args) == 0
assert writes == [(report, tmp_path / "report", True)]
def test_run_rejects_report_input_collision_before_backend_construction(
tmp_path: Path,
) -> None:
args = parse_args(
[
*base_arguments(tmp_path),
"--output-base",
str(tmp_path / "in"),
"--overwrite",
]
)
registry = Registry()
with pytest.raises(BenchmarkError, match="paths alias"):
cli.run(args, registry=registry) # type: ignore[arg-type]
assert registry.calls == []
def test_run_rejects_duplicate_names_and_bad_configuration(tmp_path: Path) -> None:
duplicate = parse_args([*base_arguments(tmp_path), "--backend", "a"])
with pytest.raises(ConfigurationError, match="unique"):
cli.run(duplicate, registry=Registry()) # type: ignore[arg-type]
for value in ("bad", "[]"):
args = parse_args(base_arguments(tmp_path))
args.backend_config_json = value
with pytest.raises(ConfigurationError):
cli.run(args, registry=Registry()) # type: ignore[arg-type]
|