# License: Apache-2.0 License # Created by: Patrick Lumbantobing, VertoX-AI # Copyright (c) 2026 VertoX-AI. All rights reserved. # # This work is licensed under the Apache-2.0 License. # To view a copy of this license, visit # https://www.apache.org/licenses/LICENSE-2.0 from __future__ import annotations import argparse from pathlib import Path from types import SimpleNamespace import pytest from speech_text_data_aligners.benchmarking.reports import BenchmarkReport from speech_text_data_aligners.cli import benchmark as cli from speech_text_data_aligners.core.errors import BenchmarkError, ConfigurationError class Registry: def __init__(self) -> None: self.calls: list[tuple[str, object]] = [] def create(self, name: str, **kwargs: object) -> object: self.calls.append((name, kwargs["config"])) return SimpleNamespace(name=name) def parse_args(arguments: list[str]) -> argparse.Namespace: parser = argparse.ArgumentParser() cli.configure_parser(parser) return parser.parse_args(arguments) def base_arguments(tmp_path: Path) -> list[str]: return [ str(tmp_path / "in.csv"), str(tmp_path / "gold.csv"), str(tmp_path / "wavs"), "--backend", "a", ] def test_run_delegates_benchmark_and_prints_selected_format( tmp_path: Path, monkeypatch: pytest.MonkeyPatch, capsys: pytest.CaptureFixture[str] ) -> None: args = parse_args( [ *base_arguments(tmp_path), "--backend", "b", "--backend-config-json", '{"a":{"x":1}}', "--strict", "--format", "json", ] ) report = BenchmarkReport(()) calls: list[dict[str, object]] = [] monkeypatch.setattr( cli, "benchmark_backends", lambda **kwargs: (calls.append(kwargs), report)[1], ) monkeypatch.setattr(cli, "render_report", lambda value, output_format: "rendered\n") registry = Registry() assert cli.run(args, registry=registry) == 0 # type: ignore[arg-type] assert registry.calls == [("a", {"x": 1}), ("b", {})] assert calls[0]["reference_csv"] == tmp_path / "gold.csv" assert calls[0]["options"].strict # type: ignore[union-attr] assert capsys.readouterr().out == "rendered\n" def test_run_writes_bundle_and_uses_default_registry( tmp_path: Path, monkeypatch: pytest.MonkeyPatch ) -> None: args = parse_args( [ *base_arguments(tmp_path), "--output-base", str(tmp_path / "report"), "--overwrite", ] ) registry = Registry() report = BenchmarkReport(()) writes: list[tuple[object, Path, bool]] = [] monkeypatch.setattr(cli, "default_backend_registry", lambda: registry) monkeypatch.setattr(cli, "benchmark_backends", lambda **kwargs: report) monkeypatch.setattr( cli, "write_report_bundle", lambda value, path, **kwargs: writes.append((value, path, kwargs["overwrite"])), ) assert cli.run(args) == 0 assert writes == [(report, tmp_path / "report", True)] def test_run_rejects_report_input_collision_before_backend_construction( tmp_path: Path, ) -> None: args = parse_args( [ *base_arguments(tmp_path), "--output-base", str(tmp_path / "in"), "--overwrite", ] ) registry = Registry() with pytest.raises(BenchmarkError, match="paths alias"): cli.run(args, registry=registry) # type: ignore[arg-type] assert registry.calls == [] def test_run_rejects_duplicate_names_and_bad_configuration(tmp_path: Path) -> None: duplicate = parse_args([*base_arguments(tmp_path), "--backend", "a"]) with pytest.raises(ConfigurationError, match="unique"): cli.run(duplicate, registry=Registry()) # type: ignore[arg-type] for value in ("bad", "[]"): args = parse_args(base_arguments(tmp_path)) args.backend_config_json = value with pytest.raises(ConfigurationError): cli.run(args, registry=Registry()) # type: ignore[arg-type]