tests/unit/cli/test_benchmark.py

# License: Apache-2.0 License
# Created by: Patrick Lumbantobing, VertoX-AI
# Copyright (c) 2026 VertoX-AI. All rights reserved.
#
# This work is licensed under the Apache-2.0 License.
# To view a copy of this license, visit
# https://www.apache.org/licenses/LICENSE-2.0

from __future__ import annotations

import argparse
from pathlib import Path
from types import SimpleNamespace

import pytest

from speech_text_data_aligners.benchmarking.reports import BenchmarkReport
from speech_text_data_aligners.cli import benchmark as cli
from speech_text_data_aligners.core.errors import BenchmarkError, ConfigurationError


class Registry:
    def __init__(self) -> None:
        self.calls: list[tuple[str, object]] = []

    def create(self, name: str, **kwargs: object) -> object:
        self.calls.append((name, kwargs["config"]))
        return SimpleNamespace(name=name)


def parse_args(arguments: list[str]) -> argparse.Namespace:
    parser = argparse.ArgumentParser()
    cli.configure_parser(parser)
    return parser.parse_args(arguments)


def base_arguments(tmp_path: Path) -> list[str]:
    return [
        str(tmp_path / "in.csv"),
        str(tmp_path / "gold.csv"),
        str(tmp_path / "wavs"),
        "--backend",
        "a",
    ]


def test_run_delegates_benchmark_and_prints_selected_format(
    tmp_path: Path, monkeypatch: pytest.MonkeyPatch, capsys: pytest.CaptureFixture[str]
) -> None:
    args = parse_args(
        [
            *base_arguments(tmp_path),
            "--backend",
            "b",
            "--backend-config-json",
            '{"a":{"x":1}}',
            "--strict",
            "--format",
            "json",
        ]
    )
    report = BenchmarkReport(())
    calls: list[dict[str, object]] = []
    monkeypatch.setattr(
        cli,
        "benchmark_backends",
        lambda **kwargs: (calls.append(kwargs), report)[1],
    )
    monkeypatch.setattr(cli, "render_report", lambda value, output_format: "rendered\n")
    registry = Registry()
    assert cli.run(args, registry=registry) == 0  # type: ignore[arg-type]
    assert registry.calls == [("a", {"x": 1}), ("b", {})]
    assert calls[0]["reference_csv"] == tmp_path / "gold.csv"
    assert calls[0]["options"].strict  # type: ignore[union-attr]
    assert capsys.readouterr().out == "rendered\n"


def test_run_writes_bundle_and_uses_default_registry(
    tmp_path: Path, monkeypatch: pytest.MonkeyPatch
) -> None:
    args = parse_args(
        [
            *base_arguments(tmp_path),
            "--output-base",
            str(tmp_path / "report"),
            "--overwrite",
        ]
    )
    registry = Registry()
    report = BenchmarkReport(())
    writes: list[tuple[object, Path, bool]] = []
    monkeypatch.setattr(cli, "default_backend_registry", lambda: registry)
    monkeypatch.setattr(cli, "benchmark_backends", lambda **kwargs: report)
    monkeypatch.setattr(
        cli,
        "write_report_bundle",
        lambda value, path, **kwargs: writes.append((value, path, kwargs["overwrite"])),
    )
    assert cli.run(args) == 0
    assert writes == [(report, tmp_path / "report", True)]


def test_run_rejects_report_input_collision_before_backend_construction(
    tmp_path: Path,
) -> None:
    args = parse_args(
        [
            *base_arguments(tmp_path),
            "--output-base",
            str(tmp_path / "in"),
            "--overwrite",
        ]
    )
    registry = Registry()
    with pytest.raises(BenchmarkError, match="paths alias"):
        cli.run(args, registry=registry)  # type: ignore[arg-type]
    assert registry.calls == []


def test_run_rejects_duplicate_names_and_bad_configuration(tmp_path: Path) -> None:
    duplicate = parse_args([*base_arguments(tmp_path), "--backend", "a"])
    with pytest.raises(ConfigurationError, match="unique"):
        cli.run(duplicate, registry=Registry())  # type: ignore[arg-type]
    for value in ("bad", "[]"):
        args = parse_args(base_arguments(tmp_path))
        args.backend_config_json = value
        with pytest.raises(ConfigurationError):
            cli.run(args, registry=Registry())  # type: ignore[arg-type]