# License: Apache-2.0 License # Created by: Patrick Lumbantobing, VertoX-AI # Copyright (c) 2026 VertoX-AI. All rights reserved. # # This work is licensed under the Apache-2.0 License. # To view a copy of this license, visit # https://www.apache.org/licenses/LICENSE-2.0 from __future__ import annotations import json import os from pathlib import Path import pytest from speech_text_data_aligners.benchmarking.metrics import ( BenchmarkObservation, ErrorSummary, GoldMetrics, PairwiseMetrics, ) from speech_text_data_aligners.benchmarking.reports import ( BackendReport, BenchmarkReport, render_report, report_bundle_paths, write_report_bundle, ) from speech_text_data_aligners.core.errors import BenchmarkError def report_fixture() -> BenchmarkReport: summary = ErrorSummary(1, 0.1, 0.1, 0.1) gold = GoldMetrics( total_rows=1, success_count=1, failure_count=0, referenced_rows=1, missing_reference_rows=0, text_reference_count=1, missing_text_reference_count=0, exact_text_match_count=1, exact_text_match_rate=1.0, first_end_reference_count=1, missing_first_end_reference_count=0, first_end_errors=summary, total_runtime_s=0.5, mean_runtime_s=0.5, ) pair = PairwiseMetrics("a", "b", 1, summary, summary, summary, summary) return BenchmarkReport( (BackendReport("a", (), gold), BackendReport("b", (), gold)), (pair,) ) def test_render_json_csv_and_markdown_are_deterministic() -> None: report = report_fixture() json_text = render_report(report, "json") assert json.loads(json_text)["backend_reports"][0]["backend_name"] == "a" assert json_text.endswith("\n") csv_text = render_report(report, "csv") assert csv_text.startswith("scope,metric,value\n") assert "a,success_count,1" in csv_text markdown = render_report(report, "markdown") assert markdown.startswith("| Scope | Metric | Value |") assert "a vs b" in markdown with pytest.raises(BenchmarkError, match="unsupported"): render_report(report, "xml") # type: ignore[arg-type] def test_render_includes_deduplicated_native_artifact_paths() -> None: report = report_fixture() observation = BenchmarkObservation( "a", "line", "audio", None, None, "failed", ("native/words.ctm",) ) backend = report.backend_reports[0] with_artifact = BenchmarkReport( ( type(backend)( backend.backend_name, (observation, observation), backend.gold_metrics, ), ) ) markdown = render_report(with_artifact, "markdown") assert markdown.count("native/words.ctm") == 1 assert "native_artifact_1" in markdown def test_render_handles_none_metrics() -> None: empty = ErrorSummary(0, None, None, None) gold = GoldMetrics( total_rows=0, success_count=0, failure_count=0, referenced_rows=0, missing_reference_rows=0, text_reference_count=0, missing_text_reference_count=0, exact_text_match_count=0, exact_text_match_rate=None, first_end_reference_count=0, missing_first_end_reference_count=0, first_end_errors=empty, total_runtime_s=0.0, mean_runtime_s=None, ) pair = PairwiseMetrics("a", "b", 0, empty, empty, empty, empty) text = render_report( BenchmarkReport((BackendReport("a", (), gold),), (pair,)), "csv" ) assert "exact_text_match_rate," in text assert "first_start_mae_s," in text def test_write_bundle_preflights_and_overwrites(tmp_path: Path) -> None: base = tmp_path / "report.any" metadata = tmp_path / "input.csv" reference = tmp_path / "reference.csv" paths = write_report_bundle( report_fixture(), base, metadata_csv=metadata, reference_csv=reference ) assert paths == ( tmp_path / "report.json", tmp_path / "report.csv", tmp_path / "report.md", ) assert all(path.exists() for path in paths) with pytest.raises(BenchmarkError, match="already"): write_report_bundle( report_fixture(), base, metadata_csv=metadata, reference_csv=reference ) write_report_bundle( report_fixture(), base, metadata_csv=metadata, reference_csv=reference, overwrite=True, ) def test_write_bundle_translates_publication_failure( tmp_path: Path, monkeypatch: pytest.MonkeyPatch ) -> None: def fail_replace(source: Path, destination: Path) -> None: raise OSError(source, destination) monkeypatch.setattr(os, "replace", fail_replace) with pytest.raises(BenchmarkError, match="publish"): write_report_bundle( report_fixture(), tmp_path / "report", metadata_csv=tmp_path / "input.csv", reference_csv=tmp_path / "reference.csv", ) assert not list(tmp_path.glob(".report.json.*.tmp")) @pytest.mark.parametrize("input_kind", ["metadata", "reference"]) @pytest.mark.parametrize("suffix", [".json", ".csv", ".md"]) def test_write_bundle_rejects_each_report_target_aliasing_an_input_without_mutation( tmp_path: Path, input_kind: str, suffix: str ) -> None: directory = tmp_path / f"{input_kind}-{suffix[1:]}" directory.mkdir() base = directory / "report" target = base.with_suffix(suffix) target.write_bytes(b"immutable input") metadata = target if input_kind == "metadata" else directory / "metadata.csv" reference = target if input_kind == "reference" else directory / "reference.csv" with pytest.raises(BenchmarkError, match="paths alias"): write_report_bundle( report_fixture(), base, metadata_csv=metadata, reference_csv=reference, overwrite=True, ) assert target.read_bytes() == b"immutable input" assert not any( path.exists() for path in report_bundle_paths(base) if path != target ) def test_write_bundle_rejects_generated_targets_that_are_same_file( tmp_path: Path, ) -> None: base = tmp_path / "report" json_path, csv_path, markdown_path = report_bundle_paths(base) json_path.write_bytes(b"existing reports") try: csv_path.hardlink_to(json_path) except OSError as error: pytest.skip(f"hard links are unavailable: {error}") with pytest.raises(BenchmarkError, match="paths alias"): write_report_bundle( report_fixture(), base, metadata_csv=tmp_path / "metadata.csv", reference_csv=tmp_path / "reference.csv", overwrite=True, ) assert json_path.read_bytes() == b"existing reports" assert csv_path.read_bytes() == b"existing reports" assert not markdown_path.exists()