| |
| |
| |
| |
| |
| |
| |
|
|
| from __future__ import annotations |
|
|
| import json |
| import os |
| from pathlib import Path |
|
|
| import pytest |
|
|
| from speech_text_data_aligners.benchmarking.metrics import ( |
| BenchmarkObservation, |
| ErrorSummary, |
| GoldMetrics, |
| PairwiseMetrics, |
| ) |
| from speech_text_data_aligners.benchmarking.reports import ( |
| BackendReport, |
| BenchmarkReport, |
| render_report, |
| report_bundle_paths, |
| write_report_bundle, |
| ) |
| from speech_text_data_aligners.core.errors import BenchmarkError |
|
|
|
|
| def report_fixture() -> BenchmarkReport: |
| summary = ErrorSummary(1, 0.1, 0.1, 0.1) |
| gold = GoldMetrics( |
| total_rows=1, |
| success_count=1, |
| failure_count=0, |
| referenced_rows=1, |
| missing_reference_rows=0, |
| text_reference_count=1, |
| missing_text_reference_count=0, |
| exact_text_match_count=1, |
| exact_text_match_rate=1.0, |
| first_end_reference_count=1, |
| missing_first_end_reference_count=0, |
| first_end_errors=summary, |
| total_runtime_s=0.5, |
| mean_runtime_s=0.5, |
| ) |
| pair = PairwiseMetrics("a", "b", 1, summary, summary, summary, summary) |
| return BenchmarkReport( |
| (BackendReport("a", (), gold), BackendReport("b", (), gold)), (pair,) |
| ) |
|
|
|
|
| def test_render_json_csv_and_markdown_are_deterministic() -> None: |
| report = report_fixture() |
| json_text = render_report(report, "json") |
| assert json.loads(json_text)["backend_reports"][0]["backend_name"] == "a" |
| assert json_text.endswith("\n") |
| csv_text = render_report(report, "csv") |
| assert csv_text.startswith("scope,metric,value\n") |
| assert "a,success_count,1" in csv_text |
| markdown = render_report(report, "markdown") |
| assert markdown.startswith("| Scope | Metric | Value |") |
| assert "a vs b" in markdown |
| with pytest.raises(BenchmarkError, match="unsupported"): |
| render_report(report, "xml") |
|
|
|
|
| def test_render_includes_deduplicated_native_artifact_paths() -> None: |
| report = report_fixture() |
| observation = BenchmarkObservation( |
| "a", "line", "audio", None, None, "failed", ("native/words.ctm",) |
| ) |
| backend = report.backend_reports[0] |
| with_artifact = BenchmarkReport( |
| ( |
| type(backend)( |
| backend.backend_name, |
| (observation, observation), |
| backend.gold_metrics, |
| ), |
| ) |
| ) |
| markdown = render_report(with_artifact, "markdown") |
| assert markdown.count("native/words.ctm") == 1 |
| assert "native_artifact_1" in markdown |
|
|
|
|
| def test_render_handles_none_metrics() -> None: |
| empty = ErrorSummary(0, None, None, None) |
| gold = GoldMetrics( |
| total_rows=0, |
| success_count=0, |
| failure_count=0, |
| referenced_rows=0, |
| missing_reference_rows=0, |
| text_reference_count=0, |
| missing_text_reference_count=0, |
| exact_text_match_count=0, |
| exact_text_match_rate=None, |
| first_end_reference_count=0, |
| missing_first_end_reference_count=0, |
| first_end_errors=empty, |
| total_runtime_s=0.0, |
| mean_runtime_s=None, |
| ) |
| pair = PairwiseMetrics("a", "b", 0, empty, empty, empty, empty) |
| text = render_report( |
| BenchmarkReport((BackendReport("a", (), gold),), (pair,)), "csv" |
| ) |
| assert "exact_text_match_rate," in text |
| assert "first_start_mae_s," in text |
|
|
|
|
| def test_write_bundle_preflights_and_overwrites(tmp_path: Path) -> None: |
| base = tmp_path / "report.any" |
| metadata = tmp_path / "input.csv" |
| reference = tmp_path / "reference.csv" |
| paths = write_report_bundle( |
| report_fixture(), base, metadata_csv=metadata, reference_csv=reference |
| ) |
| assert paths == ( |
| tmp_path / "report.json", |
| tmp_path / "report.csv", |
| tmp_path / "report.md", |
| ) |
| assert all(path.exists() for path in paths) |
| with pytest.raises(BenchmarkError, match="already"): |
| write_report_bundle( |
| report_fixture(), base, metadata_csv=metadata, reference_csv=reference |
| ) |
| write_report_bundle( |
| report_fixture(), |
| base, |
| metadata_csv=metadata, |
| reference_csv=reference, |
| overwrite=True, |
| ) |
|
|
|
|
| def test_write_bundle_translates_publication_failure( |
| tmp_path: Path, monkeypatch: pytest.MonkeyPatch |
| ) -> None: |
| def fail_replace(source: Path, destination: Path) -> None: |
| raise OSError(source, destination) |
|
|
| monkeypatch.setattr(os, "replace", fail_replace) |
| with pytest.raises(BenchmarkError, match="publish"): |
| write_report_bundle( |
| report_fixture(), |
| tmp_path / "report", |
| metadata_csv=tmp_path / "input.csv", |
| reference_csv=tmp_path / "reference.csv", |
| ) |
| assert not list(tmp_path.glob(".report.json.*.tmp")) |
|
|
|
|
| @pytest.mark.parametrize("input_kind", ["metadata", "reference"]) |
| @pytest.mark.parametrize("suffix", [".json", ".csv", ".md"]) |
| def test_write_bundle_rejects_each_report_target_aliasing_an_input_without_mutation( |
| tmp_path: Path, input_kind: str, suffix: str |
| ) -> None: |
| directory = tmp_path / f"{input_kind}-{suffix[1:]}" |
| directory.mkdir() |
| base = directory / "report" |
| target = base.with_suffix(suffix) |
| target.write_bytes(b"immutable input") |
| metadata = target if input_kind == "metadata" else directory / "metadata.csv" |
| reference = target if input_kind == "reference" else directory / "reference.csv" |
| with pytest.raises(BenchmarkError, match="paths alias"): |
| write_report_bundle( |
| report_fixture(), |
| base, |
| metadata_csv=metadata, |
| reference_csv=reference, |
| overwrite=True, |
| ) |
| assert target.read_bytes() == b"immutable input" |
| assert not any( |
| path.exists() for path in report_bundle_paths(base) if path != target |
| ) |
|
|
|
|
| def test_write_bundle_rejects_generated_targets_that_are_same_file( |
| tmp_path: Path, |
| ) -> None: |
| base = tmp_path / "report" |
| json_path, csv_path, markdown_path = report_bundle_paths(base) |
| json_path.write_bytes(b"existing reports") |
| try: |
| csv_path.hardlink_to(json_path) |
| except OSError as error: |
| pytest.skip(f"hard links are unavailable: {error}") |
| with pytest.raises(BenchmarkError, match="paths alias"): |
| write_report_bundle( |
| report_fixture(), |
| base, |
| metadata_csv=tmp_path / "metadata.csv", |
| reference_csv=tmp_path / "reference.csv", |
| overwrite=True, |
| ) |
| assert json_path.read_bytes() == b"existing reports" |
| assert csv_path.read_bytes() == b"existing reports" |
| assert not markdown_path.exists() |
|
|
