from __future__ import annotations import importlib.util import json import sys from argparse import Namespace from pathlib import Path def _load_tool(): path = Path(__file__).resolve().parents[1] / "tools" / "admit_limbu_book_ocr_candidate.py" spec = importlib.util.spec_from_file_location("admit_limbu_book_ocr_candidate", path) assert spec is not None assert spec.loader is not None module = importlib.util.module_from_spec(spec) sys.modules[spec.name] = module spec.loader.exec_module(module) return module def _write_json(path: Path, payload: object) -> None: path.parent.mkdir(parents=True, exist_ok=True) path.write_text(json.dumps(payload, ensure_ascii=False, indent=2) + "\n", encoding="utf-8") def _metrics(*, codepoint_cer: float, grapheme_cer: float = 0.6, wer: float = 0.8) -> dict[str, object]: return { "manifest_samples": 3, "scored_pages": 3, "claim_eligible_rows": 0, "missing_documents": [], "missing_references": [], "metric_means": {"text_wer": wer}, "text_cer_codepoint_micro": codepoint_cer, "text_cer_grapheme_micro": grapheme_cer, "line_detection_precision_micro": 0.7, "line_detection_recall_micro": 0.75, "line_detection_f1_micro": 0.72, "matched_line_reading_order_pair_accuracy": 0.61, "line_blank_prediction_rate": 0.0, "line_extra_prediction_rate": 0.27, "line_missed_reference_rate": 0.22, } def _coverage(*, uncovered: int = 0, extra_uncovered: int = 0) -> dict[str, object]: return { "tool": "tools/audit_limbu_book_missed_line_review_coverage.py", "missed_reference_count": 2, "uncovered_or_weak_count": uncovered, "extra_prediction_count": 3, "extra_uncovered_or_weak_count": extra_uncovered, } def _args(tmp_path: Path, *, book_output: Path, score_dir: Path) -> Namespace: return Namespace( book_output=book_output, score_dir=score_dir, out=tmp_path / "admission", book_audit=None, metrics_sha256=None, book_audit_sha256=None, champion_metrics=None, review_coverage_report=None, baseline_codepoint_cer=0.458432, baseline_grapheme_cer=None, baseline_wer=None, min_codepoint_improvement=0.0, max_grapheme_regression=0.0, max_wer_regression=0.0, max_champion_codepoint_regression=0.0, max_champion_grapheme_regression=0.0, max_champion_wer_regression=0.0, max_review_uncovered_or_weak_count=0, max_review_extra_uncovered_or_weak_count=0, min_line_f1=0.70, min_reading_order_accuracy=0.50, max_blank_line_rate=0.0, max_extra_line_rate=0.30, max_missed_line_rate=0.30, min_scored_pages=3, min_claim_pages=30, ) def test_admit_limbu_book_ocr_candidate_passes_operational_gate(tmp_path: Path) -> None: tool = _load_tool() book_output = tmp_path / "book" score_dir = tmp_path / "score" _write_json(book_output / "limbu-book-output-audit.json", {"passed": True, "output_dir": str(book_output)}) _write_json(score_dir / "metrics.json", _metrics(codepoint_cer=0.446)) report = tool.admit_candidate(_args(tmp_path, book_output=book_output, score_dir=score_dir)) assert report["admitted_operational"] is True assert report["claim_safe"] is False assert report["claim_safe_reasons"] == ["not all scored rows are claim eligible", "scored_pages below claim minimum 30"] assert report["codepoint_absolute_improvement"] == 0.458432 - 0.446 assert (tmp_path / "admission" / "limbu-book-ocr-candidate-admission.json").exists() assert (tmp_path / "admission" / "limbu-book-ocr-candidate-admission.md").exists() def test_admit_limbu_book_ocr_candidate_rejects_codepoint_regression(tmp_path: Path) -> None: tool = _load_tool() book_output = tmp_path / "book" score_dir = tmp_path / "score" _write_json(book_output / "limbu-book-output-audit.json", {"passed": True, "output_dir": str(book_output)}) _write_json(score_dir / "metrics.json", _metrics(codepoint_cer=0.46)) report = tool.admit_candidate(_args(tmp_path, book_output=book_output, score_dir=score_dir)) assert report["admitted_operational"] is False assert any("codepoint CER improvement below requirement" in issue for issue in report["issues"]) def test_admit_limbu_book_ocr_candidate_rejects_grapheme_and_wer_regressions(tmp_path: Path) -> None: tool = _load_tool() book_output = tmp_path / "book" score_dir = tmp_path / "score" _write_json(book_output / "limbu-book-output-audit.json", {"passed": True, "output_dir": str(book_output)}) _write_json(score_dir / "metrics.json", _metrics(codepoint_cer=0.446, grapheme_cer=0.64, wer=0.87)) args = _args(tmp_path, book_output=book_output, score_dir=score_dir) args.baseline_grapheme_cer = 0.62 args.baseline_wer = 0.84 report = tool.admit_candidate(args) assert report["admitted_operational"] is False assert report["grapheme_absolute_improvement"] == 0.62 - 0.64 assert report["wer_absolute_improvement"] == 0.84 - 0.87 assert any("grapheme CER regression above allowed maximum" in issue for issue in report["issues"]) assert any("WER regression above allowed maximum" in issue for issue in report["issues"]) def test_admit_limbu_book_ocr_candidate_rejects_champion_text_regression(tmp_path: Path) -> None: tool = _load_tool() book_output = tmp_path / "book" score_dir = tmp_path / "score" champion_metrics = tmp_path / "champion" / "metrics.json" _write_json(book_output / "limbu-book-output-audit.json", {"passed": True, "output_dir": str(book_output)}) _write_json(score_dir / "metrics.json", _metrics(codepoint_cer=0.44, grapheme_cer=0.56, wer=0.74)) _write_json(champion_metrics, _metrics(codepoint_cer=0.40, grapheme_cer=0.51, wer=0.71)) args = _args(tmp_path, book_output=book_output, score_dir=score_dir) args.champion_metrics = champion_metrics report = tool.admit_candidate(args) assert report["admitted_operational"] is False assert report["champion_codepoint_absolute_improvement"] == 0.40 - 0.44 assert report["champion_grapheme_absolute_improvement"] == 0.51 - 0.56 assert report["champion_wer_absolute_improvement"] == 0.71 - 0.74 assert any("champion codepoint CER regression" in issue for issue in report["issues"]) assert any("champion grapheme CER regression" in issue for issue in report["issues"]) assert any("champion WER regression" in issue for issue in report["issues"]) def test_admit_limbu_book_ocr_candidate_accepts_champion_tie(tmp_path: Path) -> None: tool = _load_tool() book_output = tmp_path / "book" score_dir = tmp_path / "score" champion_metrics = tmp_path / "champion" / "metrics.json" _write_json(book_output / "limbu-book-output-audit.json", {"passed": True, "output_dir": str(book_output)}) _write_json(score_dir / "metrics.json", _metrics(codepoint_cer=0.40, grapheme_cer=0.51, wer=0.71)) _write_json(champion_metrics, _metrics(codepoint_cer=0.40, grapheme_cer=0.51, wer=0.71)) args = _args(tmp_path, book_output=book_output, score_dir=score_dir) args.champion_metrics = champion_metrics report = tool.admit_candidate(args) assert report["admitted_operational"] is True assert report["champion_metric_values"]["text_cer_codepoint_micro"] == 0.40 assert report["champion_codepoint_absolute_improvement"] == 0.0 def test_admit_limbu_book_ocr_candidate_accepts_clean_review_coverage(tmp_path: Path) -> None: tool = _load_tool() book_output = tmp_path / "book" score_dir = tmp_path / "score" coverage_report = tmp_path / "coverage" / "missed-line-review-coverage.json" _write_json(book_output / "limbu-book-output-audit.json", {"passed": True, "output_dir": str(book_output)}) _write_json(score_dir / "metrics.json", _metrics(codepoint_cer=0.446)) _write_json(coverage_report, _coverage(uncovered=0, extra_uncovered=0)) args = _args(tmp_path, book_output=book_output, score_dir=score_dir) args.review_coverage_report = coverage_report report = tool.admit_candidate(args) assert report["admitted_operational"] is True assert report["review_coverage_values"]["uncovered_or_weak_count"] == 0 assert report["review_coverage_values"]["extra_uncovered_or_weak_count"] == 0 def test_admit_limbu_book_ocr_candidate_rejects_uncovered_review_regions(tmp_path: Path) -> None: tool = _load_tool() book_output = tmp_path / "book" score_dir = tmp_path / "score" coverage_report = tmp_path / "coverage" / "missed-line-review-coverage.json" _write_json(book_output / "limbu-book-output-audit.json", {"passed": True, "output_dir": str(book_output)}) _write_json(score_dir / "metrics.json", _metrics(codepoint_cer=0.446)) _write_json(coverage_report, _coverage(uncovered=1, extra_uncovered=2)) args = _args(tmp_path, book_output=book_output, score_dir=score_dir) args.review_coverage_report = coverage_report report = tool.admit_candidate(args) assert report["admitted_operational"] is False assert report["review_coverage_values"]["uncovered_or_weak_count"] == 1 assert report["review_coverage_values"]["extra_uncovered_or_weak_count"] == 2 assert any("review coverage uncovered_or_weak_count above maximum" in issue for issue in report["issues"]) assert any("review coverage extra_uncovered_or_weak_count above maximum" in issue for issue in report["issues"]) def test_admit_limbu_book_ocr_candidate_accepts_candidate_application_audit(tmp_path: Path) -> None: tool = _load_tool() book_output = tmp_path / "candidate-application" score_dir = tmp_path / "score" application_audit = book_output / "candidate-line-application-audit.json" _write_json( application_audit, { "audit_passed": True, "output_dir": str(book_output), "tool": "tools/audit_candidate_line_review_application.py", }, ) _write_json(score_dir / "metrics.json", _metrics(codepoint_cer=0.446)) args = _args(tmp_path, book_output=book_output, score_dir=score_dir) args.book_audit = application_audit report = tool.admit_candidate(args) assert report["admitted_operational"] is True assert report["book_audit_tool"] == "tools/audit_candidate_line_review_application.py" def test_admit_limbu_book_ocr_candidate_accepts_unified_review_application_audit(tmp_path: Path) -> None: tool = _load_tool() application_dir = tmp_path / "application" book_output = application_dir / "final-book" score_dir = tmp_path / "score" _write_json(book_output / "book-run.json", {"book": "fixture"}) final_book_run_sha = tool.sha256_file(book_output / "book-run.json") application_audit = application_dir / "book-unresolved-review-packet-application-audit.json" _write_json( application_audit, { "audit_passed": True, "application_dir": str(application_dir), "tool": "tools/audit_limbu_book_unresolved_review_packet_application.py", "final_book_run_sha256": final_book_run_sha, }, ) _write_json(score_dir / "metrics.json", _metrics(codepoint_cer=0.446)) args = _args(tmp_path, book_output=book_output, score_dir=score_dir) args.book_audit = application_audit report = tool.admit_candidate(args) assert report["admitted_operational"] is True assert report["book_audit_tool"] == "tools/audit_limbu_book_unresolved_review_packet_application.py" def test_admit_limbu_book_ocr_candidate_rejects_oracle_candidate_application_audit(tmp_path: Path) -> None: tool = _load_tool() book_output = tmp_path / "candidate-application" score_dir = tmp_path / "score" application_audit = book_output / "candidate-line-application-audit.json" _write_json( application_audit, { "audit_passed": True, "output_dir": str(book_output), "tool": "tools/audit_candidate_line_review_application.py", "operational_review_safe": False, "reference_oracle_review_count": 1, }, ) _write_json(score_dir / "metrics.json", _metrics(codepoint_cer=0.42)) args = _args(tmp_path, book_output=book_output, score_dir=score_dir) args.book_audit = application_audit report = tool.admit_candidate(args) assert report["admitted_operational"] is False assert any("non-operational" in issue for issue in report["issues"]) assert any("oracle/reference-assisted" in issue for issue in report["issues"])