Download ocr-tech/tests/test_admit_limbu_book_ocr_candidate.py from prosperious/ocr-projects-archive: direct link, hf CLI and curl.
- Browser
- Download file 12.8 kB
-
https://huggingface.co/prosperious/ocr-projects-archive/resolve/main/ocr-tech/tests/test_admit_limbu_book_ocr_candidate.py
- Command line
-
hf download hf://prosperious/ocr-projects-archive/ocr-tech/tests/test_admit_limbu_book_ocr_candidate.py
-
curl -L -o test_admit_limbu_book_ocr_candidate.py https://huggingface.co/prosperious/ocr-projects-archive/resolve/main/ocr-tech/tests/test_admit_limbu_book_ocr_candidate.py
12.8 kB
| from __future__ import annotations | |
| import importlib.util | |
| import json | |
| import sys | |
| from argparse import Namespace | |
| from pathlib import Path | |
| def _load_tool(): | |
| path = Path(__file__).resolve().parents[1] / "tools" / "admit_limbu_book_ocr_candidate.py" | |
| spec = importlib.util.spec_from_file_location("admit_limbu_book_ocr_candidate", path) | |
| assert spec is not None | |
| assert spec.loader is not None | |
| module = importlib.util.module_from_spec(spec) | |
| sys.modules[spec.name] = module | |
| spec.loader.exec_module(module) | |
| return module | |
| def _write_json(path: Path, payload: object) -> None: | |
| path.parent.mkdir(parents=True, exist_ok=True) | |
| path.write_text(json.dumps(payload, ensure_ascii=False, indent=2) + "\n", encoding="utf-8") | |
| def _metrics(*, codepoint_cer: float, grapheme_cer: float = 0.6, wer: float = 0.8) -> dict[str, object]: | |
| return { | |
| "manifest_samples": 3, | |
| "scored_pages": 3, | |
| "claim_eligible_rows": 0, | |
| "missing_documents": [], | |
| "missing_references": [], | |
| "metric_means": {"text_wer": wer}, | |
| "text_cer_codepoint_micro": codepoint_cer, | |
| "text_cer_grapheme_micro": grapheme_cer, | |
| "line_detection_precision_micro": 0.7, | |
| "line_detection_recall_micro": 0.75, | |
| "line_detection_f1_micro": 0.72, | |
| "matched_line_reading_order_pair_accuracy": 0.61, | |
| "line_blank_prediction_rate": 0.0, | |
| "line_extra_prediction_rate": 0.27, | |
| "line_missed_reference_rate": 0.22, | |
| } | |
| def _coverage(*, uncovered: int = 0, extra_uncovered: int = 0) -> dict[str, object]: | |
| return { | |
| "tool": "tools/audit_limbu_book_missed_line_review_coverage.py", | |
| "missed_reference_count": 2, | |
| "uncovered_or_weak_count": uncovered, | |
| "extra_prediction_count": 3, | |
| "extra_uncovered_or_weak_count": extra_uncovered, | |
| } | |
| def _args(tmp_path: Path, *, book_output: Path, score_dir: Path) -> Namespace: | |
| return Namespace( | |
| book_output=book_output, | |
| score_dir=score_dir, | |
| out=tmp_path / "admission", | |
| book_audit=None, | |
| metrics_sha256=None, | |
| book_audit_sha256=None, | |
| champion_metrics=None, | |
| review_coverage_report=None, | |
| baseline_codepoint_cer=0.458432, | |
| baseline_grapheme_cer=None, | |
| baseline_wer=None, | |
| min_codepoint_improvement=0.0, | |
| max_grapheme_regression=0.0, | |
| max_wer_regression=0.0, | |
| max_champion_codepoint_regression=0.0, | |
| max_champion_grapheme_regression=0.0, | |
| max_champion_wer_regression=0.0, | |
| max_review_uncovered_or_weak_count=0, | |
| max_review_extra_uncovered_or_weak_count=0, | |
| min_line_f1=0.70, | |
| min_reading_order_accuracy=0.50, | |
| max_blank_line_rate=0.0, | |
| max_extra_line_rate=0.30, | |
| max_missed_line_rate=0.30, | |
| min_scored_pages=3, | |
| min_claim_pages=30, | |
| ) | |
| def test_admit_limbu_book_ocr_candidate_passes_operational_gate(tmp_path: Path) -> None: | |
| tool = _load_tool() | |
| book_output = tmp_path / "book" | |
| score_dir = tmp_path / "score" | |
| _write_json(book_output / "limbu-book-output-audit.json", {"passed": True, "output_dir": str(book_output)}) | |
| _write_json(score_dir / "metrics.json", _metrics(codepoint_cer=0.446)) | |
| report = tool.admit_candidate(_args(tmp_path, book_output=book_output, score_dir=score_dir)) | |
| assert report["admitted_operational"] is True | |
| assert report["claim_safe"] is False | |
| assert report["claim_safe_reasons"] == ["not all scored rows are claim eligible", "scored_pages below claim minimum 30"] | |
| assert report["codepoint_absolute_improvement"] == 0.458432 - 0.446 | |
| assert (tmp_path / "admission" / "limbu-book-ocr-candidate-admission.json").exists() | |
| assert (tmp_path / "admission" / "limbu-book-ocr-candidate-admission.md").exists() | |
| def test_admit_limbu_book_ocr_candidate_rejects_codepoint_regression(tmp_path: Path) -> None: | |
| tool = _load_tool() | |
| book_output = tmp_path / "book" | |
| score_dir = tmp_path / "score" | |
| _write_json(book_output / "limbu-book-output-audit.json", {"passed": True, "output_dir": str(book_output)}) | |
| _write_json(score_dir / "metrics.json", _metrics(codepoint_cer=0.46)) | |
| report = tool.admit_candidate(_args(tmp_path, book_output=book_output, score_dir=score_dir)) | |
| assert report["admitted_operational"] is False | |
| assert any("codepoint CER improvement below requirement" in issue for issue in report["issues"]) | |
| def test_admit_limbu_book_ocr_candidate_rejects_grapheme_and_wer_regressions(tmp_path: Path) -> None: | |
| tool = _load_tool() | |
| book_output = tmp_path / "book" | |
| score_dir = tmp_path / "score" | |
| _write_json(book_output / "limbu-book-output-audit.json", {"passed": True, "output_dir": str(book_output)}) | |
| _write_json(score_dir / "metrics.json", _metrics(codepoint_cer=0.446, grapheme_cer=0.64, wer=0.87)) | |
| args = _args(tmp_path, book_output=book_output, score_dir=score_dir) | |
| args.baseline_grapheme_cer = 0.62 | |
| args.baseline_wer = 0.84 | |
| report = tool.admit_candidate(args) | |
| assert report["admitted_operational"] is False | |
| assert report["grapheme_absolute_improvement"] == 0.62 - 0.64 | |
| assert report["wer_absolute_improvement"] == 0.84 - 0.87 | |
| assert any("grapheme CER regression above allowed maximum" in issue for issue in report["issues"]) | |
| assert any("WER regression above allowed maximum" in issue for issue in report["issues"]) | |
| def test_admit_limbu_book_ocr_candidate_rejects_champion_text_regression(tmp_path: Path) -> None: | |
| tool = _load_tool() | |
| book_output = tmp_path / "book" | |
| score_dir = tmp_path / "score" | |
| champion_metrics = tmp_path / "champion" / "metrics.json" | |
| _write_json(book_output / "limbu-book-output-audit.json", {"passed": True, "output_dir": str(book_output)}) | |
| _write_json(score_dir / "metrics.json", _metrics(codepoint_cer=0.44, grapheme_cer=0.56, wer=0.74)) | |
| _write_json(champion_metrics, _metrics(codepoint_cer=0.40, grapheme_cer=0.51, wer=0.71)) | |
| args = _args(tmp_path, book_output=book_output, score_dir=score_dir) | |
| args.champion_metrics = champion_metrics | |
| report = tool.admit_candidate(args) | |
| assert report["admitted_operational"] is False | |
| assert report["champion_codepoint_absolute_improvement"] == 0.40 - 0.44 | |
| assert report["champion_grapheme_absolute_improvement"] == 0.51 - 0.56 | |
| assert report["champion_wer_absolute_improvement"] == 0.71 - 0.74 | |
| assert any("champion codepoint CER regression" in issue for issue in report["issues"]) | |
| assert any("champion grapheme CER regression" in issue for issue in report["issues"]) | |
| assert any("champion WER regression" in issue for issue in report["issues"]) | |
| def test_admit_limbu_book_ocr_candidate_accepts_champion_tie(tmp_path: Path) -> None: | |
| tool = _load_tool() | |
| book_output = tmp_path / "book" | |
| score_dir = tmp_path / "score" | |
| champion_metrics = tmp_path / "champion" / "metrics.json" | |
| _write_json(book_output / "limbu-book-output-audit.json", {"passed": True, "output_dir": str(book_output)}) | |
| _write_json(score_dir / "metrics.json", _metrics(codepoint_cer=0.40, grapheme_cer=0.51, wer=0.71)) | |
| _write_json(champion_metrics, _metrics(codepoint_cer=0.40, grapheme_cer=0.51, wer=0.71)) | |
| args = _args(tmp_path, book_output=book_output, score_dir=score_dir) | |
| args.champion_metrics = champion_metrics | |
| report = tool.admit_candidate(args) | |
| assert report["admitted_operational"] is True | |
| assert report["champion_metric_values"]["text_cer_codepoint_micro"] == 0.40 | |
| assert report["champion_codepoint_absolute_improvement"] == 0.0 | |
| def test_admit_limbu_book_ocr_candidate_accepts_clean_review_coverage(tmp_path: Path) -> None: | |
| tool = _load_tool() | |
| book_output = tmp_path / "book" | |
| score_dir = tmp_path / "score" | |
| coverage_report = tmp_path / "coverage" / "missed-line-review-coverage.json" | |
| _write_json(book_output / "limbu-book-output-audit.json", {"passed": True, "output_dir": str(book_output)}) | |
| _write_json(score_dir / "metrics.json", _metrics(codepoint_cer=0.446)) | |
| _write_json(coverage_report, _coverage(uncovered=0, extra_uncovered=0)) | |
| args = _args(tmp_path, book_output=book_output, score_dir=score_dir) | |
| args.review_coverage_report = coverage_report | |
| report = tool.admit_candidate(args) | |
| assert report["admitted_operational"] is True | |
| assert report["review_coverage_values"]["uncovered_or_weak_count"] == 0 | |
| assert report["review_coverage_values"]["extra_uncovered_or_weak_count"] == 0 | |
| def test_admit_limbu_book_ocr_candidate_rejects_uncovered_review_regions(tmp_path: Path) -> None: | |
| tool = _load_tool() | |
| book_output = tmp_path / "book" | |
| score_dir = tmp_path / "score" | |
| coverage_report = tmp_path / "coverage" / "missed-line-review-coverage.json" | |
| _write_json(book_output / "limbu-book-output-audit.json", {"passed": True, "output_dir": str(book_output)}) | |
| _write_json(score_dir / "metrics.json", _metrics(codepoint_cer=0.446)) | |
| _write_json(coverage_report, _coverage(uncovered=1, extra_uncovered=2)) | |
| args = _args(tmp_path, book_output=book_output, score_dir=score_dir) | |
| args.review_coverage_report = coverage_report | |
| report = tool.admit_candidate(args) | |
| assert report["admitted_operational"] is False | |
| assert report["review_coverage_values"]["uncovered_or_weak_count"] == 1 | |
| assert report["review_coverage_values"]["extra_uncovered_or_weak_count"] == 2 | |
| assert any("review coverage uncovered_or_weak_count above maximum" in issue for issue in report["issues"]) | |
| assert any("review coverage extra_uncovered_or_weak_count above maximum" in issue for issue in report["issues"]) | |
| def test_admit_limbu_book_ocr_candidate_accepts_candidate_application_audit(tmp_path: Path) -> None: | |
| tool = _load_tool() | |
| book_output = tmp_path / "candidate-application" | |
| score_dir = tmp_path / "score" | |
| application_audit = book_output / "candidate-line-application-audit.json" | |
| _write_json( | |
| application_audit, | |
| { | |
| "audit_passed": True, | |
| "output_dir": str(book_output), | |
| "tool": "tools/audit_candidate_line_review_application.py", | |
| }, | |
| ) | |
| _write_json(score_dir / "metrics.json", _metrics(codepoint_cer=0.446)) | |
| args = _args(tmp_path, book_output=book_output, score_dir=score_dir) | |
| args.book_audit = application_audit | |
| report = tool.admit_candidate(args) | |
| assert report["admitted_operational"] is True | |
| assert report["book_audit_tool"] == "tools/audit_candidate_line_review_application.py" | |
| def test_admit_limbu_book_ocr_candidate_accepts_unified_review_application_audit(tmp_path: Path) -> None: | |
| tool = _load_tool() | |
| application_dir = tmp_path / "application" | |
| book_output = application_dir / "final-book" | |
| score_dir = tmp_path / "score" | |
| _write_json(book_output / "book-run.json", {"book": "fixture"}) | |
| final_book_run_sha = tool.sha256_file(book_output / "book-run.json") | |
| application_audit = application_dir / "book-unresolved-review-packet-application-audit.json" | |
| _write_json( | |
| application_audit, | |
| { | |
| "audit_passed": True, | |
| "application_dir": str(application_dir), | |
| "tool": "tools/audit_limbu_book_unresolved_review_packet_application.py", | |
| "final_book_run_sha256": final_book_run_sha, | |
| }, | |
| ) | |
| _write_json(score_dir / "metrics.json", _metrics(codepoint_cer=0.446)) | |
| args = _args(tmp_path, book_output=book_output, score_dir=score_dir) | |
| args.book_audit = application_audit | |
| report = tool.admit_candidate(args) | |
| assert report["admitted_operational"] is True | |
| assert report["book_audit_tool"] == "tools/audit_limbu_book_unresolved_review_packet_application.py" | |
| def test_admit_limbu_book_ocr_candidate_rejects_oracle_candidate_application_audit(tmp_path: Path) -> None: | |
| tool = _load_tool() | |
| book_output = tmp_path / "candidate-application" | |
| score_dir = tmp_path / "score" | |
| application_audit = book_output / "candidate-line-application-audit.json" | |
| _write_json( | |
| application_audit, | |
| { | |
| "audit_passed": True, | |
| "output_dir": str(book_output), | |
| "tool": "tools/audit_candidate_line_review_application.py", | |
| "operational_review_safe": False, | |
| "reference_oracle_review_count": 1, | |
| }, | |
| ) | |
| _write_json(score_dir / "metrics.json", _metrics(codepoint_cer=0.42)) | |
| args = _args(tmp_path, book_output=book_output, score_dir=score_dir) | |
| args.book_audit = application_audit | |
| report = tool.admit_candidate(args) | |
| assert report["admitted_operational"] is False | |
| assert any("non-operational" in issue for issue in report["issues"]) | |
| assert any("oracle/reference-assisted" in issue for issue in report["issues"]) | |