ocr-projects-archive / ocr-tech /tests /test_admit_limbu_book_ocr_candidate.py
prosperious's picture
Add 1000 files (OCR_INDRA/.gitignore ...)
f9bc0c1 verified
Raw History Blame Contribute Delete
12.8 kB
from __future__ import annotations
import importlib.util
import json
import sys
from argparse import Namespace
from pathlib import Path
def _load_tool():
path = Path(__file__).resolve().parents[1] / "tools" / "admit_limbu_book_ocr_candidate.py"
spec = importlib.util.spec_from_file_location("admit_limbu_book_ocr_candidate", path)
assert spec is not None
assert spec.loader is not None
module = importlib.util.module_from_spec(spec)
sys.modules[spec.name] = module
spec.loader.exec_module(module)
return module
def _write_json(path: Path, payload: object) -> None:
path.parent.mkdir(parents=True, exist_ok=True)
path.write_text(json.dumps(payload, ensure_ascii=False, indent=2) + "\n", encoding="utf-8")
def _metrics(*, codepoint_cer: float, grapheme_cer: float = 0.6, wer: float = 0.8) -> dict[str, object]:
return {
"manifest_samples": 3,
"scored_pages": 3,
"claim_eligible_rows": 0,
"missing_documents": [],
"missing_references": [],
"metric_means": {"text_wer": wer},
"text_cer_codepoint_micro": codepoint_cer,
"text_cer_grapheme_micro": grapheme_cer,
"line_detection_precision_micro": 0.7,
"line_detection_recall_micro": 0.75,
"line_detection_f1_micro": 0.72,
"matched_line_reading_order_pair_accuracy": 0.61,
"line_blank_prediction_rate": 0.0,
"line_extra_prediction_rate": 0.27,
"line_missed_reference_rate": 0.22,
}
def _coverage(*, uncovered: int = 0, extra_uncovered: int = 0) -> dict[str, object]:
return {
"tool": "tools/audit_limbu_book_missed_line_review_coverage.py",
"missed_reference_count": 2,
"uncovered_or_weak_count": uncovered,
"extra_prediction_count": 3,
"extra_uncovered_or_weak_count": extra_uncovered,
}
def _args(tmp_path: Path, *, book_output: Path, score_dir: Path) -> Namespace:
return Namespace(
book_output=book_output,
score_dir=score_dir,
out=tmp_path / "admission",
book_audit=None,
metrics_sha256=None,
book_audit_sha256=None,
champion_metrics=None,
review_coverage_report=None,
baseline_codepoint_cer=0.458432,
baseline_grapheme_cer=None,
baseline_wer=None,
min_codepoint_improvement=0.0,
max_grapheme_regression=0.0,
max_wer_regression=0.0,
max_champion_codepoint_regression=0.0,
max_champion_grapheme_regression=0.0,
max_champion_wer_regression=0.0,
max_review_uncovered_or_weak_count=0,
max_review_extra_uncovered_or_weak_count=0,
min_line_f1=0.70,
min_reading_order_accuracy=0.50,
max_blank_line_rate=0.0,
max_extra_line_rate=0.30,
max_missed_line_rate=0.30,
min_scored_pages=3,
min_claim_pages=30,
)
def test_admit_limbu_book_ocr_candidate_passes_operational_gate(tmp_path: Path) -> None:
tool = _load_tool()
book_output = tmp_path / "book"
score_dir = tmp_path / "score"
_write_json(book_output / "limbu-book-output-audit.json", {"passed": True, "output_dir": str(book_output)})
_write_json(score_dir / "metrics.json", _metrics(codepoint_cer=0.446))
report = tool.admit_candidate(_args(tmp_path, book_output=book_output, score_dir=score_dir))
assert report["admitted_operational"] is True
assert report["claim_safe"] is False
assert report["claim_safe_reasons"] == ["not all scored rows are claim eligible", "scored_pages below claim minimum 30"]
assert report["codepoint_absolute_improvement"] == 0.458432 - 0.446
assert (tmp_path / "admission" / "limbu-book-ocr-candidate-admission.json").exists()
assert (tmp_path / "admission" / "limbu-book-ocr-candidate-admission.md").exists()
def test_admit_limbu_book_ocr_candidate_rejects_codepoint_regression(tmp_path: Path) -> None:
tool = _load_tool()
book_output = tmp_path / "book"
score_dir = tmp_path / "score"
_write_json(book_output / "limbu-book-output-audit.json", {"passed": True, "output_dir": str(book_output)})
_write_json(score_dir / "metrics.json", _metrics(codepoint_cer=0.46))
report = tool.admit_candidate(_args(tmp_path, book_output=book_output, score_dir=score_dir))
assert report["admitted_operational"] is False
assert any("codepoint CER improvement below requirement" in issue for issue in report["issues"])
def test_admit_limbu_book_ocr_candidate_rejects_grapheme_and_wer_regressions(tmp_path: Path) -> None:
tool = _load_tool()
book_output = tmp_path / "book"
score_dir = tmp_path / "score"
_write_json(book_output / "limbu-book-output-audit.json", {"passed": True, "output_dir": str(book_output)})
_write_json(score_dir / "metrics.json", _metrics(codepoint_cer=0.446, grapheme_cer=0.64, wer=0.87))
args = _args(tmp_path, book_output=book_output, score_dir=score_dir)
args.baseline_grapheme_cer = 0.62
args.baseline_wer = 0.84
report = tool.admit_candidate(args)
assert report["admitted_operational"] is False
assert report["grapheme_absolute_improvement"] == 0.62 - 0.64
assert report["wer_absolute_improvement"] == 0.84 - 0.87
assert any("grapheme CER regression above allowed maximum" in issue for issue in report["issues"])
assert any("WER regression above allowed maximum" in issue for issue in report["issues"])
def test_admit_limbu_book_ocr_candidate_rejects_champion_text_regression(tmp_path: Path) -> None:
tool = _load_tool()
book_output = tmp_path / "book"
score_dir = tmp_path / "score"
champion_metrics = tmp_path / "champion" / "metrics.json"
_write_json(book_output / "limbu-book-output-audit.json", {"passed": True, "output_dir": str(book_output)})
_write_json(score_dir / "metrics.json", _metrics(codepoint_cer=0.44, grapheme_cer=0.56, wer=0.74))
_write_json(champion_metrics, _metrics(codepoint_cer=0.40, grapheme_cer=0.51, wer=0.71))
args = _args(tmp_path, book_output=book_output, score_dir=score_dir)
args.champion_metrics = champion_metrics
report = tool.admit_candidate(args)
assert report["admitted_operational"] is False
assert report["champion_codepoint_absolute_improvement"] == 0.40 - 0.44
assert report["champion_grapheme_absolute_improvement"] == 0.51 - 0.56
assert report["champion_wer_absolute_improvement"] == 0.71 - 0.74
assert any("champion codepoint CER regression" in issue for issue in report["issues"])
assert any("champion grapheme CER regression" in issue for issue in report["issues"])
assert any("champion WER regression" in issue for issue in report["issues"])
def test_admit_limbu_book_ocr_candidate_accepts_champion_tie(tmp_path: Path) -> None:
tool = _load_tool()
book_output = tmp_path / "book"
score_dir = tmp_path / "score"
champion_metrics = tmp_path / "champion" / "metrics.json"
_write_json(book_output / "limbu-book-output-audit.json", {"passed": True, "output_dir": str(book_output)})
_write_json(score_dir / "metrics.json", _metrics(codepoint_cer=0.40, grapheme_cer=0.51, wer=0.71))
_write_json(champion_metrics, _metrics(codepoint_cer=0.40, grapheme_cer=0.51, wer=0.71))
args = _args(tmp_path, book_output=book_output, score_dir=score_dir)
args.champion_metrics = champion_metrics
report = tool.admit_candidate(args)
assert report["admitted_operational"] is True
assert report["champion_metric_values"]["text_cer_codepoint_micro"] == 0.40
assert report["champion_codepoint_absolute_improvement"] == 0.0
def test_admit_limbu_book_ocr_candidate_accepts_clean_review_coverage(tmp_path: Path) -> None:
tool = _load_tool()
book_output = tmp_path / "book"
score_dir = tmp_path / "score"
coverage_report = tmp_path / "coverage" / "missed-line-review-coverage.json"
_write_json(book_output / "limbu-book-output-audit.json", {"passed": True, "output_dir": str(book_output)})
_write_json(score_dir / "metrics.json", _metrics(codepoint_cer=0.446))
_write_json(coverage_report, _coverage(uncovered=0, extra_uncovered=0))
args = _args(tmp_path, book_output=book_output, score_dir=score_dir)
args.review_coverage_report = coverage_report
report = tool.admit_candidate(args)
assert report["admitted_operational"] is True
assert report["review_coverage_values"]["uncovered_or_weak_count"] == 0
assert report["review_coverage_values"]["extra_uncovered_or_weak_count"] == 0
def test_admit_limbu_book_ocr_candidate_rejects_uncovered_review_regions(tmp_path: Path) -> None:
tool = _load_tool()
book_output = tmp_path / "book"
score_dir = tmp_path / "score"
coverage_report = tmp_path / "coverage" / "missed-line-review-coverage.json"
_write_json(book_output / "limbu-book-output-audit.json", {"passed": True, "output_dir": str(book_output)})
_write_json(score_dir / "metrics.json", _metrics(codepoint_cer=0.446))
_write_json(coverage_report, _coverage(uncovered=1, extra_uncovered=2))
args = _args(tmp_path, book_output=book_output, score_dir=score_dir)
args.review_coverage_report = coverage_report
report = tool.admit_candidate(args)
assert report["admitted_operational"] is False
assert report["review_coverage_values"]["uncovered_or_weak_count"] == 1
assert report["review_coverage_values"]["extra_uncovered_or_weak_count"] == 2
assert any("review coverage uncovered_or_weak_count above maximum" in issue for issue in report["issues"])
assert any("review coverage extra_uncovered_or_weak_count above maximum" in issue for issue in report["issues"])
def test_admit_limbu_book_ocr_candidate_accepts_candidate_application_audit(tmp_path: Path) -> None:
tool = _load_tool()
book_output = tmp_path / "candidate-application"
score_dir = tmp_path / "score"
application_audit = book_output / "candidate-line-application-audit.json"
_write_json(
application_audit,
{
"audit_passed": True,
"output_dir": str(book_output),
"tool": "tools/audit_candidate_line_review_application.py",
},
)
_write_json(score_dir / "metrics.json", _metrics(codepoint_cer=0.446))
args = _args(tmp_path, book_output=book_output, score_dir=score_dir)
args.book_audit = application_audit
report = tool.admit_candidate(args)
assert report["admitted_operational"] is True
assert report["book_audit_tool"] == "tools/audit_candidate_line_review_application.py"
def test_admit_limbu_book_ocr_candidate_accepts_unified_review_application_audit(tmp_path: Path) -> None:
tool = _load_tool()
application_dir = tmp_path / "application"
book_output = application_dir / "final-book"
score_dir = tmp_path / "score"
_write_json(book_output / "book-run.json", {"book": "fixture"})
final_book_run_sha = tool.sha256_file(book_output / "book-run.json")
application_audit = application_dir / "book-unresolved-review-packet-application-audit.json"
_write_json(
application_audit,
{
"audit_passed": True,
"application_dir": str(application_dir),
"tool": "tools/audit_limbu_book_unresolved_review_packet_application.py",
"final_book_run_sha256": final_book_run_sha,
},
)
_write_json(score_dir / "metrics.json", _metrics(codepoint_cer=0.446))
args = _args(tmp_path, book_output=book_output, score_dir=score_dir)
args.book_audit = application_audit
report = tool.admit_candidate(args)
assert report["admitted_operational"] is True
assert report["book_audit_tool"] == "tools/audit_limbu_book_unresolved_review_packet_application.py"
def test_admit_limbu_book_ocr_candidate_rejects_oracle_candidate_application_audit(tmp_path: Path) -> None:
tool = _load_tool()
book_output = tmp_path / "candidate-application"
score_dir = tmp_path / "score"
application_audit = book_output / "candidate-line-application-audit.json"
_write_json(
application_audit,
{
"audit_passed": True,
"output_dir": str(book_output),
"tool": "tools/audit_candidate_line_review_application.py",
"operational_review_safe": False,
"reference_oracle_review_count": 1,
},
)
_write_json(score_dir / "metrics.json", _metrics(codepoint_cer=0.42))
args = _args(tmp_path, book_output=book_output, score_dir=score_dir)
args.book_audit = application_audit
report = tool.admit_candidate(args)
assert report["admitted_operational"] is False
assert any("non-operational" in issue for issue in report["issues"])
assert any("oracle/reference-assisted" in issue for issue in report["issues"])