ocr-projects-archive / ocr-tech /tools /admit_limbu_book_ocr_candidate.py
prosperious's picture
Add 1000 files (ocr-tech/tests/test_audit_limbu_book_missed_line_review_coverage.py ...)
d82db76 verified
Raw History Blame Contribute Delete
23.8 kB
#!/usr/bin/env python3
"""Admission gate for a Limbu book OCR candidate run."""
from __future__ import annotations
import argparse
import json
import platform
from datetime import UTC, datetime
from pathlib import Path
from typing import Any
from ocrtech.manifest import sha256_file
REPORT_JSON = "limbu-book-ocr-candidate-admission.json"
REPORT_MD = "limbu-book-ocr-candidate-admission.md"
REQUIRED_METRICS = {
"text_cer_codepoint_micro": "page codepoint CER",
"text_cer_grapheme_micro": "page grapheme CER",
"metric_means.text_wer": "page WER",
"line_detection_precision_micro": "line detection precision",
"line_detection_recall_micro": "line detection recall",
"line_detection_f1_micro": "line detection F1",
"matched_line_reading_order_pair_accuracy": "reading-order pair accuracy",
"line_blank_prediction_rate": "blank-line rate",
"line_extra_prediction_rate": "extra-line rate",
"line_missed_reference_rate": "missed-line rate",
}
CHAMPION_TEXT_METRICS = {
"text_cer_codepoint_micro": "codepoint CER",
"text_cer_grapheme_micro": "grapheme CER",
"metric_means.text_wer": "WER",
}
def _read_json(path: Path, label: str) -> dict[str, Any]:
if not path.is_file():
raise FileNotFoundError(f"{label} does not exist: {path}")
try:
payload = json.loads(path.read_text(encoding="utf-8"))
except json.JSONDecodeError as exc:
raise ValueError(f"{label} is invalid JSON: {path}: {exc}") from exc
if not isinstance(payload, dict):
raise ValueError(f"{label} must contain a JSON object: {path}")
return payload
def _write_json(path: Path, payload: dict[str, Any]) -> None:
path.parent.mkdir(parents=True, exist_ok=True)
path.write_text(json.dumps(payload, ensure_ascii=False, indent=2) + "\n", encoding="utf-8")
def _metric(metrics: dict[str, Any], dotted_key: str) -> Any:
value: Any = metrics
for part in dotted_key.split("."):
if not isinstance(value, dict) or part not in value:
return None
value = value[part]
return value
def _float_metric(metrics: dict[str, Any], dotted_key: str, issues: list[str]) -> float | None:
value = _metric(metrics, dotted_key)
try:
parsed = float(value)
except (TypeError, ValueError):
issues.append(f"required metric {dotted_key} is missing or non-numeric")
return None
return parsed
def _champion_float_metric(metrics: dict[str, Any], dotted_key: str, issues: list[str]) -> float | None:
value = _metric(metrics, dotted_key)
try:
return float(value)
except (TypeError, ValueError):
issues.append(f"champion metric {dotted_key} is missing or non-numeric")
return None
def _optional_float(value: Any) -> float | None:
try:
return float(value)
except (TypeError, ValueError):
return None
def _check_hash(path: Path, expected: str | None, issues: list[str], *, label: str) -> str | None:
if not path.is_file():
issues.append(f"{label} is missing: {path}")
return None
actual = sha256_file(path)
if expected is not None and actual != expected:
issues.append(f"{label} SHA-256 mismatch: recorded={expected} actual={actual}")
return actual
def _audit_passed(audit: dict[str, Any]) -> bool:
if "passed" in audit:
return audit.get("passed") is True
return audit.get("audit_passed") is True
def _audit_matches_book_output(book_audit: dict[str, Any], book_output: Path, issues: list[str]) -> None:
recorded_output = str(book_audit.get("output_dir") or "")
if recorded_output == str(book_output):
return
if book_audit.get("tool") != "tools/audit_limbu_book_unresolved_review_packet_application.py":
issues.append("book/application output audit output_dir does not match candidate output")
return
final_output = str(book_audit.get("final_book_output_dir") or "")
application_dir = str(book_audit.get("application_dir") or "")
expected_final_output = str(Path(application_dir) / "final-book") if application_dir else ""
if final_output and final_output != str(book_output):
issues.append("unified review-application audit final_book_output_dir does not match candidate output")
return
if not final_output and expected_final_output != str(book_output):
issues.append("unified review-application audit application_dir/final-book does not match candidate output")
return
final_book_run = book_output / "book-run.json"
final_book_run_sha = _check_hash(
final_book_run,
book_audit.get("final_book_run_sha256"),
issues,
label="unified review-application final book-run.json",
)
if final_book_run_sha is None:
return
def _claim_safe(metrics: dict[str, Any], args: argparse.Namespace) -> tuple[bool, list[str]]:
reasons: list[str] = []
claim_rows = _optional_float(metrics.get("claim_eligible_rows"))
manifest_samples = _optional_float(metrics.get("manifest_samples"))
scored_pages = _optional_float(metrics.get("scored_pages"))
if claim_rows is None or manifest_samples is None or claim_rows != manifest_samples:
reasons.append("not all scored rows are claim eligible")
if scored_pages is None or scored_pages < int(args.min_claim_pages):
reasons.append(f"scored_pages below claim minimum {args.min_claim_pages}")
return not reasons, reasons
def _check_champion_text_regression(
*,
candidate_value: float | None,
champion_value: float | None,
max_regression: float,
label: str,
issues: list[str],
) -> float | None:
improvement = champion_value - candidate_value if champion_value is not None and candidate_value is not None else None
if improvement is not None and improvement < -max_regression:
issues.append(
f"champion {label} regression above allowed maximum: champion={champion_value} "
f"candidate={candidate_value} improvement={improvement} max_regression={max_regression}"
)
return improvement
def _check_review_coverage(
*,
report: dict[str, Any],
max_uncovered_or_weak_count: int,
max_extra_uncovered_or_weak_count: int,
issues: list[str],
) -> dict[str, int | None]:
values: dict[str, int | None] = {}
required_integer_fields = [
"missed_reference_count",
"uncovered_or_weak_count",
"extra_prediction_count",
"extra_uncovered_or_weak_count",
]
for field in required_integer_fields:
parsed = _optional_float(report.get(field))
if parsed is None or int(parsed) != parsed:
issues.append(f"review coverage report field {field} is missing or non-integer")
values[field] = None
else:
values[field] = int(parsed)
if report.get("tool") != "tools/audit_limbu_book_missed_line_review_coverage.py":
issues.append(f"unexpected review coverage report tool: {report.get('tool')!r}")
uncovered = values.get("uncovered_or_weak_count")
if uncovered is None or uncovered > max_uncovered_or_weak_count:
issues.append(
f"review coverage uncovered_or_weak_count above maximum: "
f"candidate={uncovered} max={max_uncovered_or_weak_count}"
)
extra_uncovered = values.get("extra_uncovered_or_weak_count")
if extra_uncovered is None or extra_uncovered > max_extra_uncovered_or_weak_count:
issues.append(
f"review coverage extra_uncovered_or_weak_count above maximum: "
f"candidate={extra_uncovered} max={max_extra_uncovered_or_weak_count}"
)
return values
def admit_candidate(args: argparse.Namespace) -> dict[str, Any]:
out = Path(args.out)
out.mkdir(parents=True, exist_ok=True)
book_output = Path(args.book_output)
score_dir = Path(args.score_dir)
metrics_path = score_dir / "metrics.json"
book_audit_path = Path(args.book_audit) if args.book_audit else book_output / "limbu-book-output-audit.json"
champion_metrics_path = Path(args.champion_metrics) if getattr(args, "champion_metrics", None) else None
review_coverage_path = Path(args.review_coverage_report) if getattr(args, "review_coverage_report", None) else None
issues: list[str] = []
warnings: list[str] = []
metrics = _read_json(metrics_path, "score metrics")
champion_metrics = _read_json(champion_metrics_path, "champion score metrics") if champion_metrics_path else None
review_coverage = _read_json(review_coverage_path, "review coverage report") if review_coverage_path else None
book_audit = _read_json(book_audit_path, "book/application output audit")
if not _audit_passed(book_audit):
issues.append("book/application output audit did not pass")
if book_audit.get("operational_review_safe") is False:
issues.append("book/application output audit marks review application as non-operational")
reference_oracle_count = _optional_float(book_audit.get("reference_oracle_review_count"))
if reference_oracle_count is not None and reference_oracle_count > 0:
issues.append(f"candidate-line application contains oracle/reference-assisted review rows: {int(reference_oracle_count)}")
_audit_matches_book_output(book_audit, book_output, issues)
metrics_sha = _check_hash(metrics_path, args.metrics_sha256, issues, label="metrics.json")
champion_metrics_sha = (
_check_hash(champion_metrics_path, None, issues, label="champion metrics.json")
if champion_metrics_path is not None
else None
)
review_coverage_sha = (
_check_hash(review_coverage_path, None, issues, label="review coverage report")
if review_coverage_path is not None
else None
)
book_audit_sha = _check_hash(book_audit_path, args.book_audit_sha256, issues, label="book/application output audit")
metric_values: dict[str, float | None] = {}
for key in REQUIRED_METRICS:
metric_values[key] = _float_metric(metrics, key, issues)
codepoint_cer = metric_values["text_cer_codepoint_micro"]
line_f1 = metric_values["line_detection_f1_micro"]
reading_order = metric_values["matched_line_reading_order_pair_accuracy"]
blank_rate = metric_values["line_blank_prediction_rate"]
extra_rate = metric_values["line_extra_prediction_rate"]
missed_rate = metric_values["line_missed_reference_rate"]
baseline = float(args.baseline_codepoint_cer)
min_improvement = float(args.min_codepoint_improvement)
improvement = baseline - codepoint_cer if codepoint_cer is not None else None
if improvement is None or improvement < min_improvement:
issues.append(
f"codepoint CER improvement below requirement: baseline={baseline} "
f"candidate={codepoint_cer} required_improvement={min_improvement}"
)
grapheme_cer = metric_values["text_cer_grapheme_micro"]
baseline_grapheme_cer = _optional_float(getattr(args, "baseline_grapheme_cer", None))
max_grapheme_regression = float(getattr(args, "max_grapheme_regression", 0.0))
grapheme_improvement = baseline_grapheme_cer - grapheme_cer if baseline_grapheme_cer is not None and grapheme_cer is not None else None
if grapheme_improvement is not None and grapheme_improvement < -max_grapheme_regression:
issues.append(
f"grapheme CER regression above allowed maximum: baseline={baseline_grapheme_cer} "
f"candidate={grapheme_cer} improvement={grapheme_improvement} "
f"max_regression={max_grapheme_regression}"
)
wer = metric_values["metric_means.text_wer"]
baseline_wer = _optional_float(getattr(args, "baseline_wer", None))
max_wer_regression = float(getattr(args, "max_wer_regression", 0.0))
wer_improvement = baseline_wer - wer if baseline_wer is not None and wer is not None else None
if wer_improvement is not None and wer_improvement < -max_wer_regression:
issues.append(
f"WER regression above allowed maximum: baseline={baseline_wer} "
f"candidate={wer} improvement={wer_improvement} "
f"max_regression={max_wer_regression}"
)
champion_metric_values: dict[str, float | None] = {}
champion_improvements: dict[str, float | None] = {}
max_champion_regressions = {
"text_cer_codepoint_micro": float(getattr(args, "max_champion_codepoint_regression", 0.0)),
"text_cer_grapheme_micro": float(getattr(args, "max_champion_grapheme_regression", 0.0)),
"metric_means.text_wer": float(getattr(args, "max_champion_wer_regression", 0.0)),
}
if champion_metrics is not None:
for key, label in CHAMPION_TEXT_METRICS.items():
champion_metric_values[key] = _champion_float_metric(champion_metrics, key, issues)
champion_improvements[key] = _check_champion_text_regression(
candidate_value=metric_values.get(key),
champion_value=champion_metric_values[key],
max_regression=max_champion_regressions[key],
label=label,
issues=issues,
)
review_coverage_values: dict[str, int | None] = {}
max_review_uncovered_or_weak_count = int(getattr(args, "max_review_uncovered_or_weak_count", 0))
max_review_extra_uncovered_or_weak_count = int(getattr(args, "max_review_extra_uncovered_or_weak_count", 0))
if review_coverage is not None:
review_coverage_values = _check_review_coverage(
report=review_coverage,
max_uncovered_or_weak_count=max_review_uncovered_or_weak_count,
max_extra_uncovered_or_weak_count=max_review_extra_uncovered_or_weak_count,
issues=issues,
)
if line_f1 is not None and line_f1 < float(args.min_line_f1):
issues.append(f"line_detection_f1_micro below minimum {args.min_line_f1}: {line_f1}")
if reading_order is not None and reading_order < float(args.min_reading_order_accuracy):
issues.append(
f"matched_line_reading_order_pair_accuracy below minimum {args.min_reading_order_accuracy}: {reading_order}"
)
if blank_rate is not None and blank_rate > float(args.max_blank_line_rate):
issues.append(f"line_blank_prediction_rate above maximum {args.max_blank_line_rate}: {blank_rate}")
if extra_rate is not None and extra_rate > float(args.max_extra_line_rate):
issues.append(f"line_extra_prediction_rate above maximum {args.max_extra_line_rate}: {extra_rate}")
if missed_rate is not None and missed_rate > float(args.max_missed_line_rate):
issues.append(f"line_missed_reference_rate above maximum {args.max_missed_line_rate}: {missed_rate}")
scored_pages = _optional_float(metrics.get("scored_pages"))
if scored_pages is None or scored_pages < int(args.min_scored_pages):
issues.append(f"scored_pages below required minimum {args.min_scored_pages}: {scored_pages}")
missing_documents = metrics.get("missing_documents")
missing_references = metrics.get("missing_references")
if missing_documents:
issues.append(f"score metrics report missing_documents={missing_documents!r}")
if missing_references:
issues.append(f"score metrics report missing_references={missing_references!r}")
claim_safe, claim_reasons = _claim_safe(metrics, args)
if claim_reasons:
warnings.extend(claim_reasons)
admitted = not issues
report = {
"created_at_utc": datetime.now(UTC).isoformat(),
"tool": "tools/admit_limbu_book_ocr_candidate.py",
"machine": platform.node(),
"book_output": str(book_output),
"book_audit": str(book_audit_path),
"book_audit_sha256": book_audit_sha,
"book_audit_tool": book_audit.get("tool"),
"score_dir": str(score_dir),
"metrics": str(metrics_path),
"metrics_sha256": metrics_sha,
"champion_metrics": str(champion_metrics_path) if champion_metrics_path is not None else None,
"champion_metrics_sha256": champion_metrics_sha,
"review_coverage_report": str(review_coverage_path) if review_coverage_path is not None else None,
"review_coverage_report_sha256": review_coverage_sha,
"baseline_codepoint_cer": baseline,
"baseline_grapheme_cer": baseline_grapheme_cer,
"baseline_wer": baseline_wer,
"codepoint_absolute_improvement": improvement,
"grapheme_absolute_improvement": grapheme_improvement,
"wer_absolute_improvement": wer_improvement,
"champion_codepoint_absolute_improvement": champion_improvements.get("text_cer_codepoint_micro"),
"champion_grapheme_absolute_improvement": champion_improvements.get("text_cer_grapheme_micro"),
"champion_wer_absolute_improvement": champion_improvements.get("metric_means.text_wer"),
"max_grapheme_regression": max_grapheme_regression,
"max_wer_regression": max_wer_regression,
"max_champion_codepoint_regression": max_champion_regressions["text_cer_codepoint_micro"],
"max_champion_grapheme_regression": max_champion_regressions["text_cer_grapheme_micro"],
"max_champion_wer_regression": max_champion_regressions["metric_means.text_wer"],
"max_review_uncovered_or_weak_count": max_review_uncovered_or_weak_count,
"max_review_extra_uncovered_or_weak_count": max_review_extra_uncovered_or_weak_count,
"metric_values": metric_values,
"champion_metric_values": champion_metric_values,
"review_coverage_values": review_coverage_values,
"required_metrics": REQUIRED_METRICS,
"admitted_operational": admitted,
"claim_safe": claim_safe and admitted,
"claim_safe_reasons": claim_reasons,
"issues": issues,
"warnings": warnings,
"claim_boundary": (
"Operational Limbu book OCR candidate gate. Passing means artifact audit and configured "
"metric thresholds passed; claim_safe remains false unless the score run is fully claim eligible."
),
}
json_path = out / REPORT_JSON
md_path = out / REPORT_MD
_write_json(json_path, report)
_write_markdown(md_path, report)
report["report_json"] = str(json_path)
report["report_md"] = str(md_path)
_write_json(json_path, report)
return report
def _fmt(value: Any) -> str:
return "" if value is None else str(value)
def _write_markdown(path: Path, report: dict[str, Any]) -> None:
lines = [
"# Limbu Book OCR Candidate Admission",
"",
f"- admitted operational: `{report['admitted_operational']}`",
f"- claim safe: `{report['claim_safe']}`",
f"- baseline codepoint CER: `{report['baseline_codepoint_cer']}`",
f"- candidate codepoint CER: `{_fmt(report['metric_values'].get('text_cer_codepoint_micro'))}`",
f"- codepoint absolute improvement: `{_fmt(report['codepoint_absolute_improvement'])}`",
f"- baseline grapheme CER: `{_fmt(report.get('baseline_grapheme_cer'))}`",
f"- candidate grapheme CER: `{_fmt(report['metric_values'].get('text_cer_grapheme_micro'))}`",
f"- grapheme absolute improvement: `{_fmt(report.get('grapheme_absolute_improvement'))}`",
f"- baseline WER: `{_fmt(report.get('baseline_wer'))}`",
f"- candidate WER: `{_fmt(report['metric_values'].get('metric_means.text_wer'))}`",
f"- WER absolute improvement: `{_fmt(report.get('wer_absolute_improvement'))}`",
f"- champion metrics: `{_fmt(report.get('champion_metrics'))}`",
f"- champion codepoint CER: `{_fmt(report.get('champion_metric_values', {}).get('text_cer_codepoint_micro'))}`",
f"- champion codepoint absolute improvement: `{_fmt(report.get('champion_codepoint_absolute_improvement'))}`",
f"- champion grapheme CER: `{_fmt(report.get('champion_metric_values', {}).get('text_cer_grapheme_micro'))}`",
f"- champion grapheme absolute improvement: `{_fmt(report.get('champion_grapheme_absolute_improvement'))}`",
f"- champion WER: `{_fmt(report.get('champion_metric_values', {}).get('metric_means.text_wer'))}`",
f"- champion WER absolute improvement: `{_fmt(report.get('champion_wer_absolute_improvement'))}`",
f"- review coverage report: `{_fmt(report.get('review_coverage_report'))}`",
f"- review coverage missed references: `{_fmt(report.get('review_coverage_values', {}).get('missed_reference_count'))}`",
f"- review coverage uncovered/weak: `{_fmt(report.get('review_coverage_values', {}).get('uncovered_or_weak_count'))}`",
f"- review coverage extra predictions: `{_fmt(report.get('review_coverage_values', {}).get('extra_prediction_count'))}`",
f"- review coverage extra uncovered/weak: `{_fmt(report.get('review_coverage_values', {}).get('extra_uncovered_or_weak_count'))}`",
f"- line F1: `{_fmt(report['metric_values'].get('line_detection_f1_micro'))}`",
f"- reading-order accuracy: `{_fmt(report['metric_values'].get('matched_line_reading_order_pair_accuracy'))}`",
"",
"## Issues",
"",
]
lines.extend(f"- {issue}" for issue in report["issues"]) if report["issues"] else lines.append("- none")
lines.extend(["", "## Warnings", ""])
lines.extend(f"- {warning}" for warning in report["warnings"]) if report["warnings"] else lines.append("- none")
path.write_text("\n".join(lines).rstrip() + "\n", encoding="utf-8")
def main(argv: list[str] | None = None) -> int:
parser = argparse.ArgumentParser(description=__doc__)
parser.add_argument("--book-output", type=Path, required=True)
parser.add_argument("--score-dir", type=Path, required=True)
parser.add_argument("--out", type=Path, required=True)
parser.add_argument("--book-audit", type=Path)
parser.add_argument("--metrics-sha256")
parser.add_argument("--book-audit-sha256")
parser.add_argument("--champion-metrics", type=Path)
parser.add_argument("--review-coverage-report", type=Path)
parser.add_argument("--baseline-codepoint-cer", type=float, default=0.458432)
parser.add_argument("--baseline-grapheme-cer", type=float)
parser.add_argument("--baseline-wer", type=float)
parser.add_argument("--min-codepoint-improvement", type=float, default=0.0)
parser.add_argument("--max-grapheme-regression", type=float, default=0.0)
parser.add_argument("--max-wer-regression", type=float, default=0.0)
parser.add_argument("--max-champion-codepoint-regression", type=float, default=0.0)
parser.add_argument("--max-champion-grapheme-regression", type=float, default=0.0)
parser.add_argument("--max-champion-wer-regression", type=float, default=0.0)
parser.add_argument("--max-review-uncovered-or-weak-count", type=int, default=0)
parser.add_argument("--max-review-extra-uncovered-or-weak-count", type=int, default=0)
parser.add_argument("--min-line-f1", type=float, default=0.0)
parser.add_argument("--min-reading-order-accuracy", type=float, default=0.0)
parser.add_argument("--max-blank-line-rate", type=float, default=0.0)
parser.add_argument("--max-extra-line-rate", type=float, default=1.0)
parser.add_argument("--max-missed-line-rate", type=float, default=1.0)
parser.add_argument("--min-scored-pages", type=int, default=1)
parser.add_argument("--min-claim-pages", type=int, default=30)
args = parser.parse_args(argv)
report = admit_candidate(args)
print(report["report_json"])
print(report["report_md"])
print(f"admitted_operational={report['admitted_operational']} claim_safe={report['claim_safe']}")
return 0 if report["admitted_operational"] else 1
if __name__ == "__main__":
raise SystemExit(main())