#!/usr/bin/env python3 """Admission gate for a Limbu book OCR candidate run.""" from __future__ import annotations import argparse import json import platform from datetime import UTC, datetime from pathlib import Path from typing import Any from ocrtech.manifest import sha256_file REPORT_JSON = "limbu-book-ocr-candidate-admission.json" REPORT_MD = "limbu-book-ocr-candidate-admission.md" REQUIRED_METRICS = { "text_cer_codepoint_micro": "page codepoint CER", "text_cer_grapheme_micro": "page grapheme CER", "metric_means.text_wer": "page WER", "line_detection_precision_micro": "line detection precision", "line_detection_recall_micro": "line detection recall", "line_detection_f1_micro": "line detection F1", "matched_line_reading_order_pair_accuracy": "reading-order pair accuracy", "line_blank_prediction_rate": "blank-line rate", "line_extra_prediction_rate": "extra-line rate", "line_missed_reference_rate": "missed-line rate", } CHAMPION_TEXT_METRICS = { "text_cer_codepoint_micro": "codepoint CER", "text_cer_grapheme_micro": "grapheme CER", "metric_means.text_wer": "WER", } def _read_json(path: Path, label: str) -> dict[str, Any]: if not path.is_file(): raise FileNotFoundError(f"{label} does not exist: {path}") try: payload = json.loads(path.read_text(encoding="utf-8")) except json.JSONDecodeError as exc: raise ValueError(f"{label} is invalid JSON: {path}: {exc}") from exc if not isinstance(payload, dict): raise ValueError(f"{label} must contain a JSON object: {path}") return payload def _write_json(path: Path, payload: dict[str, Any]) -> None: path.parent.mkdir(parents=True, exist_ok=True) path.write_text(json.dumps(payload, ensure_ascii=False, indent=2) + "\n", encoding="utf-8") def _metric(metrics: dict[str, Any], dotted_key: str) -> Any: value: Any = metrics for part in dotted_key.split("."): if not isinstance(value, dict) or part not in value: return None value = value[part] return value def _float_metric(metrics: dict[str, Any], dotted_key: str, issues: list[str]) -> float | None: value = _metric(metrics, dotted_key) try: parsed = float(value) except (TypeError, ValueError): issues.append(f"required metric {dotted_key} is missing or non-numeric") return None return parsed def _champion_float_metric(metrics: dict[str, Any], dotted_key: str, issues: list[str]) -> float | None: value = _metric(metrics, dotted_key) try: return float(value) except (TypeError, ValueError): issues.append(f"champion metric {dotted_key} is missing or non-numeric") return None def _optional_float(value: Any) -> float | None: try: return float(value) except (TypeError, ValueError): return None def _check_hash(path: Path, expected: str | None, issues: list[str], *, label: str) -> str | None: if not path.is_file(): issues.append(f"{label} is missing: {path}") return None actual = sha256_file(path) if expected is not None and actual != expected: issues.append(f"{label} SHA-256 mismatch: recorded={expected} actual={actual}") return actual def _audit_passed(audit: dict[str, Any]) -> bool: if "passed" in audit: return audit.get("passed") is True return audit.get("audit_passed") is True def _audit_matches_book_output(book_audit: dict[str, Any], book_output: Path, issues: list[str]) -> None: recorded_output = str(book_audit.get("output_dir") or "") if recorded_output == str(book_output): return if book_audit.get("tool") != "tools/audit_limbu_book_unresolved_review_packet_application.py": issues.append("book/application output audit output_dir does not match candidate output") return final_output = str(book_audit.get("final_book_output_dir") or "") application_dir = str(book_audit.get("application_dir") or "") expected_final_output = str(Path(application_dir) / "final-book") if application_dir else "" if final_output and final_output != str(book_output): issues.append("unified review-application audit final_book_output_dir does not match candidate output") return if not final_output and expected_final_output != str(book_output): issues.append("unified review-application audit application_dir/final-book does not match candidate output") return final_book_run = book_output / "book-run.json" final_book_run_sha = _check_hash( final_book_run, book_audit.get("final_book_run_sha256"), issues, label="unified review-application final book-run.json", ) if final_book_run_sha is None: return def _claim_safe(metrics: dict[str, Any], args: argparse.Namespace) -> tuple[bool, list[str]]: reasons: list[str] = [] claim_rows = _optional_float(metrics.get("claim_eligible_rows")) manifest_samples = _optional_float(metrics.get("manifest_samples")) scored_pages = _optional_float(metrics.get("scored_pages")) if claim_rows is None or manifest_samples is None or claim_rows != manifest_samples: reasons.append("not all scored rows are claim eligible") if scored_pages is None or scored_pages < int(args.min_claim_pages): reasons.append(f"scored_pages below claim minimum {args.min_claim_pages}") return not reasons, reasons def _check_champion_text_regression( *, candidate_value: float | None, champion_value: float | None, max_regression: float, label: str, issues: list[str], ) -> float | None: improvement = champion_value - candidate_value if champion_value is not None and candidate_value is not None else None if improvement is not None and improvement < -max_regression: issues.append( f"champion {label} regression above allowed maximum: champion={champion_value} " f"candidate={candidate_value} improvement={improvement} max_regression={max_regression}" ) return improvement def _check_review_coverage( *, report: dict[str, Any], max_uncovered_or_weak_count: int, max_extra_uncovered_or_weak_count: int, issues: list[str], ) -> dict[str, int | None]: values: dict[str, int | None] = {} required_integer_fields = [ "missed_reference_count", "uncovered_or_weak_count", "extra_prediction_count", "extra_uncovered_or_weak_count", ] for field in required_integer_fields: parsed = _optional_float(report.get(field)) if parsed is None or int(parsed) != parsed: issues.append(f"review coverage report field {field} is missing or non-integer") values[field] = None else: values[field] = int(parsed) if report.get("tool") != "tools/audit_limbu_book_missed_line_review_coverage.py": issues.append(f"unexpected review coverage report tool: {report.get('tool')!r}") uncovered = values.get("uncovered_or_weak_count") if uncovered is None or uncovered > max_uncovered_or_weak_count: issues.append( f"review coverage uncovered_or_weak_count above maximum: " f"candidate={uncovered} max={max_uncovered_or_weak_count}" ) extra_uncovered = values.get("extra_uncovered_or_weak_count") if extra_uncovered is None or extra_uncovered > max_extra_uncovered_or_weak_count: issues.append( f"review coverage extra_uncovered_or_weak_count above maximum: " f"candidate={extra_uncovered} max={max_extra_uncovered_or_weak_count}" ) return values def admit_candidate(args: argparse.Namespace) -> dict[str, Any]: out = Path(args.out) out.mkdir(parents=True, exist_ok=True) book_output = Path(args.book_output) score_dir = Path(args.score_dir) metrics_path = score_dir / "metrics.json" book_audit_path = Path(args.book_audit) if args.book_audit else book_output / "limbu-book-output-audit.json" champion_metrics_path = Path(args.champion_metrics) if getattr(args, "champion_metrics", None) else None review_coverage_path = Path(args.review_coverage_report) if getattr(args, "review_coverage_report", None) else None issues: list[str] = [] warnings: list[str] = [] metrics = _read_json(metrics_path, "score metrics") champion_metrics = _read_json(champion_metrics_path, "champion score metrics") if champion_metrics_path else None review_coverage = _read_json(review_coverage_path, "review coverage report") if review_coverage_path else None book_audit = _read_json(book_audit_path, "book/application output audit") if not _audit_passed(book_audit): issues.append("book/application output audit did not pass") if book_audit.get("operational_review_safe") is False: issues.append("book/application output audit marks review application as non-operational") reference_oracle_count = _optional_float(book_audit.get("reference_oracle_review_count")) if reference_oracle_count is not None and reference_oracle_count > 0: issues.append(f"candidate-line application contains oracle/reference-assisted review rows: {int(reference_oracle_count)}") _audit_matches_book_output(book_audit, book_output, issues) metrics_sha = _check_hash(metrics_path, args.metrics_sha256, issues, label="metrics.json") champion_metrics_sha = ( _check_hash(champion_metrics_path, None, issues, label="champion metrics.json") if champion_metrics_path is not None else None ) review_coverage_sha = ( _check_hash(review_coverage_path, None, issues, label="review coverage report") if review_coverage_path is not None else None ) book_audit_sha = _check_hash(book_audit_path, args.book_audit_sha256, issues, label="book/application output audit") metric_values: dict[str, float | None] = {} for key in REQUIRED_METRICS: metric_values[key] = _float_metric(metrics, key, issues) codepoint_cer = metric_values["text_cer_codepoint_micro"] line_f1 = metric_values["line_detection_f1_micro"] reading_order = metric_values["matched_line_reading_order_pair_accuracy"] blank_rate = metric_values["line_blank_prediction_rate"] extra_rate = metric_values["line_extra_prediction_rate"] missed_rate = metric_values["line_missed_reference_rate"] baseline = float(args.baseline_codepoint_cer) min_improvement = float(args.min_codepoint_improvement) improvement = baseline - codepoint_cer if codepoint_cer is not None else None if improvement is None or improvement < min_improvement: issues.append( f"codepoint CER improvement below requirement: baseline={baseline} " f"candidate={codepoint_cer} required_improvement={min_improvement}" ) grapheme_cer = metric_values["text_cer_grapheme_micro"] baseline_grapheme_cer = _optional_float(getattr(args, "baseline_grapheme_cer", None)) max_grapheme_regression = float(getattr(args, "max_grapheme_regression", 0.0)) grapheme_improvement = baseline_grapheme_cer - grapheme_cer if baseline_grapheme_cer is not None and grapheme_cer is not None else None if grapheme_improvement is not None and grapheme_improvement < -max_grapheme_regression: issues.append( f"grapheme CER regression above allowed maximum: baseline={baseline_grapheme_cer} " f"candidate={grapheme_cer} improvement={grapheme_improvement} " f"max_regression={max_grapheme_regression}" ) wer = metric_values["metric_means.text_wer"] baseline_wer = _optional_float(getattr(args, "baseline_wer", None)) max_wer_regression = float(getattr(args, "max_wer_regression", 0.0)) wer_improvement = baseline_wer - wer if baseline_wer is not None and wer is not None else None if wer_improvement is not None and wer_improvement < -max_wer_regression: issues.append( f"WER regression above allowed maximum: baseline={baseline_wer} " f"candidate={wer} improvement={wer_improvement} " f"max_regression={max_wer_regression}" ) champion_metric_values: dict[str, float | None] = {} champion_improvements: dict[str, float | None] = {} max_champion_regressions = { "text_cer_codepoint_micro": float(getattr(args, "max_champion_codepoint_regression", 0.0)), "text_cer_grapheme_micro": float(getattr(args, "max_champion_grapheme_regression", 0.0)), "metric_means.text_wer": float(getattr(args, "max_champion_wer_regression", 0.0)), } if champion_metrics is not None: for key, label in CHAMPION_TEXT_METRICS.items(): champion_metric_values[key] = _champion_float_metric(champion_metrics, key, issues) champion_improvements[key] = _check_champion_text_regression( candidate_value=metric_values.get(key), champion_value=champion_metric_values[key], max_regression=max_champion_regressions[key], label=label, issues=issues, ) review_coverage_values: dict[str, int | None] = {} max_review_uncovered_or_weak_count = int(getattr(args, "max_review_uncovered_or_weak_count", 0)) max_review_extra_uncovered_or_weak_count = int(getattr(args, "max_review_extra_uncovered_or_weak_count", 0)) if review_coverage is not None: review_coverage_values = _check_review_coverage( report=review_coverage, max_uncovered_or_weak_count=max_review_uncovered_or_weak_count, max_extra_uncovered_or_weak_count=max_review_extra_uncovered_or_weak_count, issues=issues, ) if line_f1 is not None and line_f1 < float(args.min_line_f1): issues.append(f"line_detection_f1_micro below minimum {args.min_line_f1}: {line_f1}") if reading_order is not None and reading_order < float(args.min_reading_order_accuracy): issues.append( f"matched_line_reading_order_pair_accuracy below minimum {args.min_reading_order_accuracy}: {reading_order}" ) if blank_rate is not None and blank_rate > float(args.max_blank_line_rate): issues.append(f"line_blank_prediction_rate above maximum {args.max_blank_line_rate}: {blank_rate}") if extra_rate is not None and extra_rate > float(args.max_extra_line_rate): issues.append(f"line_extra_prediction_rate above maximum {args.max_extra_line_rate}: {extra_rate}") if missed_rate is not None and missed_rate > float(args.max_missed_line_rate): issues.append(f"line_missed_reference_rate above maximum {args.max_missed_line_rate}: {missed_rate}") scored_pages = _optional_float(metrics.get("scored_pages")) if scored_pages is None or scored_pages < int(args.min_scored_pages): issues.append(f"scored_pages below required minimum {args.min_scored_pages}: {scored_pages}") missing_documents = metrics.get("missing_documents") missing_references = metrics.get("missing_references") if missing_documents: issues.append(f"score metrics report missing_documents={missing_documents!r}") if missing_references: issues.append(f"score metrics report missing_references={missing_references!r}") claim_safe, claim_reasons = _claim_safe(metrics, args) if claim_reasons: warnings.extend(claim_reasons) admitted = not issues report = { "created_at_utc": datetime.now(UTC).isoformat(), "tool": "tools/admit_limbu_book_ocr_candidate.py", "machine": platform.node(), "book_output": str(book_output), "book_audit": str(book_audit_path), "book_audit_sha256": book_audit_sha, "book_audit_tool": book_audit.get("tool"), "score_dir": str(score_dir), "metrics": str(metrics_path), "metrics_sha256": metrics_sha, "champion_metrics": str(champion_metrics_path) if champion_metrics_path is not None else None, "champion_metrics_sha256": champion_metrics_sha, "review_coverage_report": str(review_coverage_path) if review_coverage_path is not None else None, "review_coverage_report_sha256": review_coverage_sha, "baseline_codepoint_cer": baseline, "baseline_grapheme_cer": baseline_grapheme_cer, "baseline_wer": baseline_wer, "codepoint_absolute_improvement": improvement, "grapheme_absolute_improvement": grapheme_improvement, "wer_absolute_improvement": wer_improvement, "champion_codepoint_absolute_improvement": champion_improvements.get("text_cer_codepoint_micro"), "champion_grapheme_absolute_improvement": champion_improvements.get("text_cer_grapheme_micro"), "champion_wer_absolute_improvement": champion_improvements.get("metric_means.text_wer"), "max_grapheme_regression": max_grapheme_regression, "max_wer_regression": max_wer_regression, "max_champion_codepoint_regression": max_champion_regressions["text_cer_codepoint_micro"], "max_champion_grapheme_regression": max_champion_regressions["text_cer_grapheme_micro"], "max_champion_wer_regression": max_champion_regressions["metric_means.text_wer"], "max_review_uncovered_or_weak_count": max_review_uncovered_or_weak_count, "max_review_extra_uncovered_or_weak_count": max_review_extra_uncovered_or_weak_count, "metric_values": metric_values, "champion_metric_values": champion_metric_values, "review_coverage_values": review_coverage_values, "required_metrics": REQUIRED_METRICS, "admitted_operational": admitted, "claim_safe": claim_safe and admitted, "claim_safe_reasons": claim_reasons, "issues": issues, "warnings": warnings, "claim_boundary": ( "Operational Limbu book OCR candidate gate. Passing means artifact audit and configured " "metric thresholds passed; claim_safe remains false unless the score run is fully claim eligible." ), } json_path = out / REPORT_JSON md_path = out / REPORT_MD _write_json(json_path, report) _write_markdown(md_path, report) report["report_json"] = str(json_path) report["report_md"] = str(md_path) _write_json(json_path, report) return report def _fmt(value: Any) -> str: return "" if value is None else str(value) def _write_markdown(path: Path, report: dict[str, Any]) -> None: lines = [ "# Limbu Book OCR Candidate Admission", "", f"- admitted operational: `{report['admitted_operational']}`", f"- claim safe: `{report['claim_safe']}`", f"- baseline codepoint CER: `{report['baseline_codepoint_cer']}`", f"- candidate codepoint CER: `{_fmt(report['metric_values'].get('text_cer_codepoint_micro'))}`", f"- codepoint absolute improvement: `{_fmt(report['codepoint_absolute_improvement'])}`", f"- baseline grapheme CER: `{_fmt(report.get('baseline_grapheme_cer'))}`", f"- candidate grapheme CER: `{_fmt(report['metric_values'].get('text_cer_grapheme_micro'))}`", f"- grapheme absolute improvement: `{_fmt(report.get('grapheme_absolute_improvement'))}`", f"- baseline WER: `{_fmt(report.get('baseline_wer'))}`", f"- candidate WER: `{_fmt(report['metric_values'].get('metric_means.text_wer'))}`", f"- WER absolute improvement: `{_fmt(report.get('wer_absolute_improvement'))}`", f"- champion metrics: `{_fmt(report.get('champion_metrics'))}`", f"- champion codepoint CER: `{_fmt(report.get('champion_metric_values', {}).get('text_cer_codepoint_micro'))}`", f"- champion codepoint absolute improvement: `{_fmt(report.get('champion_codepoint_absolute_improvement'))}`", f"- champion grapheme CER: `{_fmt(report.get('champion_metric_values', {}).get('text_cer_grapheme_micro'))}`", f"- champion grapheme absolute improvement: `{_fmt(report.get('champion_grapheme_absolute_improvement'))}`", f"- champion WER: `{_fmt(report.get('champion_metric_values', {}).get('metric_means.text_wer'))}`", f"- champion WER absolute improvement: `{_fmt(report.get('champion_wer_absolute_improvement'))}`", f"- review coverage report: `{_fmt(report.get('review_coverage_report'))}`", f"- review coverage missed references: `{_fmt(report.get('review_coverage_values', {}).get('missed_reference_count'))}`", f"- review coverage uncovered/weak: `{_fmt(report.get('review_coverage_values', {}).get('uncovered_or_weak_count'))}`", f"- review coverage extra predictions: `{_fmt(report.get('review_coverage_values', {}).get('extra_prediction_count'))}`", f"- review coverage extra uncovered/weak: `{_fmt(report.get('review_coverage_values', {}).get('extra_uncovered_or_weak_count'))}`", f"- line F1: `{_fmt(report['metric_values'].get('line_detection_f1_micro'))}`", f"- reading-order accuracy: `{_fmt(report['metric_values'].get('matched_line_reading_order_pair_accuracy'))}`", "", "## Issues", "", ] lines.extend(f"- {issue}" for issue in report["issues"]) if report["issues"] else lines.append("- none") lines.extend(["", "## Warnings", ""]) lines.extend(f"- {warning}" for warning in report["warnings"]) if report["warnings"] else lines.append("- none") path.write_text("\n".join(lines).rstrip() + "\n", encoding="utf-8") def main(argv: list[str] | None = None) -> int: parser = argparse.ArgumentParser(description=__doc__) parser.add_argument("--book-output", type=Path, required=True) parser.add_argument("--score-dir", type=Path, required=True) parser.add_argument("--out", type=Path, required=True) parser.add_argument("--book-audit", type=Path) parser.add_argument("--metrics-sha256") parser.add_argument("--book-audit-sha256") parser.add_argument("--champion-metrics", type=Path) parser.add_argument("--review-coverage-report", type=Path) parser.add_argument("--baseline-codepoint-cer", type=float, default=0.458432) parser.add_argument("--baseline-grapheme-cer", type=float) parser.add_argument("--baseline-wer", type=float) parser.add_argument("--min-codepoint-improvement", type=float, default=0.0) parser.add_argument("--max-grapheme-regression", type=float, default=0.0) parser.add_argument("--max-wer-regression", type=float, default=0.0) parser.add_argument("--max-champion-codepoint-regression", type=float, default=0.0) parser.add_argument("--max-champion-grapheme-regression", type=float, default=0.0) parser.add_argument("--max-champion-wer-regression", type=float, default=0.0) parser.add_argument("--max-review-uncovered-or-weak-count", type=int, default=0) parser.add_argument("--max-review-extra-uncovered-or-weak-count", type=int, default=0) parser.add_argument("--min-line-f1", type=float, default=0.0) parser.add_argument("--min-reading-order-accuracy", type=float, default=0.0) parser.add_argument("--max-blank-line-rate", type=float, default=0.0) parser.add_argument("--max-extra-line-rate", type=float, default=1.0) parser.add_argument("--max-missed-line-rate", type=float, default=1.0) parser.add_argument("--min-scored-pages", type=int, default=1) parser.add_argument("--min-claim-pages", type=int, default=30) args = parser.parse_args(argv) report = admit_candidate(args) print(report["report_json"]) print(report["report_md"]) print(f"admitted_operational={report['admitted_operational']} claim_safe={report['claim_safe']}") return 0 if report["admitted_operational"] else 1 if __name__ == "__main__": raise SystemExit(main())