ocr-projects-archive / ocr-tech /tools /apply_limbu_book_split_worksheet_review.py
prosperious's picture
Add 1000 files (ocr-tech/tests/test_audit_limbu_book_missed_line_review_coverage.py ...)
d82db76 verified
Raw History Blame Contribute Delete
7.48 kB
#!/usr/bin/env python3
"""Compile a reviewed split worksheet and apply it to a Limbu book projection."""
from __future__ import annotations
import argparse
import importlib.util
import json
import sys
from argparse import Namespace
from datetime import UTC, datetime
from pathlib import Path
from typing import Any
from ocrtech.manifest import sha256_file
REPORT_JSON = "book-line-split-worksheet-review-application.json"
DEFAULT_ACCEPTED_STATUSES = ("accept", "accepted", "approved", "corrected", "verified", "done")
def _load_sibling_module(module_name: str) -> Any:
path = Path(__file__).with_name(f"{module_name}.py")
spec = importlib.util.spec_from_file_location(module_name, path)
if spec is None or spec.loader is None:
raise ValueError(f"could not load sibling tool module: {path}")
module = importlib.util.module_from_spec(spec)
sys.modules[module_name] = module
spec.loader.exec_module(module)
return module
def _file_record(path: str | Path | None) -> dict[str, str | None]:
if path is None:
return {"path": None, "sha256": None}
file_path = Path(path)
if not file_path.is_file():
raise ValueError(f"expected file is missing: {file_path}")
return {"path": str(file_path), "sha256": sha256_file(file_path)}
def apply_split_worksheet_review(args: argparse.Namespace) -> dict[str, Any]:
book_output = Path(args.book_output_dir)
prefilled_packet = Path(args.prefilled_packet)
worksheet_report = Path(args.worksheet_report)
out = Path(args.out)
mark_status = str(args.mark_status or "").strip()
accepted_statuses = [str(status).strip().lower() for status in args.accepted_statuses if str(status).strip()]
if not accepted_statuses:
raise ValueError("--accepted-statuses must contain at least one status")
if not mark_status:
raise ValueError("--mark-status is required so compiled rows can be selected by the packet applicator")
if mark_status.strip().lower() not in set(accepted_statuses):
raise ValueError(f"--mark-status {mark_status!r} must be included in --accepted-statuses")
if not (book_output / "book-run.json").is_file():
raise ValueError(f"book output is missing book-run.json: {book_output}")
if not prefilled_packet.is_file():
raise ValueError(f"prefilled packet TSV is missing: {prefilled_packet}")
if not worksheet_report.is_file():
raise ValueError(f"worksheet report is missing: {worksheet_report}")
worksheet_tool = _load_sibling_module("apply_limbu_book_split_transcription_worksheet")
packet_tool = _load_sibling_module("apply_limbu_book_unresolved_review_packet")
audit_tool = _load_sibling_module("audit_limbu_book_unresolved_review_packet_application")
out.mkdir(parents=True, exist_ok=True)
compiled_out = out / "compiled-packet"
application_out = out / "application"
compile_report = worksheet_tool.apply_worksheet(
Namespace(
prefilled_packet=prefilled_packet,
worksheet_report=worksheet_report,
out=compiled_out,
mark_status=mark_status,
text_statuses=list(args.text_statuses),
drop_statuses=list(args.drop_statuses),
)
)
application_report = packet_tool.apply_unresolved_review_packet(
Namespace(
book_output_dir=book_output,
review_packet=Path(str(compile_report["output_tsv"])),
out=application_out,
accepted_statuses=accepted_statuses,
)
)
application_audit = audit_tool.audit_application(application_out)
report = {
"tool": "tools/apply_limbu_book_split_worksheet_review.py",
"created_at_utc": datetime.now(UTC).isoformat(),
"source_book_output_dir": str(book_output),
"source_book_run": str(book_output / "book-run.json"),
"source_book_run_sha256": sha256_file(book_output / "book-run.json"),
"prefilled_packet": str(prefilled_packet),
"prefilled_packet_sha256": sha256_file(prefilled_packet),
"worksheet_report": str(worksheet_report),
"worksheet_report_sha256": sha256_file(worksheet_report),
"out": str(out),
"mark_status": mark_status,
"accepted_statuses": accepted_statuses,
"compiled_dir": str(compiled_out),
"compiled_report": _file_record(compile_report.get("report_json")),
"compiled_packet_tsv": _file_record(compile_report.get("output_tsv")),
"compiled_count": compile_report.get("compiled_count"),
"compiled_packet_row_ids": compile_report.get("compiled_packet_row_ids"),
"compile_skipped": compile_report.get("skipped"),
"application_dir": str(application_out),
"application_report": _file_record(application_report.get("report_json")),
"application_audit": _file_record(application_out / "book-unresolved-review-packet-application-audit.json"),
"application_audit_passed": application_audit.get("audit_passed"),
"operational_application_safe": application_audit.get("operational_application_safe"),
"final_book_run": _file_record(application_report.get("final_book_run", {}).get("path")),
"final_book_md": _file_record(application_report.get("final_book_md", {}).get("path")),
"final_book_body_md": _file_record(application_report.get("final_book_body_md", {}).get("path")),
"packet_rows": application_report.get("packet_rows"),
"accepted_counts": application_report.get("accepted_counts"),
"book_line_applied_count": application_report.get("book_line_applied_count"),
"book_line_changed_count": application_report.get("book_line_changed_count"),
"candidate_line_applied_count": application_report.get("candidate_line_applied_count"),
"claim_scope": "worksheet compilation plus review-packet application workflow; OCR-quality claims require separate scoring/admission",
"boundary": "This applies only completed worksheet groups and accepted packet rows; run frozen-page scoring before claiming OCR improvement.",
}
report_path = out / REPORT_JSON
report["report_json"] = str(report_path)
report_path.write_text(json.dumps(report, ensure_ascii=False, indent=2) + "\n", encoding="utf-8")
return report
def main(argv: list[str] | None = None) -> int:
parser = argparse.ArgumentParser(description=__doc__)
parser.add_argument("book_output_dir", type=Path)
parser.add_argument("--prefilled-packet", type=Path, required=True)
parser.add_argument("--worksheet-report", type=Path, required=True)
parser.add_argument("--out", type=Path, required=True)
parser.add_argument("--mark-status", default="corrected")
parser.add_argument("--accepted-statuses", nargs="+", default=list(DEFAULT_ACCEPTED_STATUSES))
worksheet_tool = _load_sibling_module("apply_limbu_book_split_transcription_worksheet")
parser.add_argument("--text-statuses", nargs="+", default=list(worksheet_tool.DEFAULT_TEXT_STATUSES))
parser.add_argument("--drop-statuses", nargs="+", default=list(worksheet_tool.DEFAULT_DROP_STATUSES))
args = parser.parse_args(argv)
report = apply_split_worksheet_review(args)
print(report["report_json"])
print(report["final_book_run"]["path"])
print(f"compiled={report['compiled_count']} book_line_applied={report['book_line_applied_count']}")
return 0
if __name__ == "__main__":
raise SystemExit(main())