Add 1000 files (ocr-tech/tests/test_audit_limbu_book_missed_line_review_coverage.py ...)
d82db76 verified Download ocr-tech/tools/apply_limbu_book_split_worksheet_review.py from prosperious/ocr-projects-archive: direct link, hf CLI and curl.
- Browser
- Download file 7.48 kB
-
https://huggingface.co/prosperious/ocr-projects-archive/resolve/main/ocr-tech/tools/apply_limbu_book_split_worksheet_review.py
- Command line
-
hf download hf://prosperious/ocr-projects-archive/ocr-tech/tools/apply_limbu_book_split_worksheet_review.py
-
curl -L -o apply_limbu_book_split_worksheet_review.py https://huggingface.co/prosperious/ocr-projects-archive/resolve/main/ocr-tech/tools/apply_limbu_book_split_worksheet_review.py
7.48 kB
| #!/usr/bin/env python3 | |
| """Compile a reviewed split worksheet and apply it to a Limbu book projection.""" | |
| from __future__ import annotations | |
| import argparse | |
| import importlib.util | |
| import json | |
| import sys | |
| from argparse import Namespace | |
| from datetime import UTC, datetime | |
| from pathlib import Path | |
| from typing import Any | |
| from ocrtech.manifest import sha256_file | |
| REPORT_JSON = "book-line-split-worksheet-review-application.json" | |
| DEFAULT_ACCEPTED_STATUSES = ("accept", "accepted", "approved", "corrected", "verified", "done") | |
| def _load_sibling_module(module_name: str) -> Any: | |
| path = Path(__file__).with_name(f"{module_name}.py") | |
| spec = importlib.util.spec_from_file_location(module_name, path) | |
| if spec is None or spec.loader is None: | |
| raise ValueError(f"could not load sibling tool module: {path}") | |
| module = importlib.util.module_from_spec(spec) | |
| sys.modules[module_name] = module | |
| spec.loader.exec_module(module) | |
| return module | |
| def _file_record(path: str | Path | None) -> dict[str, str | None]: | |
| if path is None: | |
| return {"path": None, "sha256": None} | |
| file_path = Path(path) | |
| if not file_path.is_file(): | |
| raise ValueError(f"expected file is missing: {file_path}") | |
| return {"path": str(file_path), "sha256": sha256_file(file_path)} | |
| def apply_split_worksheet_review(args: argparse.Namespace) -> dict[str, Any]: | |
| book_output = Path(args.book_output_dir) | |
| prefilled_packet = Path(args.prefilled_packet) | |
| worksheet_report = Path(args.worksheet_report) | |
| out = Path(args.out) | |
| mark_status = str(args.mark_status or "").strip() | |
| accepted_statuses = [str(status).strip().lower() for status in args.accepted_statuses if str(status).strip()] | |
| if not accepted_statuses: | |
| raise ValueError("--accepted-statuses must contain at least one status") | |
| if not mark_status: | |
| raise ValueError("--mark-status is required so compiled rows can be selected by the packet applicator") | |
| if mark_status.strip().lower() not in set(accepted_statuses): | |
| raise ValueError(f"--mark-status {mark_status!r} must be included in --accepted-statuses") | |
| if not (book_output / "book-run.json").is_file(): | |
| raise ValueError(f"book output is missing book-run.json: {book_output}") | |
| if not prefilled_packet.is_file(): | |
| raise ValueError(f"prefilled packet TSV is missing: {prefilled_packet}") | |
| if not worksheet_report.is_file(): | |
| raise ValueError(f"worksheet report is missing: {worksheet_report}") | |
| worksheet_tool = _load_sibling_module("apply_limbu_book_split_transcription_worksheet") | |
| packet_tool = _load_sibling_module("apply_limbu_book_unresolved_review_packet") | |
| audit_tool = _load_sibling_module("audit_limbu_book_unresolved_review_packet_application") | |
| out.mkdir(parents=True, exist_ok=True) | |
| compiled_out = out / "compiled-packet" | |
| application_out = out / "application" | |
| compile_report = worksheet_tool.apply_worksheet( | |
| Namespace( | |
| prefilled_packet=prefilled_packet, | |
| worksheet_report=worksheet_report, | |
| out=compiled_out, | |
| mark_status=mark_status, | |
| text_statuses=list(args.text_statuses), | |
| drop_statuses=list(args.drop_statuses), | |
| ) | |
| ) | |
| application_report = packet_tool.apply_unresolved_review_packet( | |
| Namespace( | |
| book_output_dir=book_output, | |
| review_packet=Path(str(compile_report["output_tsv"])), | |
| out=application_out, | |
| accepted_statuses=accepted_statuses, | |
| ) | |
| ) | |
| application_audit = audit_tool.audit_application(application_out) | |
| report = { | |
| "tool": "tools/apply_limbu_book_split_worksheet_review.py", | |
| "created_at_utc": datetime.now(UTC).isoformat(), | |
| "source_book_output_dir": str(book_output), | |
| "source_book_run": str(book_output / "book-run.json"), | |
| "source_book_run_sha256": sha256_file(book_output / "book-run.json"), | |
| "prefilled_packet": str(prefilled_packet), | |
| "prefilled_packet_sha256": sha256_file(prefilled_packet), | |
| "worksheet_report": str(worksheet_report), | |
| "worksheet_report_sha256": sha256_file(worksheet_report), | |
| "out": str(out), | |
| "mark_status": mark_status, | |
| "accepted_statuses": accepted_statuses, | |
| "compiled_dir": str(compiled_out), | |
| "compiled_report": _file_record(compile_report.get("report_json")), | |
| "compiled_packet_tsv": _file_record(compile_report.get("output_tsv")), | |
| "compiled_count": compile_report.get("compiled_count"), | |
| "compiled_packet_row_ids": compile_report.get("compiled_packet_row_ids"), | |
| "compile_skipped": compile_report.get("skipped"), | |
| "application_dir": str(application_out), | |
| "application_report": _file_record(application_report.get("report_json")), | |
| "application_audit": _file_record(application_out / "book-unresolved-review-packet-application-audit.json"), | |
| "application_audit_passed": application_audit.get("audit_passed"), | |
| "operational_application_safe": application_audit.get("operational_application_safe"), | |
| "final_book_run": _file_record(application_report.get("final_book_run", {}).get("path")), | |
| "final_book_md": _file_record(application_report.get("final_book_md", {}).get("path")), | |
| "final_book_body_md": _file_record(application_report.get("final_book_body_md", {}).get("path")), | |
| "packet_rows": application_report.get("packet_rows"), | |
| "accepted_counts": application_report.get("accepted_counts"), | |
| "book_line_applied_count": application_report.get("book_line_applied_count"), | |
| "book_line_changed_count": application_report.get("book_line_changed_count"), | |
| "candidate_line_applied_count": application_report.get("candidate_line_applied_count"), | |
| "claim_scope": "worksheet compilation plus review-packet application workflow; OCR-quality claims require separate scoring/admission", | |
| "boundary": "This applies only completed worksheet groups and accepted packet rows; run frozen-page scoring before claiming OCR improvement.", | |
| } | |
| report_path = out / REPORT_JSON | |
| report["report_json"] = str(report_path) | |
| report_path.write_text(json.dumps(report, ensure_ascii=False, indent=2) + "\n", encoding="utf-8") | |
| return report | |
| def main(argv: list[str] | None = None) -> int: | |
| parser = argparse.ArgumentParser(description=__doc__) | |
| parser.add_argument("book_output_dir", type=Path) | |
| parser.add_argument("--prefilled-packet", type=Path, required=True) | |
| parser.add_argument("--worksheet-report", type=Path, required=True) | |
| parser.add_argument("--out", type=Path, required=True) | |
| parser.add_argument("--mark-status", default="corrected") | |
| parser.add_argument("--accepted-statuses", nargs="+", default=list(DEFAULT_ACCEPTED_STATUSES)) | |
| worksheet_tool = _load_sibling_module("apply_limbu_book_split_transcription_worksheet") | |
| parser.add_argument("--text-statuses", nargs="+", default=list(worksheet_tool.DEFAULT_TEXT_STATUSES)) | |
| parser.add_argument("--drop-statuses", nargs="+", default=list(worksheet_tool.DEFAULT_DROP_STATUSES)) | |
| args = parser.parse_args(argv) | |
| report = apply_split_worksheet_review(args) | |
| print(report["report_json"]) | |
| print(report["final_book_run"]["path"]) | |
| print(f"compiled={report['compiled_count']} book_line_applied={report['book_line_applied_count']}") | |
| return 0 | |
| if __name__ == "__main__": | |
| raise SystemExit(main()) | |