Add 1000 files (ocr-tech/tests/test_audit_limbu_book_missed_line_review_coverage.py ...)
d82db76 verified Download ocr-tech/tools/apply_candidate_line_reviews.py from prosperious/ocr-projects-archive: direct link, hf CLI and curl.
- Browser
- Download file 28 kB
-
https://huggingface.co/prosperious/ocr-projects-archive/resolve/main/ocr-tech/tools/apply_candidate_line_reviews.py
- Command line
-
hf download hf://prosperious/ocr-projects-archive/ocr-tech/tools/apply_candidate_line_reviews.py
-
curl -L -o apply_candidate_line_reviews.py https://huggingface.co/prosperious/ocr-projects-archive/resolve/main/ocr-tech/tools/apply_candidate_line_reviews.py
28 kB
| #!/usr/bin/env python3 | |
| """Apply accepted candidate-only line review rows to a new Limbu book projection.""" | |
| from __future__ import annotations | |
| import argparse | |
| import csv | |
| import json | |
| from datetime import UTC, datetime | |
| from pathlib import Path | |
| from typing import Any | |
| from ocrtech.markdown import render_document_markdown | |
| from ocrtech.manifest import sha256_file | |
| from ocrtech.normalization import normalize_ocr_text | |
| from ocrtech.schemas import BBox, Block, Document, TextLine | |
| DEFAULT_ACCEPTED_STATUSES = ("accept", "accepted", "approved", "corrected", "verified", "done") | |
| FRONT_MATTER_STRUCTURAL_ROLES = {"page_marker"} | |
| FRONT_MATTER_ANCHOR_ROLES = {"title"} | |
| def _read_json_object(path: Path) -> dict[str, Any]: | |
| try: | |
| payload = json.loads(path.read_text(encoding="utf-8")) | |
| except FileNotFoundError as exc: | |
| raise ValueError(f"required JSON file is missing: {path}") from exc | |
| except json.JSONDecodeError as exc: | |
| raise ValueError(f"invalid JSON in {path}: {exc}") from exc | |
| if not isinstance(payload, dict): | |
| raise ValueError(f"JSON file must contain an object: {path}") | |
| return payload | |
| def _read_jsonl(path: Path) -> list[dict[str, Any]]: | |
| try: | |
| lines = path.read_text(encoding="utf-8").splitlines() | |
| except FileNotFoundError as exc: | |
| raise ValueError(f"candidate review JSONL is missing: {path}") from exc | |
| rows: list[dict[str, Any]] = [] | |
| for line_number, line in enumerate(lines, start=1): | |
| if not line.strip(): | |
| continue | |
| try: | |
| payload = json.loads(line) | |
| except json.JSONDecodeError as exc: | |
| raise ValueError(f"candidate review JSONL has invalid JSON at {path}:{line_number}: {exc}") from exc | |
| if not isinstance(payload, dict): | |
| raise ValueError(f"candidate review JSONL row must be an object at {path}:{line_number}") | |
| payload["_row_number"] = line_number | |
| rows.append(payload) | |
| return rows | |
| def _maybe_json(value: str) -> Any: | |
| text = str(value or "").strip() | |
| if not text: | |
| return None | |
| if not (text.startswith("[") or text.startswith("{")): | |
| return value | |
| try: | |
| return json.loads(text) | |
| except json.JSONDecodeError: | |
| return value | |
| def _maybe_float(value: Any) -> Any: | |
| if value in (None, ""): | |
| return None | |
| if isinstance(value, int | float) and not isinstance(value, bool): | |
| return value | |
| try: | |
| return float(value) | |
| except (TypeError, ValueError): | |
| return value | |
| def _unescape_cell(value: Any) -> Any: | |
| if not isinstance(value, str): | |
| return value | |
| return value.replace(r"\n", "\n") | |
| def _normalize_review_row_aliases(payload: dict[str, Any]) -> dict[str, Any]: | |
| aliases = { | |
| "candidate_row_id": "row_id", | |
| "candidate_sample_id": "sample_id", | |
| "book_page_id": "sample_id", | |
| } | |
| for source, target in aliases.items(): | |
| if not payload.get(target) and payload.get(source): | |
| payload[target] = payload[source] | |
| return payload | |
| def _read_tsv(path: Path) -> list[dict[str, Any]]: | |
| try: | |
| handle = path.open("r", encoding="utf-8", newline="") | |
| except FileNotFoundError as exc: | |
| raise ValueError(f"candidate review TSV is missing: {path}") from exc | |
| rows: list[dict[str, Any]] = [] | |
| with handle: | |
| reader = csv.DictReader(handle, delimiter="\t") | |
| if reader.fieldnames is None: | |
| raise ValueError(f"candidate review TSV has no header: {path}") | |
| for row_number, row in enumerate(reader, start=2): | |
| payload: dict[str, Any] = {key: _unescape_cell(value) for key, value in row.items() if key is not None} | |
| payload = _normalize_review_row_aliases(payload) | |
| payload["_row_number"] = row_number | |
| for key in ("candidate_bbox", "candidate_source_bboxes", "candidate_source_line_ids", "best_selected_bbox"): | |
| if key in payload: | |
| payload[key] = _maybe_json(str(payload.get(key) or "")) | |
| for key in ("candidate_confidence", "max_selected_iou"): | |
| if key in payload: | |
| payload[key] = _maybe_float(payload.get(key)) | |
| rows.append(payload) | |
| return rows | |
| def _read_review_rows(path: Path) -> list[dict[str, Any]]: | |
| suffix = path.suffix.lower() | |
| if suffix == ".tsv": | |
| return _read_tsv(path) | |
| if suffix in {".jsonl", ".ndjson", ""}: | |
| return _read_jsonl(path) | |
| raise ValueError(f"unsupported candidate review file extension {path.suffix!r}; expected .jsonl or .tsv") | |
| def _validate_candidate_crop_ref(row: dict[str, Any], *, review_path: Path) -> None: | |
| crop_path_value = str(row.get("crop_path") or "").strip() | |
| crop_sha256 = str(row.get("crop_sha256") or "").strip() | |
| if not crop_path_value and not crop_sha256: | |
| return | |
| if not crop_path_value or not crop_sha256: | |
| raise ValueError(f"accepted candidate row {row.get('row_id')!r} must include both crop_path and crop_sha256") | |
| crop_path = Path(crop_path_value) | |
| if not crop_path.is_absolute(): | |
| crop_path = review_path.parent / crop_path | |
| if not crop_path.is_file(): | |
| raise ValueError(f"accepted candidate row {row.get('row_id')!r} crop is missing: {crop_path}") | |
| actual_sha = sha256_file(crop_path) | |
| if actual_sha != crop_sha256: | |
| raise ValueError( | |
| f"accepted candidate row {row.get('row_id')!r} crop SHA-256 mismatch: " | |
| f"recorded={crop_sha256} actual={actual_sha} path={crop_path}" | |
| ) | |
| row["crop_path"] = str(crop_path) | |
| row["crop_sha256"] = actual_sha | |
| def _accepted_rows(rows: list[dict[str, Any]], accepted_statuses: set[str], *, review_path: Path) -> tuple[list[dict[str, Any]], list[dict[str, Any]]]: | |
| accepted: list[dict[str, Any]] = [] | |
| skipped: list[dict[str, Any]] = [] | |
| seen: set[str] = set() | |
| for row in rows: | |
| row_id = str(row.get("row_id") or "").strip() | |
| status = str(row.get("review_status") or "").strip().lower() | |
| if status not in accepted_statuses: | |
| skipped.append( | |
| { | |
| "row_number": row.get("_row_number"), | |
| "row_id": row_id, | |
| "status": status or "blank", | |
| "reason": "review_status_not_accepted", | |
| } | |
| ) | |
| continue | |
| if not row_id: | |
| raise ValueError(f"accepted candidate row {row.get('_row_number')} missing row_id") | |
| if row_id in seen: | |
| raise ValueError(f"duplicate accepted candidate row_id: {row_id}") | |
| seen.add(row_id) | |
| text = normalize_ocr_text(str(row.get("corrected_text") or "")) | |
| if not text.strip(): | |
| raise ValueError(f"accepted candidate row {row_id} has empty corrected_text") | |
| copied = dict(row) | |
| _validate_candidate_crop_ref(copied, review_path=review_path) | |
| copied["_normalized_inserted_text"] = text | |
| copied["_normalized_status"] = status | |
| accepted.append(copied) | |
| return accepted, skipped | |
| def _bbox_from_row(row: dict[str, Any]) -> BBox: | |
| try: | |
| return BBox.from_any(row.get("candidate_bbox")) | |
| except Exception as exc: | |
| raise ValueError(f"candidate row {row.get('row_id')!r} has invalid candidate_bbox") from exc | |
| def _line_geometry_key(line: TextLine) -> tuple[int, float, float, str]: | |
| return (line.page_index, line.bbox.y, line.bbox.x, line.line_id or "") | |
| def _block_geometry_key(block: Block) -> tuple[int, float, float, str]: | |
| return (block.page_index, block.bbox.y, block.bbox.x, block.block_id) | |
| def _is_candidate_inserted_line(line: TextLine) -> bool: | |
| metadata = line.metadata if isinstance(line.metadata, dict) else {} | |
| return isinstance(metadata.get("candidate_line_review_insertion"), dict) | |
| def _is_candidate_inserted_block(block: Block) -> bool: | |
| metadata = block.metadata if isinstance(block.metadata, dict) else {} | |
| return isinstance(metadata.get("candidate_line_review_insertion"), dict) | |
| def _line_structural_role(line: TextLine) -> str: | |
| metadata = line.metadata if isinstance(line.metadata, dict) else {} | |
| role = metadata.get("structural_role") | |
| return str(role or "text").strip().lower() or "text" | |
| def _block_structural_role(block: Block) -> str: | |
| metadata = block.metadata if isinstance(block.metadata, dict) else {} | |
| role = metadata.get("structural_role") | |
| if isinstance(role, str) and role.strip(): | |
| return role.strip().lower() | |
| return str(block.block_type or "text").strip().lower() or "text" | |
| def _front_matter_insert_at(ordered: list[Any], role_func: Any) -> int: | |
| insert_at = 0 | |
| while insert_at < len(ordered) and role_func(ordered[insert_at]) in FRONT_MATTER_ANCHOR_ROLES: | |
| insert_at += 1 | |
| return insert_at | |
| def _insert_by_geometry_preserving_existing_order(items: list[Any], inserted_items: list[Any], key_func: Any) -> list[Any]: | |
| ordered = list(items) | |
| for inserted in sorted(inserted_items, key=key_func): | |
| insert_at = len(ordered) | |
| inserted_key = key_func(inserted) | |
| for index, item in enumerate(ordered): | |
| if key_func(item) > inserted_key: | |
| insert_at = index | |
| break | |
| ordered.insert(insert_at, inserted) | |
| return ordered | |
| def _place_insertions( | |
| existing_items: list[Any], | |
| inserted_items: list[Any], | |
| *, | |
| key_func: Any, | |
| role_func: Any, | |
| ) -> list[Any]: | |
| front_matter = [item for item in inserted_items if role_func(item) in FRONT_MATTER_STRUCTURAL_ROLES] | |
| body_insertions = [item for item in inserted_items if role_func(item) not in FRONT_MATTER_STRUCTURAL_ROLES] | |
| ordered = list(existing_items) | |
| for inserted in sorted(front_matter, key=key_func): | |
| ordered.insert(_front_matter_insert_at(ordered, role_func), inserted) | |
| return _insert_by_geometry_preserving_existing_order(ordered, body_insertions, key_func) | |
| def _place_candidate_insertions(document: Document) -> None: | |
| for page in document.pages: | |
| existing_lines = [line for line in page.text_lines if not _is_candidate_inserted_line(line)] | |
| inserted_lines = [line for line in page.text_lines if _is_candidate_inserted_line(line)] | |
| page.text_lines = _place_insertions( | |
| existing_lines, | |
| inserted_lines, | |
| key_func=_line_geometry_key, | |
| role_func=_line_structural_role, | |
| ) | |
| existing_blocks = sorted([block for block in page.blocks if not _is_candidate_inserted_block(block)], key=lambda item: item.order) | |
| inserted_blocks = [block for block in page.blocks if _is_candidate_inserted_block(block)] | |
| page.blocks = _place_insertions( | |
| existing_blocks, | |
| inserted_blocks, | |
| key_func=_block_geometry_key, | |
| role_func=_block_structural_role, | |
| ) | |
| for index, block in enumerate(page.blocks, start=1): | |
| block.order = index | |
| def _write_document_artifacts(document_dir: Path, document: Document) -> None: | |
| document_dir.mkdir(parents=True, exist_ok=True) | |
| (document_dir / "document.json").write_text( | |
| json.dumps(document.to_dict(), ensure_ascii=False, indent=2) + "\n", | |
| encoding="utf-8", | |
| ) | |
| (document_dir / "document.md").write_text(render_document_markdown(document), encoding="utf-8") | |
| (document_dir / "document.body.md").write_text( | |
| render_document_markdown(document, include_structural_roles=False), | |
| encoding="utf-8", | |
| ) | |
| def _write_aggregate_markdown(path: Path, page_reports: list[dict[str, Any]], *, body: bool) -> None: | |
| filename = "document.body.md" if body else "document.md" | |
| lines = ["# Limbu Book OCR", ""] | |
| for page in page_reports: | |
| lines.extend([f"## Page {page['page_number']}: {page['page_id']}", ""]) | |
| text = (Path(str(page["document_dir"])) / filename).read_text(encoding="utf-8").strip() | |
| if text: | |
| lines.extend([text, ""]) | |
| path.write_text("\n".join(lines).rstrip() + "\n", encoding="utf-8") | |
| def _write_score_projection(out: Path, page_reports: list[dict[str, Any]]) -> Path: | |
| score_dir = out / "score-documents" | |
| score_dir.mkdir(parents=True, exist_ok=True) | |
| for page in page_reports: | |
| sample_id = str(page["score_document_sample_id"]) | |
| target_dir = score_dir / sample_id | |
| target_dir.mkdir(parents=True, exist_ok=True) | |
| document_json = Path(str(page["document_dir"])) / "document.json" | |
| target = target_dir / "document.json" | |
| if target.exists() or target.is_symlink(): | |
| target.unlink() | |
| try: | |
| target.symlink_to(document_json.resolve()) | |
| mode = "symlink" | |
| except OSError: | |
| target.write_bytes(document_json.read_bytes()) | |
| mode = "copy" | |
| page["score_document_path"] = str(target) | |
| page["score_document_projection"] = mode | |
| page["score_document_sha256"] = sha256_file(target) | |
| return score_dir | |
| def _document_line_count(document: Document) -> int: | |
| return sum(len(page.text_lines) for page in document.pages) | |
| def _write_chainable_book_run( | |
| *, | |
| out: Path, | |
| source_run: dict[str, Any], | |
| source_out: Path, | |
| source_run_path: Path, | |
| review_path: Path, | |
| score_dir: Path, | |
| page_reports: list[dict[str, Any]], | |
| book_md: Path, | |
| book_body_md: Path, | |
| ) -> Path: | |
| reports_by_page_id = {str(page["page_id"]): page for page in page_reports} | |
| source_pages = source_run.get("pages") | |
| if not isinstance(source_pages, list): | |
| raise ValueError("source book-run pages must be a list") | |
| pages: list[dict[str, Any]] = [] | |
| for page_record in source_pages: | |
| if not isinstance(page_record, dict): | |
| raise ValueError("source book-run pages must contain objects") | |
| page_id = str(page_record.get("page_id") or "") | |
| report = reports_by_page_id.get(page_id) | |
| if report is None: | |
| raise ValueError(f"missing candidate-applied page report for {page_id!r}") | |
| copied = dict(page_record) | |
| source_document = str(page_record.get("book_document_json") or page_record.get("document_json") or "") | |
| copied["source_book_document_json"] = source_document | |
| if source_document and Path(source_document).is_file(): | |
| copied["source_book_document_json_sha256"] = sha256_file(Path(source_document)) | |
| copied["book_document_json"] = str(report["document_json"]) | |
| copied["document_json"] = str(report["document_json"]) | |
| copied["book_document_json_sha256"] = str(report["document_json_sha256"]) | |
| copied["document_json_sha256"] = str(report["document_json_sha256"]) | |
| copied["book_document_dir"] = str(report["document_dir"]) | |
| copied["book_document_md_sha256"] = str(report["document_md_sha256"]) | |
| copied["document_md_sha256"] = str(report["document_md_sha256"]) | |
| copied["book_document_body_md_sha256"] = str(report["document_body_md_sha256"]) | |
| copied["document_body_md_sha256"] = str(report["document_body_md_sha256"]) | |
| copied["score_document_sample_id"] = str(report["score_document_sample_id"]) | |
| copied["score_document_path"] = str(report.get("score_document_path") or "") | |
| copied["score_document_sha256"] = str(report.get("score_document_sha256") or "") | |
| copied["candidate_line_review_applied_count"] = int(report["applied_count"]) | |
| copied["line_count"] = int(report["line_count"]) | |
| pages.append(copied) | |
| run = dict(source_run) | |
| run.update( | |
| { | |
| "tool": "tools/apply_candidate_line_reviews.py", | |
| "created_at_utc": datetime.now(UTC).isoformat(), | |
| "source_book_output_dir": str(source_out), | |
| "source_book_run": str(source_run_path), | |
| "source_book_run_sha256": sha256_file(source_run_path), | |
| "candidate_line_review_queue": str(review_path), | |
| "candidate_line_review_queue_sha256": sha256_file(review_path), | |
| "output_dir": str(out), | |
| "page_count": len(pages), | |
| "line_count": sum(int(page.get("line_count") or 0) for page in pages), | |
| "book_md": str(book_md), | |
| "book_md_sha256": sha256_file(book_md), | |
| "book_body_md": str(book_body_md), | |
| "book_body_md_sha256": sha256_file(book_body_md), | |
| "score_documents_dir": str(score_dir), | |
| "pages": pages, | |
| "claim_scope": "chainable candidate-line reviewed book projection; not ground truth unless review protocol is separately audited", | |
| } | |
| ) | |
| run_path = out / "book-run.json" | |
| run["book_run"] = str(run_path) | |
| run_path.write_text(json.dumps(run, ensure_ascii=False, indent=2) + "\n", encoding="utf-8") | |
| return run_path | |
| def _rows_by_sample(rows: list[dict[str, Any]]) -> dict[str, list[dict[str, Any]]]: | |
| grouped: dict[str, list[dict[str, Any]]] = {} | |
| for row in rows: | |
| sample_id = str(row.get("sample_id") or "").strip() | |
| if not sample_id: | |
| raise ValueError(f"accepted candidate row {row.get('row_id')!r} missing sample_id") | |
| grouped.setdefault(sample_id, []).append(row) | |
| return grouped | |
| def _insert_candidate_line(document: Document, row: dict[str, Any], *, inserted_index: int, review_path: Path) -> dict[str, Any]: | |
| page_index = int(row.get("page_index") or 0) | |
| pages = [page for page in document.pages if page.page_index == page_index] | |
| if not pages: | |
| raise ValueError(f"candidate row {row.get('row_id')!r} references missing page_index {page_index}") | |
| page = pages[0] | |
| bbox = _bbox_from_row(row) | |
| line_id = f"candidate-insert-{inserted_index:04d}" | |
| existing_ids = {line.line_id for doc_page in document.pages for line in doc_page.text_lines if line.line_id} | |
| while line_id in existing_ids: | |
| inserted_index += 1 | |
| line_id = f"candidate-insert-{inserted_index:04d}" | |
| inserted_text = str(row["_normalized_inserted_text"]) | |
| structural_role = str(row.get("candidate_structural_role") or "text").strip() or "text" | |
| metadata = { | |
| "structural_role": structural_role, | |
| "candidate_line_review_insertion": { | |
| "source_review_jsonl": str(review_path), | |
| "source_review_jsonl_sha256": sha256_file(review_path), | |
| "row_id": row.get("row_id"), | |
| "row_number": row.get("_row_number"), | |
| "review_status": row.get("_normalized_status"), | |
| "reviewer": row.get("reviewer") or "", | |
| "notes": row.get("notes") or "", | |
| "candidate_line_id": row.get("candidate_line_id") or "", | |
| "candidate_source_line_ids": row.get("candidate_source_line_ids") or [], | |
| "candidate_text": row.get("candidate_text") or "", | |
| "corrected_text": row.get("corrected_text") or "", | |
| "inserted_text": inserted_text, | |
| "candidate_confidence": row.get("candidate_confidence"), | |
| "candidate_detector_source_pass": row.get("candidate_detector_source_pass"), | |
| "max_selected_iou": row.get("max_selected_iou"), | |
| "best_selected_line_id": row.get("best_selected_line_id") or "", | |
| "best_selected_bbox": row.get("best_selected_bbox"), | |
| "crop_path": row.get("crop_path"), | |
| "crop_sha256": row.get("crop_sha256"), | |
| "placement_policy": "front_matter_structural" if structural_role in FRONT_MATTER_STRUCTURAL_ROLES else "geometry", | |
| }, | |
| } | |
| line = TextLine( | |
| text=inserted_text, | |
| bbox=bbox, | |
| confidence=row.get("candidate_confidence"), | |
| page_index=page_index, | |
| line_id=line_id, | |
| metadata=metadata, | |
| ) | |
| page.text_lines.append(line) | |
| block = Block( | |
| block_id=f"candidate-block-{inserted_index:04d}", | |
| block_type="text", | |
| page_index=page_index, | |
| bbox=bbox, | |
| order=len(page.blocks) + 1, | |
| text=inserted_text, | |
| confidence=line.confidence, | |
| line_ids=[line_id], | |
| metadata={"structural_role": structural_role, "candidate_line_review_insertion": metadata["candidate_line_review_insertion"]}, | |
| ) | |
| page.blocks.append(block) | |
| return { | |
| "row_id": row.get("row_id"), | |
| "sample_id": row.get("sample_id"), | |
| "candidate_line_id": row.get("candidate_line_id"), | |
| "inserted_line_id": line_id, | |
| "inserted_block_id": block.block_id, | |
| "review_status": row.get("_normalized_status"), | |
| "reviewer": row.get("reviewer") or "", | |
| "candidate_text": row.get("candidate_text") or "", | |
| "inserted_text": inserted_text, | |
| "candidate_bbox": bbox.to_list(), | |
| "candidate_confidence": row.get("candidate_confidence"), | |
| "candidate_detector_source_pass": row.get("candidate_detector_source_pass"), | |
| "structural_role": structural_role, | |
| "crop_path": row.get("crop_path") or "", | |
| "crop_sha256": row.get("crop_sha256") or "", | |
| } | |
| def apply_candidate_line_reviews(args: argparse.Namespace) -> dict[str, Any]: | |
| source_out = Path(args.book_output_dir) | |
| review_path = Path(args.candidate_review_jsonl) | |
| out = Path(args.out) | |
| accepted_statuses = {str(status).strip().lower() for status in args.accepted_statuses if str(status).strip()} | |
| if not accepted_statuses: | |
| raise ValueError("--accepted-statuses must contain at least one status") | |
| run_path = source_out / "book-run.json" | |
| run = _read_json_object(run_path) | |
| pages = run.get("pages") | |
| if not isinstance(pages, list) or not all(isinstance(page, dict) for page in pages): | |
| raise ValueError(f"book-run.json pages must be a list of objects: {run_path}") | |
| review_rows = _read_review_rows(review_path) | |
| accepted, skipped = _accepted_rows(review_rows, accepted_statuses, review_path=review_path) | |
| rows_by_sample = _rows_by_sample(accepted) | |
| out.mkdir(parents=True, exist_ok=True) | |
| documents_dir = out / "book-documents" | |
| applied_rows: list[dict[str, Any]] = [] | |
| page_reports: list[dict[str, Any]] = [] | |
| matched_samples: set[str] = set() | |
| inserted_index = 1 | |
| for page_record in pages: | |
| page_id = str(page_record.get("page_id") or "") | |
| sample_id = str(page_record.get("score_document_sample_id") or page_id) | |
| source_doc = Path(str(page_record.get("book_document_json") or page_record.get("document_json") or "")) | |
| if not source_doc.is_file(): | |
| raise ValueError(f"{page_id}: source book document is missing: {source_doc}") | |
| document = Document.from_dict(_read_json_object(source_doc)) | |
| page_applied: list[dict[str, Any]] = [] | |
| for row in rows_by_sample.get(sample_id, []): | |
| applied = _insert_candidate_line(document, row, inserted_index=inserted_index, review_path=review_path) | |
| inserted_index += 1 | |
| applied_rows.append(applied) | |
| page_applied.append(applied) | |
| matched_samples.add(sample_id) | |
| if page_applied: | |
| _place_candidate_insertions(document) | |
| line_count = _document_line_count(document) | |
| document_dir = documents_dir / sample_id | |
| _write_document_artifacts(document_dir, document) | |
| page_reports.append( | |
| { | |
| "page_number": page_record.get("page_number"), | |
| "page_id": page_id, | |
| "source_document": str(source_doc), | |
| "source_document_sha256": sha256_file(source_doc), | |
| "document_dir": str(document_dir), | |
| "document_json": str(document_dir / "document.json"), | |
| "document_json_sha256": sha256_file(document_dir / "document.json"), | |
| "document_md_sha256": sha256_file(document_dir / "document.md"), | |
| "document_body_md_sha256": sha256_file(document_dir / "document.body.md"), | |
| "score_document_sample_id": sample_id, | |
| "applied_count": len(page_applied), | |
| "line_count": line_count, | |
| } | |
| ) | |
| unmatched_samples = sorted(set(rows_by_sample) - matched_samples) | |
| if unmatched_samples: | |
| raise ValueError(f"accepted candidate review rows did not match book samples: {unmatched_samples}") | |
| book_md = out / "book.md" | |
| book_body_md = out / "book.body.md" | |
| _write_aggregate_markdown(book_md, page_reports, body=False) | |
| _write_aggregate_markdown(book_body_md, page_reports, body=True) | |
| score_dir = _write_score_projection(out, page_reports) | |
| output_run_path = _write_chainable_book_run( | |
| out=out, | |
| source_run=run, | |
| source_out=source_out, | |
| source_run_path=run_path, | |
| review_path=review_path, | |
| score_dir=score_dir, | |
| page_reports=page_reports, | |
| book_md=book_md, | |
| book_body_md=book_body_md, | |
| ) | |
| audit = { | |
| "tool": "tools/apply_candidate_line_reviews.py", | |
| "created_at_utc": datetime.now(UTC).isoformat(), | |
| "source_book_output_dir": str(source_out), | |
| "source_book_run": str(run_path), | |
| "source_book_run_sha256": sha256_file(run_path), | |
| "candidate_review_jsonl": str(review_path), | |
| "candidate_review_jsonl_sha256": sha256_file(review_path), | |
| "accepted_statuses": sorted(accepted_statuses), | |
| "review_rows": len(review_rows), | |
| "applied_count": len(applied_rows), | |
| "skipped_count": len(skipped), | |
| "documents_dir": str(documents_dir), | |
| "book_md": str(book_md), | |
| "book_md_sha256": sha256_file(book_md), | |
| "book_body_md": str(book_body_md), | |
| "book_body_md_sha256": sha256_file(book_body_md), | |
| "book_run": str(output_run_path), | |
| "book_run_sha256": sha256_file(output_run_path), | |
| "score_documents_dir": str(score_dir), | |
| "pages": page_reports, | |
| "applied_rows": applied_rows, | |
| "skipped_rows": skipped, | |
| "claim_scope": "candidate-line review applied editable book projection; not ground truth unless review protocol is separately audited", | |
| } | |
| pairs_path = out / "candidate-line-insertion-pairs.jsonl" | |
| pairs_path.write_text( | |
| "".join( | |
| json.dumps( | |
| { | |
| "sample_id": f"{row['sample_id']}:{row['inserted_line_id']}", | |
| "language": "limbu", | |
| "noisy_text": row["candidate_text"], | |
| "clean_text": row["inserted_text"], | |
| "source": "candidate_line_review_packet", | |
| "review_status": row["review_status"], | |
| "reviewer": row["reviewer"], | |
| "candidate_line_id": row["candidate_line_id"], | |
| "inserted_line_id": row["inserted_line_id"], | |
| "crop_path": row.get("crop_path") or "", | |
| "crop_sha256": row.get("crop_sha256") or "", | |
| }, | |
| ensure_ascii=False, | |
| ) | |
| + "\n" | |
| for row in applied_rows | |
| ), | |
| encoding="utf-8", | |
| ) | |
| audit_path = out / "candidate-line-insertion-audit.json" | |
| audit["audit_path"] = str(audit_path) | |
| audit["insertion_pairs"] = str(pairs_path) | |
| audit["insertion_pairs_sha256"] = sha256_file(pairs_path) | |
| audit_path.write_text(json.dumps(audit, ensure_ascii=False, indent=2) + "\n", encoding="utf-8") | |
| return audit | |
| def main(argv: list[str] | None = None) -> int: | |
| parser = argparse.ArgumentParser(description=__doc__) | |
| parser.add_argument("book_output_dir", type=Path) | |
| parser.add_argument("--candidate-review-jsonl", "--candidate-review-file", dest="candidate_review_jsonl", type=Path, required=True) | |
| parser.add_argument("--out", type=Path, required=True) | |
| parser.add_argument("--accepted-statuses", nargs="+", default=list(DEFAULT_ACCEPTED_STATUSES)) | |
| args = parser.parse_args(argv) | |
| result = apply_candidate_line_reviews(args) | |
| print(result["audit_path"]) | |
| print(result["insertion_pairs"]) | |
| print(f"applied_candidate_lines={result['applied_count']}") | |
| return 0 | |
| if __name__ == "__main__": | |
| raise SystemExit(main()) | |