#!/usr/bin/env python3 """Apply accepted candidate-only line review rows to a new Limbu book projection.""" from __future__ import annotations import argparse import csv import json from datetime import UTC, datetime from pathlib import Path from typing import Any from ocrtech.markdown import render_document_markdown from ocrtech.manifest import sha256_file from ocrtech.normalization import normalize_ocr_text from ocrtech.schemas import BBox, Block, Document, TextLine DEFAULT_ACCEPTED_STATUSES = ("accept", "accepted", "approved", "corrected", "verified", "done") FRONT_MATTER_STRUCTURAL_ROLES = {"page_marker"} FRONT_MATTER_ANCHOR_ROLES = {"title"} def _read_json_object(path: Path) -> dict[str, Any]: try: payload = json.loads(path.read_text(encoding="utf-8")) except FileNotFoundError as exc: raise ValueError(f"required JSON file is missing: {path}") from exc except json.JSONDecodeError as exc: raise ValueError(f"invalid JSON in {path}: {exc}") from exc if not isinstance(payload, dict): raise ValueError(f"JSON file must contain an object: {path}") return payload def _read_jsonl(path: Path) -> list[dict[str, Any]]: try: lines = path.read_text(encoding="utf-8").splitlines() except FileNotFoundError as exc: raise ValueError(f"candidate review JSONL is missing: {path}") from exc rows: list[dict[str, Any]] = [] for line_number, line in enumerate(lines, start=1): if not line.strip(): continue try: payload = json.loads(line) except json.JSONDecodeError as exc: raise ValueError(f"candidate review JSONL has invalid JSON at {path}:{line_number}: {exc}") from exc if not isinstance(payload, dict): raise ValueError(f"candidate review JSONL row must be an object at {path}:{line_number}") payload["_row_number"] = line_number rows.append(payload) return rows def _maybe_json(value: str) -> Any: text = str(value or "").strip() if not text: return None if not (text.startswith("[") or text.startswith("{")): return value try: return json.loads(text) except json.JSONDecodeError: return value def _maybe_float(value: Any) -> Any: if value in (None, ""): return None if isinstance(value, int | float) and not isinstance(value, bool): return value try: return float(value) except (TypeError, ValueError): return value def _unescape_cell(value: Any) -> Any: if not isinstance(value, str): return value return value.replace(r"\n", "\n") def _normalize_review_row_aliases(payload: dict[str, Any]) -> dict[str, Any]: aliases = { "candidate_row_id": "row_id", "candidate_sample_id": "sample_id", "book_page_id": "sample_id", } for source, target in aliases.items(): if not payload.get(target) and payload.get(source): payload[target] = payload[source] return payload def _read_tsv(path: Path) -> list[dict[str, Any]]: try: handle = path.open("r", encoding="utf-8", newline="") except FileNotFoundError as exc: raise ValueError(f"candidate review TSV is missing: {path}") from exc rows: list[dict[str, Any]] = [] with handle: reader = csv.DictReader(handle, delimiter="\t") if reader.fieldnames is None: raise ValueError(f"candidate review TSV has no header: {path}") for row_number, row in enumerate(reader, start=2): payload: dict[str, Any] = {key: _unescape_cell(value) for key, value in row.items() if key is not None} payload = _normalize_review_row_aliases(payload) payload["_row_number"] = row_number for key in ("candidate_bbox", "candidate_source_bboxes", "candidate_source_line_ids", "best_selected_bbox"): if key in payload: payload[key] = _maybe_json(str(payload.get(key) or "")) for key in ("candidate_confidence", "max_selected_iou"): if key in payload: payload[key] = _maybe_float(payload.get(key)) rows.append(payload) return rows def _read_review_rows(path: Path) -> list[dict[str, Any]]: suffix = path.suffix.lower() if suffix == ".tsv": return _read_tsv(path) if suffix in {".jsonl", ".ndjson", ""}: return _read_jsonl(path) raise ValueError(f"unsupported candidate review file extension {path.suffix!r}; expected .jsonl or .tsv") def _validate_candidate_crop_ref(row: dict[str, Any], *, review_path: Path) -> None: crop_path_value = str(row.get("crop_path") or "").strip() crop_sha256 = str(row.get("crop_sha256") or "").strip() if not crop_path_value and not crop_sha256: return if not crop_path_value or not crop_sha256: raise ValueError(f"accepted candidate row {row.get('row_id')!r} must include both crop_path and crop_sha256") crop_path = Path(crop_path_value) if not crop_path.is_absolute(): crop_path = review_path.parent / crop_path if not crop_path.is_file(): raise ValueError(f"accepted candidate row {row.get('row_id')!r} crop is missing: {crop_path}") actual_sha = sha256_file(crop_path) if actual_sha != crop_sha256: raise ValueError( f"accepted candidate row {row.get('row_id')!r} crop SHA-256 mismatch: " f"recorded={crop_sha256} actual={actual_sha} path={crop_path}" ) row["crop_path"] = str(crop_path) row["crop_sha256"] = actual_sha def _accepted_rows(rows: list[dict[str, Any]], accepted_statuses: set[str], *, review_path: Path) -> tuple[list[dict[str, Any]], list[dict[str, Any]]]: accepted: list[dict[str, Any]] = [] skipped: list[dict[str, Any]] = [] seen: set[str] = set() for row in rows: row_id = str(row.get("row_id") or "").strip() status = str(row.get("review_status") or "").strip().lower() if status not in accepted_statuses: skipped.append( { "row_number": row.get("_row_number"), "row_id": row_id, "status": status or "blank", "reason": "review_status_not_accepted", } ) continue if not row_id: raise ValueError(f"accepted candidate row {row.get('_row_number')} missing row_id") if row_id in seen: raise ValueError(f"duplicate accepted candidate row_id: {row_id}") seen.add(row_id) text = normalize_ocr_text(str(row.get("corrected_text") or "")) if not text.strip(): raise ValueError(f"accepted candidate row {row_id} has empty corrected_text") copied = dict(row) _validate_candidate_crop_ref(copied, review_path=review_path) copied["_normalized_inserted_text"] = text copied["_normalized_status"] = status accepted.append(copied) return accepted, skipped def _bbox_from_row(row: dict[str, Any]) -> BBox: try: return BBox.from_any(row.get("candidate_bbox")) except Exception as exc: raise ValueError(f"candidate row {row.get('row_id')!r} has invalid candidate_bbox") from exc def _line_geometry_key(line: TextLine) -> tuple[int, float, float, str]: return (line.page_index, line.bbox.y, line.bbox.x, line.line_id or "") def _block_geometry_key(block: Block) -> tuple[int, float, float, str]: return (block.page_index, block.bbox.y, block.bbox.x, block.block_id) def _is_candidate_inserted_line(line: TextLine) -> bool: metadata = line.metadata if isinstance(line.metadata, dict) else {} return isinstance(metadata.get("candidate_line_review_insertion"), dict) def _is_candidate_inserted_block(block: Block) -> bool: metadata = block.metadata if isinstance(block.metadata, dict) else {} return isinstance(metadata.get("candidate_line_review_insertion"), dict) def _line_structural_role(line: TextLine) -> str: metadata = line.metadata if isinstance(line.metadata, dict) else {} role = metadata.get("structural_role") return str(role or "text").strip().lower() or "text" def _block_structural_role(block: Block) -> str: metadata = block.metadata if isinstance(block.metadata, dict) else {} role = metadata.get("structural_role") if isinstance(role, str) and role.strip(): return role.strip().lower() return str(block.block_type or "text").strip().lower() or "text" def _front_matter_insert_at(ordered: list[Any], role_func: Any) -> int: insert_at = 0 while insert_at < len(ordered) and role_func(ordered[insert_at]) in FRONT_MATTER_ANCHOR_ROLES: insert_at += 1 return insert_at def _insert_by_geometry_preserving_existing_order(items: list[Any], inserted_items: list[Any], key_func: Any) -> list[Any]: ordered = list(items) for inserted in sorted(inserted_items, key=key_func): insert_at = len(ordered) inserted_key = key_func(inserted) for index, item in enumerate(ordered): if key_func(item) > inserted_key: insert_at = index break ordered.insert(insert_at, inserted) return ordered def _place_insertions( existing_items: list[Any], inserted_items: list[Any], *, key_func: Any, role_func: Any, ) -> list[Any]: front_matter = [item for item in inserted_items if role_func(item) in FRONT_MATTER_STRUCTURAL_ROLES] body_insertions = [item for item in inserted_items if role_func(item) not in FRONT_MATTER_STRUCTURAL_ROLES] ordered = list(existing_items) for inserted in sorted(front_matter, key=key_func): ordered.insert(_front_matter_insert_at(ordered, role_func), inserted) return _insert_by_geometry_preserving_existing_order(ordered, body_insertions, key_func) def _place_candidate_insertions(document: Document) -> None: for page in document.pages: existing_lines = [line for line in page.text_lines if not _is_candidate_inserted_line(line)] inserted_lines = [line for line in page.text_lines if _is_candidate_inserted_line(line)] page.text_lines = _place_insertions( existing_lines, inserted_lines, key_func=_line_geometry_key, role_func=_line_structural_role, ) existing_blocks = sorted([block for block in page.blocks if not _is_candidate_inserted_block(block)], key=lambda item: item.order) inserted_blocks = [block for block in page.blocks if _is_candidate_inserted_block(block)] page.blocks = _place_insertions( existing_blocks, inserted_blocks, key_func=_block_geometry_key, role_func=_block_structural_role, ) for index, block in enumerate(page.blocks, start=1): block.order = index def _write_document_artifacts(document_dir: Path, document: Document) -> None: document_dir.mkdir(parents=True, exist_ok=True) (document_dir / "document.json").write_text( json.dumps(document.to_dict(), ensure_ascii=False, indent=2) + "\n", encoding="utf-8", ) (document_dir / "document.md").write_text(render_document_markdown(document), encoding="utf-8") (document_dir / "document.body.md").write_text( render_document_markdown(document, include_structural_roles=False), encoding="utf-8", ) def _write_aggregate_markdown(path: Path, page_reports: list[dict[str, Any]], *, body: bool) -> None: filename = "document.body.md" if body else "document.md" lines = ["# Limbu Book OCR", ""] for page in page_reports: lines.extend([f"## Page {page['page_number']}: {page['page_id']}", ""]) text = (Path(str(page["document_dir"])) / filename).read_text(encoding="utf-8").strip() if text: lines.extend([text, ""]) path.write_text("\n".join(lines).rstrip() + "\n", encoding="utf-8") def _write_score_projection(out: Path, page_reports: list[dict[str, Any]]) -> Path: score_dir = out / "score-documents" score_dir.mkdir(parents=True, exist_ok=True) for page in page_reports: sample_id = str(page["score_document_sample_id"]) target_dir = score_dir / sample_id target_dir.mkdir(parents=True, exist_ok=True) document_json = Path(str(page["document_dir"])) / "document.json" target = target_dir / "document.json" if target.exists() or target.is_symlink(): target.unlink() try: target.symlink_to(document_json.resolve()) mode = "symlink" except OSError: target.write_bytes(document_json.read_bytes()) mode = "copy" page["score_document_path"] = str(target) page["score_document_projection"] = mode page["score_document_sha256"] = sha256_file(target) return score_dir def _document_line_count(document: Document) -> int: return sum(len(page.text_lines) for page in document.pages) def _write_chainable_book_run( *, out: Path, source_run: dict[str, Any], source_out: Path, source_run_path: Path, review_path: Path, score_dir: Path, page_reports: list[dict[str, Any]], book_md: Path, book_body_md: Path, ) -> Path: reports_by_page_id = {str(page["page_id"]): page for page in page_reports} source_pages = source_run.get("pages") if not isinstance(source_pages, list): raise ValueError("source book-run pages must be a list") pages: list[dict[str, Any]] = [] for page_record in source_pages: if not isinstance(page_record, dict): raise ValueError("source book-run pages must contain objects") page_id = str(page_record.get("page_id") or "") report = reports_by_page_id.get(page_id) if report is None: raise ValueError(f"missing candidate-applied page report for {page_id!r}") copied = dict(page_record) source_document = str(page_record.get("book_document_json") or page_record.get("document_json") or "") copied["source_book_document_json"] = source_document if source_document and Path(source_document).is_file(): copied["source_book_document_json_sha256"] = sha256_file(Path(source_document)) copied["book_document_json"] = str(report["document_json"]) copied["document_json"] = str(report["document_json"]) copied["book_document_json_sha256"] = str(report["document_json_sha256"]) copied["document_json_sha256"] = str(report["document_json_sha256"]) copied["book_document_dir"] = str(report["document_dir"]) copied["book_document_md_sha256"] = str(report["document_md_sha256"]) copied["document_md_sha256"] = str(report["document_md_sha256"]) copied["book_document_body_md_sha256"] = str(report["document_body_md_sha256"]) copied["document_body_md_sha256"] = str(report["document_body_md_sha256"]) copied["score_document_sample_id"] = str(report["score_document_sample_id"]) copied["score_document_path"] = str(report.get("score_document_path") or "") copied["score_document_sha256"] = str(report.get("score_document_sha256") or "") copied["candidate_line_review_applied_count"] = int(report["applied_count"]) copied["line_count"] = int(report["line_count"]) pages.append(copied) run = dict(source_run) run.update( { "tool": "tools/apply_candidate_line_reviews.py", "created_at_utc": datetime.now(UTC).isoformat(), "source_book_output_dir": str(source_out), "source_book_run": str(source_run_path), "source_book_run_sha256": sha256_file(source_run_path), "candidate_line_review_queue": str(review_path), "candidate_line_review_queue_sha256": sha256_file(review_path), "output_dir": str(out), "page_count": len(pages), "line_count": sum(int(page.get("line_count") or 0) for page in pages), "book_md": str(book_md), "book_md_sha256": sha256_file(book_md), "book_body_md": str(book_body_md), "book_body_md_sha256": sha256_file(book_body_md), "score_documents_dir": str(score_dir), "pages": pages, "claim_scope": "chainable candidate-line reviewed book projection; not ground truth unless review protocol is separately audited", } ) run_path = out / "book-run.json" run["book_run"] = str(run_path) run_path.write_text(json.dumps(run, ensure_ascii=False, indent=2) + "\n", encoding="utf-8") return run_path def _rows_by_sample(rows: list[dict[str, Any]]) -> dict[str, list[dict[str, Any]]]: grouped: dict[str, list[dict[str, Any]]] = {} for row in rows: sample_id = str(row.get("sample_id") or "").strip() if not sample_id: raise ValueError(f"accepted candidate row {row.get('row_id')!r} missing sample_id") grouped.setdefault(sample_id, []).append(row) return grouped def _insert_candidate_line(document: Document, row: dict[str, Any], *, inserted_index: int, review_path: Path) -> dict[str, Any]: page_index = int(row.get("page_index") or 0) pages = [page for page in document.pages if page.page_index == page_index] if not pages: raise ValueError(f"candidate row {row.get('row_id')!r} references missing page_index {page_index}") page = pages[0] bbox = _bbox_from_row(row) line_id = f"candidate-insert-{inserted_index:04d}" existing_ids = {line.line_id for doc_page in document.pages for line in doc_page.text_lines if line.line_id} while line_id in existing_ids: inserted_index += 1 line_id = f"candidate-insert-{inserted_index:04d}" inserted_text = str(row["_normalized_inserted_text"]) structural_role = str(row.get("candidate_structural_role") or "text").strip() or "text" metadata = { "structural_role": structural_role, "candidate_line_review_insertion": { "source_review_jsonl": str(review_path), "source_review_jsonl_sha256": sha256_file(review_path), "row_id": row.get("row_id"), "row_number": row.get("_row_number"), "review_status": row.get("_normalized_status"), "reviewer": row.get("reviewer") or "", "notes": row.get("notes") or "", "candidate_line_id": row.get("candidate_line_id") or "", "candidate_source_line_ids": row.get("candidate_source_line_ids") or [], "candidate_text": row.get("candidate_text") or "", "corrected_text": row.get("corrected_text") or "", "inserted_text": inserted_text, "candidate_confidence": row.get("candidate_confidence"), "candidate_detector_source_pass": row.get("candidate_detector_source_pass"), "max_selected_iou": row.get("max_selected_iou"), "best_selected_line_id": row.get("best_selected_line_id") or "", "best_selected_bbox": row.get("best_selected_bbox"), "crop_path": row.get("crop_path"), "crop_sha256": row.get("crop_sha256"), "placement_policy": "front_matter_structural" if structural_role in FRONT_MATTER_STRUCTURAL_ROLES else "geometry", }, } line = TextLine( text=inserted_text, bbox=bbox, confidence=row.get("candidate_confidence"), page_index=page_index, line_id=line_id, metadata=metadata, ) page.text_lines.append(line) block = Block( block_id=f"candidate-block-{inserted_index:04d}", block_type="text", page_index=page_index, bbox=bbox, order=len(page.blocks) + 1, text=inserted_text, confidence=line.confidence, line_ids=[line_id], metadata={"structural_role": structural_role, "candidate_line_review_insertion": metadata["candidate_line_review_insertion"]}, ) page.blocks.append(block) return { "row_id": row.get("row_id"), "sample_id": row.get("sample_id"), "candidate_line_id": row.get("candidate_line_id"), "inserted_line_id": line_id, "inserted_block_id": block.block_id, "review_status": row.get("_normalized_status"), "reviewer": row.get("reviewer") or "", "candidate_text": row.get("candidate_text") or "", "inserted_text": inserted_text, "candidate_bbox": bbox.to_list(), "candidate_confidence": row.get("candidate_confidence"), "candidate_detector_source_pass": row.get("candidate_detector_source_pass"), "structural_role": structural_role, "crop_path": row.get("crop_path") or "", "crop_sha256": row.get("crop_sha256") or "", } def apply_candidate_line_reviews(args: argparse.Namespace) -> dict[str, Any]: source_out = Path(args.book_output_dir) review_path = Path(args.candidate_review_jsonl) out = Path(args.out) accepted_statuses = {str(status).strip().lower() for status in args.accepted_statuses if str(status).strip()} if not accepted_statuses: raise ValueError("--accepted-statuses must contain at least one status") run_path = source_out / "book-run.json" run = _read_json_object(run_path) pages = run.get("pages") if not isinstance(pages, list) or not all(isinstance(page, dict) for page in pages): raise ValueError(f"book-run.json pages must be a list of objects: {run_path}") review_rows = _read_review_rows(review_path) accepted, skipped = _accepted_rows(review_rows, accepted_statuses, review_path=review_path) rows_by_sample = _rows_by_sample(accepted) out.mkdir(parents=True, exist_ok=True) documents_dir = out / "book-documents" applied_rows: list[dict[str, Any]] = [] page_reports: list[dict[str, Any]] = [] matched_samples: set[str] = set() inserted_index = 1 for page_record in pages: page_id = str(page_record.get("page_id") or "") sample_id = str(page_record.get("score_document_sample_id") or page_id) source_doc = Path(str(page_record.get("book_document_json") or page_record.get("document_json") or "")) if not source_doc.is_file(): raise ValueError(f"{page_id}: source book document is missing: {source_doc}") document = Document.from_dict(_read_json_object(source_doc)) page_applied: list[dict[str, Any]] = [] for row in rows_by_sample.get(sample_id, []): applied = _insert_candidate_line(document, row, inserted_index=inserted_index, review_path=review_path) inserted_index += 1 applied_rows.append(applied) page_applied.append(applied) matched_samples.add(sample_id) if page_applied: _place_candidate_insertions(document) line_count = _document_line_count(document) document_dir = documents_dir / sample_id _write_document_artifacts(document_dir, document) page_reports.append( { "page_number": page_record.get("page_number"), "page_id": page_id, "source_document": str(source_doc), "source_document_sha256": sha256_file(source_doc), "document_dir": str(document_dir), "document_json": str(document_dir / "document.json"), "document_json_sha256": sha256_file(document_dir / "document.json"), "document_md_sha256": sha256_file(document_dir / "document.md"), "document_body_md_sha256": sha256_file(document_dir / "document.body.md"), "score_document_sample_id": sample_id, "applied_count": len(page_applied), "line_count": line_count, } ) unmatched_samples = sorted(set(rows_by_sample) - matched_samples) if unmatched_samples: raise ValueError(f"accepted candidate review rows did not match book samples: {unmatched_samples}") book_md = out / "book.md" book_body_md = out / "book.body.md" _write_aggregate_markdown(book_md, page_reports, body=False) _write_aggregate_markdown(book_body_md, page_reports, body=True) score_dir = _write_score_projection(out, page_reports) output_run_path = _write_chainable_book_run( out=out, source_run=run, source_out=source_out, source_run_path=run_path, review_path=review_path, score_dir=score_dir, page_reports=page_reports, book_md=book_md, book_body_md=book_body_md, ) audit = { "tool": "tools/apply_candidate_line_reviews.py", "created_at_utc": datetime.now(UTC).isoformat(), "source_book_output_dir": str(source_out), "source_book_run": str(run_path), "source_book_run_sha256": sha256_file(run_path), "candidate_review_jsonl": str(review_path), "candidate_review_jsonl_sha256": sha256_file(review_path), "accepted_statuses": sorted(accepted_statuses), "review_rows": len(review_rows), "applied_count": len(applied_rows), "skipped_count": len(skipped), "documents_dir": str(documents_dir), "book_md": str(book_md), "book_md_sha256": sha256_file(book_md), "book_body_md": str(book_body_md), "book_body_md_sha256": sha256_file(book_body_md), "book_run": str(output_run_path), "book_run_sha256": sha256_file(output_run_path), "score_documents_dir": str(score_dir), "pages": page_reports, "applied_rows": applied_rows, "skipped_rows": skipped, "claim_scope": "candidate-line review applied editable book projection; not ground truth unless review protocol is separately audited", } pairs_path = out / "candidate-line-insertion-pairs.jsonl" pairs_path.write_text( "".join( json.dumps( { "sample_id": f"{row['sample_id']}:{row['inserted_line_id']}", "language": "limbu", "noisy_text": row["candidate_text"], "clean_text": row["inserted_text"], "source": "candidate_line_review_packet", "review_status": row["review_status"], "reviewer": row["reviewer"], "candidate_line_id": row["candidate_line_id"], "inserted_line_id": row["inserted_line_id"], "crop_path": row.get("crop_path") or "", "crop_sha256": row.get("crop_sha256") or "", }, ensure_ascii=False, ) + "\n" for row in applied_rows ), encoding="utf-8", ) audit_path = out / "candidate-line-insertion-audit.json" audit["audit_path"] = str(audit_path) audit["insertion_pairs"] = str(pairs_path) audit["insertion_pairs_sha256"] = sha256_file(pairs_path) audit_path.write_text(json.dumps(audit, ensure_ascii=False, indent=2) + "\n", encoding="utf-8") return audit def main(argv: list[str] | None = None) -> int: parser = argparse.ArgumentParser(description=__doc__) parser.add_argument("book_output_dir", type=Path) parser.add_argument("--candidate-review-jsonl", "--candidate-review-file", dest="candidate_review_jsonl", type=Path, required=True) parser.add_argument("--out", type=Path, required=True) parser.add_argument("--accepted-statuses", nargs="+", default=list(DEFAULT_ACCEPTED_STATUSES)) args = parser.parse_args(argv) result = apply_candidate_line_reviews(args) print(result["audit_path"]) print(result["insertion_pairs"]) print(f"applied_candidate_lines={result['applied_count']}") return 0 if __name__ == "__main__": raise SystemExit(main())