from __future__ import annotations import importlib.util import json import sys from argparse import Namespace from pathlib import Path from PIL import Image from ocrtech.manifest import sha256_file def _load_tool(): path = Path(__file__).resolve().parents[1] / "tools" / "apply_candidate_line_reviews.py" spec = importlib.util.spec_from_file_location("apply_candidate_line_reviews", path) assert spec is not None assert spec.loader is not None module = importlib.util.module_from_spec(spec) sys.modules[spec.name] = module spec.loader.exec_module(module) return module def _load_audit_tool(): path = Path(__file__).resolve().parents[1] / "tools" / "audit_candidate_line_review_application.py" spec = importlib.util.spec_from_file_location("audit_candidate_line_review_application", path) assert spec is not None assert spec.loader is not None module = importlib.util.module_from_spec(spec) sys.modules[spec.name] = module spec.loader.exec_module(module) return module def _document(source: Path, *, structural_marker: bool = False) -> dict[str, object]: role = "page_marker" if structural_marker else "text" return { "schema_version": "1.0", "source_path": str(source), "pages": [ { "page_index": 0, "width": 120, "height": 100, "text_lines": [ { "line_id": "line-001", "page_index": 0, "text": "पहिलो", "bbox": [10, 10, 50, 12], "confidence": 0.9, "metadata": {"structural_role": "text"}, } ], "blocks": [ { "block_id": "block-001", "block_type": "text", "page_index": 0, "bbox": [10, 10, 50, 12], "order": 1, "text": "पहिलो", "confidence": 0.9, "line_ids": ["line-001"], "metadata": {"structural_role": role if structural_marker else "text"}, } ], } ], "tables": [], "figures": [], "metadata": {}, } def _write_source_book(tmp_path: Path) -> Path: image = tmp_path / "page.png" Image.new("RGB", (120, 100), "white").save(image) book = tmp_path / "book" doc_dir = book / "book-documents" / "sample-1" doc_dir.mkdir(parents=True) document = _document(image) (doc_dir / "document.json").write_text(json.dumps(document, ensure_ascii=False), encoding="utf-8") (doc_dir / "document.md").write_text("पहिलो\n", encoding="utf-8") (doc_dir / "document.body.md").write_text("पहिलो\n", encoding="utf-8") run = { "pages": [ { "page_number": 1, "page_id": "page-1", "book_document_json": str(doc_dir / "document.json"), "score_document_sample_id": "sample-1", } ] } (book / "book-run.json").write_text(json.dumps(run, ensure_ascii=False), encoding="utf-8") return book def test_apply_candidate_line_reviews_inserts_accepted_candidate_line(tmp_path: Path) -> None: tool = _load_tool() audit_tool = _load_audit_tool() book = _write_source_book(tmp_path) review_jsonl = tmp_path / "candidate-line-review.jsonl" review_jsonl.write_text( json.dumps( { "row_id": "row-1", "review_status": "accepted", "reviewer": "reviewer-a", "sample_id": "sample-1", "page_index": 0, "candidate_line_id": "cand-1", "candidate_source_line_ids": ["cand-1"], "candidate_bbox": [10, 40, 60, 12], "candidate_text": "दोस्रो", "corrected_text": "दोस्रो", "candidate_confidence": 0.77, "candidate_detector_source_pass": "secondary", "candidate_structural_role": "text", "max_selected_iou": 0.0, "best_selected_line_id": "", }, ensure_ascii=False, ) + "\n", encoding="utf-8", ) result = tool.apply_candidate_line_reviews( Namespace( book_output_dir=book, candidate_review_jsonl=review_jsonl, out=tmp_path / "candidate-applied", accepted_statuses=["accepted"], ) ) assert result["applied_count"] == 1 book_md = (tmp_path / "candidate-applied" / "book.md").read_text(encoding="utf-8") assert "पहिलो" in book_md assert "दोस्रो" in book_md document = json.loads((tmp_path / "candidate-applied" / "score-documents" / "sample-1" / "document.json").read_text(encoding="utf-8")) lines = document["pages"][0]["text_lines"] assert [line["text"] for line in lines] == ["पहिलो", "दोस्रो"] inserted = lines[1] assert inserted["line_id"] == "candidate-insert-0001" assert inserted["metadata"]["candidate_line_review_insertion"]["candidate_line_id"] == "cand-1" assert inserted["metadata"]["candidate_line_review_insertion"]["reviewer"] == "reviewer-a" final_run = json.loads((tmp_path / "candidate-applied" / "book-run.json").read_text(encoding="utf-8")) assert final_run["line_count"] == 2 assert final_run["pages"][0]["line_count"] == 2 assert final_run["pages"][0]["candidate_line_review_applied_count"] == 1 assert final_run["pages"][0]["book_document_json"].endswith("candidate-applied/book-documents/sample-1/document.json") source_document_json = Path(final_run["pages"][0]["source_book_document_json"]) assert final_run["pages"][0]["source_book_document_json_sha256"] == sha256_file(source_document_json) assert result["book_run_sha256"] == sha256_file(tmp_path / "candidate-applied" / "book-run.json") assert Path(result["insertion_pairs"]).exists() audit = audit_tool.audit_application(Namespace(output_dir=tmp_path / "candidate-applied", out=tmp_path / "candidate-applied" / "audit.json")) assert audit["audit_passed"] is True assert audit["applied_count"] == 1 assert audit["inserted_line_count"] == 1 assert audit["total_inserted_line_count"] == 1 assert audit["inherited_inserted_line_count"] == 0 assert audit["insertion_pair_count"] == 1 def test_audit_candidate_line_review_application_counts_only_current_review_insertions(tmp_path: Path) -> None: tool = _load_tool() audit_tool = _load_audit_tool() book = _write_source_book(tmp_path) first_review = tmp_path / "first-candidate-line-review.jsonl" first_review.write_text( json.dumps( { "row_id": "marker", "review_status": "accepted", "sample_id": "sample-1", "page_index": 0, "candidate_line_id": "cand-marker", "candidate_bbox": [90, 80, 10, 10], "candidate_text": "१", "corrected_text": "१", "candidate_structural_role": "page_marker", }, ensure_ascii=False, ) + "\n", encoding="utf-8", ) first_result = tool.apply_candidate_line_reviews( Namespace( book_output_dir=book, candidate_review_jsonl=first_review, out=tmp_path / "first-applied", accepted_statuses=["accepted"], ) ) assert first_result["applied_count"] == 1 second_review = tmp_path / "second-candidate-line-review.jsonl" second_review.write_text( json.dumps( { "row_id": "body", "review_status": "accepted", "sample_id": "sample-1", "page_index": 0, "candidate_line_id": "cand-body", "candidate_bbox": [10, 40, 60, 12], "candidate_text": "दोस्रो", "corrected_text": "दोस्रो", "candidate_structural_role": "text", }, ensure_ascii=False, ) + "\n", encoding="utf-8", ) second_result = tool.apply_candidate_line_reviews( Namespace( book_output_dir=tmp_path / "first-applied", candidate_review_jsonl=second_review, out=tmp_path / "second-applied", accepted_statuses=["accepted"], ) ) audit = audit_tool.audit_application(Namespace(output_dir=tmp_path / "second-applied", out=tmp_path / "second-applied" / "audit.json")) assert second_result["applied_count"] == 1 assert audit["audit_passed"] is True assert audit["applied_count"] == 1 assert audit["inserted_line_count"] == 1 assert audit["total_inserted_line_count"] == 2 assert audit["inherited_inserted_line_count"] == 1 assert audit["insertion_pair_count"] == 1 def test_apply_candidate_line_reviews_places_page_marker_before_body_text(tmp_path: Path) -> None: tool = _load_tool() book = _write_source_book(tmp_path) review_jsonl = tmp_path / "candidate-line-review.jsonl" review_jsonl.write_text( json.dumps( { "row_id": "row-1", "review_status": "accepted", "reviewer": "reviewer-a", "sample_id": "sample-1", "page_index": 0, "candidate_line_id": "cand-page-marker", "candidate_source_line_ids": ["cand-page-marker"], "candidate_bbox": [100, 85, 10, 10], "candidate_text": "१", "corrected_text": "१", "candidate_confidence": 0.88, "candidate_structural_role": "page_marker", "max_selected_iou": 0.0, "best_selected_line_id": "", }, ensure_ascii=False, ) + "\n", encoding="utf-8", ) result = tool.apply_candidate_line_reviews( Namespace( book_output_dir=book, candidate_review_jsonl=review_jsonl, out=tmp_path / "candidate-applied", accepted_statuses=["accepted"], ) ) assert result["applied_count"] == 1 document = json.loads((tmp_path / "candidate-applied" / "score-documents" / "sample-1" / "document.json").read_text(encoding="utf-8")) page = document["pages"][0] assert [line["text"] for line in page["text_lines"]] == ["१", "पहिलो"] assert [block["line_ids"] for block in sorted(page["blocks"], key=lambda item: item["order"])] == [ ["candidate-insert-0001"], ["line-001"], ] inserted = page["text_lines"][0] assert inserted["metadata"]["structural_role"] == "page_marker" assert inserted["metadata"]["candidate_line_review_insertion"]["placement_policy"] == "front_matter_structural" book_md = (tmp_path / "candidate-applied" / "book.md").read_text(encoding="utf-8") assert book_md.index("१") < book_md.index("पहिलो") def test_apply_candidate_line_reviews_accepts_review_tsv(tmp_path: Path) -> None: tool = _load_tool() audit_tool = _load_audit_tool() book = _write_source_book(tmp_path) crop = tmp_path / "crop.png" Image.new("RGB", (60, 12), "white").save(crop) review_tsv = tmp_path / "prioritized-candidate-line-review.tsv" review_tsv.write_text( "\n".join( [ "\t".join( [ "priority_rank", "row_id", "sample_id", "candidate_line_id", "candidate_structural_role", "candidate_confidence", "max_selected_iou", "candidate_bbox", "candidate_text", "best_selected_text", "crop_path", "crop_sha256", "review_status", "corrected_text", "reviewer", "notes", ] ), "\t".join( [ "1", "row-1", "sample-1", "cand-1", "text", "0.77", "0.12", "[10,40,60,12]", "bad noisy text", "nearby", str(crop), sha256_file(crop), "accepted", "दोस्रो", "reviewer-a", "from prioritized queue", ] ), ] ) + "\n", encoding="utf-8", ) result = tool.apply_candidate_line_reviews( Namespace( book_output_dir=book, candidate_review_jsonl=review_tsv, out=tmp_path / "candidate-applied", accepted_statuses=["accepted"], ) ) assert result["applied_count"] == 1 document = json.loads((tmp_path / "candidate-applied" / "score-documents" / "sample-1" / "document.json").read_text(encoding="utf-8")) inserted = document["pages"][0]["text_lines"][1] assert inserted["text"] == "दोस्रो" assert inserted["confidence"] == 0.77 assert inserted["metadata"]["candidate_line_review_insertion"]["candidate_text"] == "bad noisy text" assert inserted["metadata"]["candidate_line_review_insertion"]["corrected_text"] == "दोस्रो" assert inserted["metadata"]["candidate_line_review_insertion"]["max_selected_iou"] == 0.12 assert inserted["metadata"]["candidate_line_review_insertion"]["crop_path"] == str(crop) assert inserted["metadata"]["candidate_line_review_insertion"]["crop_sha256"] == sha256_file(crop) pairs = [json.loads(line) for line in Path(result["insertion_pairs"]).read_text(encoding="utf-8").splitlines()] assert pairs[0]["crop_path"] == str(crop) assert pairs[0]["crop_sha256"] == sha256_file(crop) audit = audit_tool.audit_application(Namespace(output_dir=tmp_path / "candidate-applied", out=tmp_path / "candidate-applied" / "audit.json")) assert audit["audit_passed"] is True assert audit["review_row_count"] == 1 def test_apply_candidate_line_reviews_accepts_book_missing_line_review_queue_tsv(tmp_path: Path) -> None: tool = _load_tool() book = _write_source_book(tmp_path) crop = tmp_path / "missing-crop.png" Image.new("RGB", (60, 12), "white").save(crop) review_tsv = tmp_path / "book-missing-line-review-queue.tsv" review_tsv.write_text( "\n".join( [ "\t".join( [ "book_page_number", "book_page_id", "book_page_output_dir", "book_document_dir", "priority_rank", "review_priority", "review_category", "suggested_review_action", "candidate_row_id", "candidate_sample_id", "candidate_line_id", "candidate_structural_role", "candidate_confidence", "max_selected_iou", "candidate_bbox", "candidate_text", "best_selected_text", "crop_path", "crop_sha256", "review_status", "corrected_text", "reviewer", "notes", ] ), "\t".join( [ "1", "sample-1", "book/pages/sample-1", "book/book-documents/sample-1", "1", "high", "body_line_candidate", "transcribe_visible_line_or_reject", "candidate-only-0001-sample-1", "sample-1", "cand-1", "text", "0.77", "0.12", "[10,40,60,12]", "bad noisy text", "nearby", str(crop), sha256_file(crop), "accepted", "दोस्रो", "reviewer-a", "from book missing-line queue", ] ), ] ) + "\n", encoding="utf-8", ) result = tool.apply_candidate_line_reviews( Namespace( book_output_dir=book, candidate_review_jsonl=review_tsv, out=tmp_path / "candidate-applied", accepted_statuses=["accepted"], ) ) assert result["applied_count"] == 1 document = json.loads((tmp_path / "candidate-applied" / "score-documents" / "sample-1" / "document.json").read_text(encoding="utf-8")) inserted = document["pages"][0]["text_lines"][1] assert inserted["text"] == "दोस्रो" insertion = inserted["metadata"]["candidate_line_review_insertion"] assert insertion["row_id"] == "candidate-only-0001-sample-1" assert insertion["candidate_line_id"] == "cand-1" assert insertion["reviewer"] == "reviewer-a" def test_apply_candidate_line_reviews_rejects_accepted_candidate_without_corrected_text(tmp_path: Path) -> None: tool = _load_tool() book = _write_source_book(tmp_path) review_jsonl = tmp_path / "candidate-line-review.jsonl" review_jsonl.write_text( json.dumps( { "row_id": "row-1", "review_status": "accepted", "sample_id": "sample-1", "page_index": 0, "candidate_line_id": "cand-1", "candidate_bbox": [10, 40, 60, 12], "candidate_text": "दोस्रो", }, ensure_ascii=False, ) + "\n", encoding="utf-8", ) try: tool.apply_candidate_line_reviews( Namespace( book_output_dir=book, candidate_review_jsonl=review_jsonl, out=tmp_path / "candidate-applied", accepted_statuses=["accepted"], ) ) except ValueError as exc: assert "empty corrected_text" in str(exc) else: raise AssertionError("expected accepted candidate without corrected_text to be rejected") def test_apply_candidate_line_reviews_rejects_crop_hash_mismatch(tmp_path: Path) -> None: tool = _load_tool() book = _write_source_book(tmp_path) crop = tmp_path / "bad-crop.png" Image.new("RGB", (60, 12), "white").save(crop) review_jsonl = tmp_path / "candidate-line-review.jsonl" review_jsonl.write_text( json.dumps( { "row_id": "row-1", "review_status": "accepted", "sample_id": "sample-1", "page_index": 0, "candidate_line_id": "cand-1", "candidate_bbox": [10, 40, 60, 12], "candidate_text": "दोस्रो", "corrected_text": "दोस्रो", "crop_path": str(crop), "crop_sha256": "0" * 64, }, ensure_ascii=False, ) + "\n", encoding="utf-8", ) try: tool.apply_candidate_line_reviews( Namespace( book_output_dir=book, candidate_review_jsonl=review_jsonl, out=tmp_path / "candidate-applied", accepted_statuses=["accepted"], ) ) except ValueError as exc: assert "crop SHA-256 mismatch" in str(exc) else: raise AssertionError("expected crop hash mismatch to be rejected") def test_audit_candidate_line_review_application_rejects_tampered_document(tmp_path: Path) -> None: tool = _load_tool() audit_tool = _load_audit_tool() book = _write_source_book(tmp_path) review_jsonl = tmp_path / "candidate-line-review.jsonl" review_jsonl.write_text( json.dumps( { "row_id": "row-1", "review_status": "accepted", "sample_id": "sample-1", "page_index": 0, "candidate_line_id": "cand-1", "candidate_bbox": [10, 40, 60, 12], "candidate_text": "दोस्रो", "corrected_text": "दोस्रो", }, ensure_ascii=False, ) + "\n", encoding="utf-8", ) tool.apply_candidate_line_reviews( Namespace( book_output_dir=book, candidate_review_jsonl=review_jsonl, out=tmp_path / "candidate-applied", accepted_statuses=["accepted"], ) ) document = tmp_path / "candidate-applied" / "book-documents" / "sample-1" / "document.json" document.write_text(document.read_text(encoding="utf-8") + "\n", encoding="utf-8") audit = audit_tool.audit_application(Namespace(output_dir=tmp_path / "candidate-applied", out=tmp_path / "candidate-applied" / "audit.json")) assert audit["audit_passed"] is False assert any("document.json SHA-256 mismatch" in issue for issue in audit["issues"]) def test_audit_candidate_line_review_application_rejects_inserted_line_without_corrected_text_metadata(tmp_path: Path) -> None: tool = _load_tool() audit_tool = _load_audit_tool() book = _write_source_book(tmp_path) review_jsonl = tmp_path / "candidate-line-review.jsonl" review_jsonl.write_text( json.dumps( { "row_id": "row-1", "review_status": "accepted", "sample_id": "sample-1", "page_index": 0, "candidate_line_id": "cand-1", "candidate_bbox": [10, 40, 60, 12], "candidate_text": "machine", "corrected_text": "दोस्रो", }, ensure_ascii=False, ) + "\n", encoding="utf-8", ) tool.apply_candidate_line_reviews( Namespace( book_output_dir=book, candidate_review_jsonl=review_jsonl, out=tmp_path / "candidate-applied", accepted_statuses=["accepted"], ) ) document_path = tmp_path / "candidate-applied" / "book-documents" / "sample-1" / "document.json" document = json.loads(document_path.read_text(encoding="utf-8")) insertion = document["pages"][0]["text_lines"][1]["metadata"]["candidate_line_review_insertion"] insertion["corrected_text"] = "" document_path.write_text(json.dumps(document, ensure_ascii=False, indent=2) + "\n", encoding="utf-8") application_path = tmp_path / "candidate-applied" / "candidate-line-insertion-audit.json" application = json.loads(application_path.read_text(encoding="utf-8")) application["pages"][0]["document_json_sha256"] = sha256_file(document_path) application["pages"][0]["score_document_sha256"] = sha256_file(tmp_path / "candidate-applied" / "score-documents" / "sample-1" / "document.json") application_path.write_text(json.dumps(application, ensure_ascii=False, indent=2) + "\n", encoding="utf-8") audit = audit_tool.audit_application(Namespace(output_dir=tmp_path / "candidate-applied", out=tmp_path / "candidate-applied" / "audit.json")) assert audit["audit_passed"] is False assert any("missing corrected_text" in issue for issue in audit["issues"]) def test_audit_candidate_line_review_application_flags_oracle_rows(tmp_path: Path) -> None: tool = _load_tool() audit_tool = _load_audit_tool() book = _write_source_book(tmp_path) review_jsonl = tmp_path / "candidate-line-review.jsonl" review_jsonl.write_text( json.dumps( { "row_id": "row-1", "review_status": "accepted", "reviewer": "oracle-reference-overlap-v1", "sample_id": "sample-1", "page_index": 0, "candidate_line_id": "cand-1", "candidate_bbox": [10, 40, 60, 12], "candidate_text": "noisy", "corrected_text": "reference text", "selection_policy": "oracle_reference_overlap_v1", "oracle_reference_line_id": "ref-1", }, ensure_ascii=False, ) + "\n", encoding="utf-8", ) tool.apply_candidate_line_reviews( Namespace( book_output_dir=book, candidate_review_jsonl=review_jsonl, out=tmp_path / "candidate-applied", accepted_statuses=["accepted"], ) ) audit = audit_tool.audit_application(Namespace(output_dir=tmp_path / "candidate-applied", out=tmp_path / "candidate-applied" / "audit.json")) assert audit["audit_passed"] is True assert audit["operational_review_safe"] is False assert audit["reference_oracle_review_count"] == 1 assert any("diagnostic-only" in warning for warning in audit["warnings"]) def test_audit_candidate_line_review_application_does_not_flag_non_oracle_rows(tmp_path: Path) -> None: tool = _load_tool() audit_tool = _load_audit_tool() book = _write_source_book(tmp_path) review_jsonl = tmp_path / "candidate-line-review.jsonl" review_jsonl.write_text( json.dumps( { "row_id": "row-1", "review_status": "accepted", "reviewer": "machine_refined_crop_recognizer_probe", "sample_id": "sample-1", "page_index": 0, "candidate_line_id": "cand-1", "candidate_bbox": [10, 40, 60, 12], "candidate_text": "noisy", "corrected_text": "machine text", "selection_policy": "refined_crop_recognizer_probe_v1", "claim_boundary": "non-oracle machine recognizer probe selection; page-level admission required", }, ensure_ascii=False, ) + "\n", encoding="utf-8", ) tool.apply_candidate_line_reviews( Namespace( book_output_dir=book, candidate_review_jsonl=review_jsonl, out=tmp_path / "candidate-applied", accepted_statuses=["accepted"], ) ) audit = audit_tool.audit_application(Namespace(output_dir=tmp_path / "candidate-applied", out=tmp_path / "candidate-applied" / "audit.json")) assert audit["audit_passed"] is True assert audit["operational_review_safe"] is True assert audit["reference_oracle_review_count"] == 0 def test_apply_candidate_line_reviews_keeps_structural_marker_out_of_body(tmp_path: Path) -> None: tool = _load_tool() book = _write_source_book(tmp_path) review_jsonl = tmp_path / "candidate-line-review.jsonl" review_jsonl.write_text( json.dumps( { "row_id": "row-1", "review_status": "accept", "sample_id": "sample-1", "page_index": 0, "candidate_line_id": "cand-marker", "candidate_bbox": [80, 80, 10, 10], "candidate_text": "१", "corrected_text": "१", "candidate_confidence": 0.93, "candidate_structural_role": "page_marker", }, ensure_ascii=False, ) + "\n", encoding="utf-8", ) tool.apply_candidate_line_reviews( Namespace( book_output_dir=book, candidate_review_jsonl=review_jsonl, out=tmp_path / "candidate-applied", accepted_statuses=["accept"], ) ) assert "१" in (tmp_path / "candidate-applied" / "book.md").read_text(encoding="utf-8") assert "१" not in (tmp_path / "candidate-applied" / "book.body.md").read_text(encoding="utf-8") def test_apply_candidate_line_reviews_rejects_pending_rows(tmp_path: Path) -> None: tool = _load_tool() book = _write_source_book(tmp_path) review_jsonl = tmp_path / "candidate-line-review.jsonl" review_jsonl.write_text( json.dumps( { "row_id": "row-1", "review_status": "pending", "sample_id": "sample-1", "page_index": 0, "candidate_line_id": "cand-1", "candidate_bbox": [10, 40, 60, 12], "candidate_text": "दोस्रो", }, ensure_ascii=False, ) + "\n", encoding="utf-8", ) result = tool.apply_candidate_line_reviews( Namespace( book_output_dir=book, candidate_review_jsonl=review_jsonl, out=tmp_path / "candidate-applied", accepted_statuses=["accepted"], ) ) assert result["applied_count"] == 0 assert result["skipped_count"] == 1 assert "दोस्रो" not in (tmp_path / "candidate-applied" / "book.md").read_text(encoding="utf-8") final_run = json.loads((tmp_path / "candidate-applied" / "book-run.json").read_text(encoding="utf-8")) assert final_run["line_count"] == 1 assert final_run["pages"][0]["candidate_line_review_applied_count"] == 0