Download ocr-tech/tests/test_apply_limbu_book_split_worksheet_review.py from prosperious/ocr-projects-archive: direct link, hf CLI and curl.
- Browser
- Download file 11.2 kB
-
https://huggingface.co/prosperious/ocr-projects-archive/resolve/main/ocr-tech/tests/test_apply_limbu_book_split_worksheet_review.py
- Command line
-
hf download hf://prosperious/ocr-projects-archive/ocr-tech/tests/test_apply_limbu_book_split_worksheet_review.py
-
curl -L -o test_apply_limbu_book_split_worksheet_review.py https://huggingface.co/prosperious/ocr-projects-archive/resolve/main/ocr-tech/tests/test_apply_limbu_book_split_worksheet_review.py
11.2 kB
| from __future__ import annotations | |
| import csv | |
| import importlib.util | |
| import json | |
| import sys | |
| from argparse import Namespace | |
| from pathlib import Path | |
| import pytest | |
| from PIL import Image, ImageDraw | |
| from ocrtech.manifest import sha256_file | |
| KNOWN_PACKET_DIR_BUG = pytest.mark.xfail( | |
| strict=True, | |
| raises=ValueError, | |
| reason=( | |
| "known bug: the wrapper writes its reviewed TSV into a new directory but does not pass the " | |
| "original unresolved packet dir (packet_dir) to apply_unresolved_review_packet, whose audit " | |
| "needs book-unresolved-review-packet.json there; nothing in the chain records that dir" | |
| ), | |
| ) | |
| def _load_tool(name: str): | |
| path = Path(__file__).resolve().parents[1] / "tools" / f"{name}.py" | |
| spec = importlib.util.spec_from_file_location(name, path) | |
| assert spec is not None | |
| assert spec.loader is not None | |
| module = importlib.util.module_from_spec(spec) | |
| sys.modules[spec.name] = module | |
| spec.loader.exec_module(module) | |
| return module | |
| def _write_book(tmp_path: Path) -> Path: | |
| image = tmp_path / "page.png" | |
| page = Image.new("RGB", (260, 100), "white") | |
| draw = ImageDraw.Draw(page) | |
| draw.rectangle([20, 24, 70, 42], fill="black") | |
| draw.rectangle([105, 24, 150, 42], fill="black") | |
| draw.rectangle([180, 24, 230, 42], fill="black") | |
| page.save(image) | |
| book = tmp_path / "book" | |
| doc_dir = book / "book-documents" / "p001" | |
| doc_dir.mkdir(parents=True) | |
| document = { | |
| "schema_version": "1.0", | |
| "source_path": str(image), | |
| "pages": [ | |
| { | |
| "page_index": 0, | |
| "width": 260, | |
| "height": 100, | |
| "text_lines": [ | |
| { | |
| "line_id": "line-1", | |
| "page_index": 0, | |
| "text": "old merged text", | |
| "bbox": [10, 18, 230, 32], | |
| "confidence": 0.6, | |
| "metadata": {"structural_role": "text"}, | |
| } | |
| ], | |
| "blocks": [ | |
| { | |
| "block_id": "block-1", | |
| "block_type": "text", | |
| "page_index": 0, | |
| "bbox": [10, 18, 230, 32], | |
| "order": 1, | |
| "text": "old merged text", | |
| "confidence": 0.6, | |
| "line_ids": ["line-1"], | |
| "metadata": {"structural_role": "text"}, | |
| } | |
| ], | |
| } | |
| ], | |
| "tables": [], | |
| "figures": [], | |
| "metadata": {}, | |
| } | |
| document_json = doc_dir / "document.json" | |
| document_json.write_text(json.dumps(document, ensure_ascii=False) + "\n", encoding="utf-8") | |
| (doc_dir / "document.md").write_text("old merged text\n", encoding="utf-8") | |
| (doc_dir / "document.body.md").write_text("old merged text\n", encoding="utf-8") | |
| (book / "book-run.json").write_text( | |
| json.dumps( | |
| { | |
| "tool": "test", | |
| "output_dir": str(book), | |
| "page_count": 1, | |
| "line_count": 1, | |
| "review_line_count": 0, | |
| "script_counts": {"devanagari_limbu": 1}, | |
| "book_md": str(book / "book.md"), | |
| "book_body_md": str(book / "book.body.md"), | |
| "pages": [ | |
| { | |
| "page_number": 1, | |
| "page_id": "p001", | |
| "book_document_json": str(document_json), | |
| "book_document_dir": str(doc_dir), | |
| "score_document_sample_id": "p001", | |
| "line_count": 1, | |
| "review_line_count": 0, | |
| "script_counts": {"devanagari_limbu": 1}, | |
| } | |
| ], | |
| }, | |
| ensure_ascii=False, | |
| ) | |
| + "\n", | |
| encoding="utf-8", | |
| ) | |
| (book / "book.md").write_text("# Limbu Book OCR\n\n## Page 1: p001\n\nold merged text\n", encoding="utf-8") | |
| (book / "book.body.md").write_text("# Limbu Book OCR\n\n## Page 1: p001\n\nold merged text\n", encoding="utf-8") | |
| return book | |
| def _write_prefilled_packet(tmp_path: Path, book: Path) -> Path: | |
| crop = tmp_path / "crop.png" | |
| Image.open(tmp_path / "page.png").crop((10, 18, 240, 50)).save(crop) | |
| overlay = tmp_path / "overlay.png" | |
| Image.open(tmp_path / "page.png").crop((10, 18, 240, 50)).save(overlay) | |
| document_json = book / "book-documents" / "p001" / "document.json" | |
| packet = tmp_path / "prefilled.tsv" | |
| with packet.open("w", encoding="utf-8", newline="") as handle: | |
| fieldnames = [ | |
| "packet_row_id", | |
| "review_item_type", | |
| "book_page_number", | |
| "book_page_id", | |
| "book_document", | |
| "book_line_id", | |
| "candidate_row_id", | |
| "candidate_line_id", | |
| "candidate_structural_role", | |
| "review_category", | |
| "suggested_review_action", | |
| "confidence", | |
| "bbox", | |
| "text", | |
| "source_texts", | |
| "review_reasons", | |
| "crop_path", | |
| "crop_sha256", | |
| "review_status", | |
| "corrected_text", | |
| "split_corrected_texts", | |
| "split_bboxes", | |
| "reviewer", | |
| "notes", | |
| "split_suggestion_overlay_path", | |
| "split_suggestion_overlay_sha256", | |
| "split_suggestion_segment_count", | |
| ] | |
| writer = csv.DictWriter(handle, fieldnames=fieldnames, delimiter="\t", lineterminator="\n") | |
| writer.writeheader() | |
| writer.writerow( | |
| { | |
| "packet_row_id": "existing-1", | |
| "review_item_type": "existing_book_line", | |
| "book_page_number": "1", | |
| "book_page_id": "p001", | |
| "book_document": str(document_json), | |
| "book_line_id": "line-1", | |
| "review_category": "existing_low_confidence_line", | |
| "suggested_review_action": "correct_existing_line_or_accept", | |
| "confidence": "0.6", | |
| "bbox": "[10,18,230,32]", | |
| "text": "old merged text", | |
| "source_texts": "", | |
| "review_reasons": "low_confidence", | |
| "crop_path": str(crop), | |
| "crop_sha256": sha256_file(crop), | |
| "review_status": "pending", | |
| "corrected_text": "", | |
| "split_corrected_texts": "", | |
| "split_bboxes": json.dumps([[20, 24, 50, 18], [105, 24, 45, 18], [180, 24, 50, 18]], ensure_ascii=False), | |
| "reviewer": "", | |
| "notes": "", | |
| "split_suggestion_overlay_path": str(overlay), | |
| "split_suggestion_overlay_sha256": sha256_file(overlay), | |
| "split_suggestion_segment_count": "3", | |
| } | |
| ) | |
| return packet | |
| def _write_worksheet(tmp_path: Path, packet: Path, *, reviewed: bool) -> Path: | |
| prepare = _load_tool("prepare_limbu_book_split_transcription_worksheet") | |
| report = prepare.prepare_worksheet( | |
| Namespace( | |
| prefilled_packet=packet, | |
| out=tmp_path / "worksheet", | |
| segment_padding=2, | |
| include_reviewed=False, | |
| ) | |
| ) | |
| report_path = Path(report["report_json"]) | |
| if reviewed: | |
| worksheet_tsv = Path(report["worksheet_tsv"]) | |
| with worksheet_tsv.open("r", encoding="utf-8", newline="") as handle: | |
| reader = csv.DictReader(handle, delimiter="\t") | |
| fieldnames = reader.fieldnames | |
| rows = list(reader) | |
| assert fieldnames is not None | |
| edits = {"1": ("text", "पहिलो"), "2": ("non_text", ""), "3": ("verified", "दोस्रो")} | |
| for row in rows: | |
| row["segment_status"], row["segment_text"] = edits[row["segment_index"]] | |
| with worksheet_tsv.open("w", encoding="utf-8", newline="") as handle: | |
| writer = csv.DictWriter(handle, fieldnames=fieldnames, delimiter="\t", lineterminator="\n") | |
| writer.writeheader() | |
| writer.writerows(rows) | |
| payload = json.loads(report_path.read_text(encoding="utf-8")) | |
| payload["worksheet_tsv_sha256"] = sha256_file(worksheet_tsv) | |
| report_path.write_text(json.dumps(payload, ensure_ascii=False, indent=2) + "\n", encoding="utf-8") | |
| return report_path | |
| def test_apply_limbu_book_split_worksheet_review_applies_reviewed_split(tmp_path: Path) -> None: | |
| tool = _load_tool("apply_limbu_book_split_worksheet_review") | |
| book = _write_book(tmp_path) | |
| packet = _write_prefilled_packet(tmp_path, book) | |
| worksheet_report = _write_worksheet(tmp_path, packet, reviewed=True) | |
| report = tool.apply_split_worksheet_review( | |
| Namespace( | |
| book_output_dir=book, | |
| prefilled_packet=packet, | |
| worksheet_report=worksheet_report, | |
| out=tmp_path / "reviewed-application", | |
| mark_status="corrected", | |
| accepted_statuses=["corrected"], | |
| text_statuses=["text", "verified"], | |
| drop_statuses=["non_text"], | |
| ) | |
| ) | |
| assert report["compiled_count"] == 1 | |
| assert report["accepted_counts"] == {"existing_book_line": 1} | |
| assert report["book_line_applied_count"] == 1 | |
| assert report["candidate_line_applied_count"] == 0 | |
| assert report["application_audit_passed"] is True | |
| final_book = Path(report["application_dir"]) / "final-book" | |
| book_md = (final_book / "book.md").read_text(encoding="utf-8") | |
| assert "पहिलो" in book_md | |
| assert "दोस्रो" in book_md | |
| assert "old merged text" not in book_md | |
| final_run = json.loads((final_book / "book-run.json").read_text(encoding="utf-8")) | |
| assert final_run["line_count"] == 2 | |
| assert final_run["pages"][0]["book_line_review_split_count"] == 1 | |
| def test_apply_limbu_book_split_worksheet_review_keeps_pending_noop(tmp_path: Path) -> None: | |
| tool = _load_tool("apply_limbu_book_split_worksheet_review") | |
| book = _write_book(tmp_path) | |
| packet = _write_prefilled_packet(tmp_path, book) | |
| worksheet_report = _write_worksheet(tmp_path, packet, reviewed=False) | |
| report = tool.apply_split_worksheet_review( | |
| Namespace( | |
| book_output_dir=book, | |
| prefilled_packet=packet, | |
| worksheet_report=worksheet_report, | |
| out=tmp_path / "pending-application", | |
| mark_status="corrected", | |
| accepted_statuses=["corrected"], | |
| text_statuses=["text", "verified"], | |
| drop_statuses=["non_text"], | |
| ) | |
| ) | |
| assert report["compiled_count"] == 0 | |
| assert report["accepted_counts"] == {} | |
| assert report["book_line_applied_count"] == 0 | |
| assert report["candidate_line_applied_count"] == 0 | |
| assert report["application_audit_passed"] is True | |
| final_book = Path(report["application_dir"]) / "final-book" | |
| assert "old merged text" in (final_book / "book.md").read_text(encoding="utf-8") | |