Add 1000 files (ocr-tech/tests/test_audit_limbu_book_missed_line_review_coverage.py ...)
d82db76 verified Download ocr-tech/tools/apply_page_reading_order_strategy.py from prosperious/ocr-projects-archive: direct link, hf CLI and curl.
- Browser
- Download file 10.6 kB
-
https://huggingface.co/prosperious/ocr-projects-archive/resolve/main/ocr-tech/tools/apply_page_reading_order_strategy.py
- Command line
-
hf download hf://prosperious/ocr-projects-archive/ocr-tech/tools/apply_page_reading_order_strategy.py
-
curl -L -o apply_page_reading_order_strategy.py https://huggingface.co/prosperious/ocr-projects-archive/resolve/main/ocr-tech/tools/apply_page_reading_order_strategy.py
10.6 kB
| #!/usr/bin/env python3 | |
| """Apply simple geometry reading-order strategies to existing OCR documents.""" | |
| from __future__ import annotations | |
| import argparse | |
| import json | |
| import math | |
| import shutil | |
| from datetime import datetime, timezone | |
| from pathlib import Path | |
| from statistics import median | |
| from typing import Any | |
| from ocrtech.linearization import column_major_indices | |
| from ocrtech.markdown import render_document_markdown | |
| from ocrtech.manifest import sha256_file | |
| from ocrtech.schemas import Block, Document, TextLine, union_bbox | |
| REPORT_JSON = "reading-order-strategy-application.json" | |
| def _read_json_object(path: Path) -> dict[str, Any]: | |
| try: | |
| payload = json.loads(path.read_text(encoding="utf-8")) | |
| except FileNotFoundError as exc: | |
| raise ValueError(f"required JSON file is missing: {path}") from exc | |
| except json.JSONDecodeError as exc: | |
| raise ValueError(f"invalid JSON in {path}: {exc}") from exc | |
| if not isinstance(payload, dict): | |
| raise ValueError(f"JSON file must contain an object: {path}") | |
| return payload | |
| def _document_paths(documents_dir: Path) -> dict[str, Path]: | |
| if not documents_dir.is_dir(): | |
| raise ValueError(f"documents directory does not exist: {documents_dir}") | |
| paths: dict[str, Path] = {} | |
| for child in sorted(documents_dir.iterdir()): | |
| if not child.is_dir(): | |
| continue | |
| document_path = child / "document.json" | |
| if document_path.is_file(): | |
| paths[child.name] = document_path | |
| if not paths: | |
| raise ValueError(f"documents directory contains no sample document.json files: {documents_dir}") | |
| return paths | |
| def _has_two_column_shape(lines: list[TextLine], image_width: float | None) -> bool: | |
| if len(lines) < 8 or not image_width or image_width <= 0: | |
| return False | |
| midpoint = image_width / 2.0 | |
| left = [line for line in lines if line.bbox.x + line.bbox.w / 2.0 < midpoint] | |
| right = [line for line in lines if line.bbox.x + line.bbox.w / 2.0 >= midpoint] | |
| if len(left) < 3 or len(right) < 3: | |
| return False | |
| left_median = median(line.bbox.x + line.bbox.w / 2.0 for line in left) | |
| right_median = median(line.bbox.x + line.bbox.w / 2.0 for line in right) | |
| return (right_median - left_median) >= image_width * 0.22 | |
| def _line_key(line: TextLine, *, mode: str, image_width: float | None, row_height: float) -> tuple[float, float, float, str]: | |
| x_center = line.bbox.x + line.bbox.w / 2.0 | |
| y_center = line.bbox.y + line.bbox.h / 2.0 | |
| width = image_width or 0.0 | |
| if mode == "Columns LTR": | |
| return (0 if x_center < width / 2.0 else 1, y_center, x_center, line.line_id or "") | |
| if mode == "Columns RTL": | |
| return (0 if x_center >= width / 2.0 else 1, y_center, x_center, line.line_id or "") | |
| if mode == "Rows RTL": | |
| return (math.floor(y_center / row_height), -x_center, y_center, line.line_id or "") | |
| return (math.floor(y_center / row_height), x_center, y_center, line.line_id or "") | |
| def _sort_top_anchored_rows(lines: list[TextLine], *, mode: str, row_height: float) -> list[TextLine]: | |
| if not lines: | |
| return [] | |
| row_tolerance = max(12.0, row_height * 0.75) | |
| rows: list[list[TextLine]] = [] | |
| row_anchors: list[float] = [] | |
| for line in sorted(lines, key=lambda item: (item.bbox.y, item.bbox.x, item.line_id or "")): | |
| anchor = line.bbox.y | |
| if not rows or abs(anchor - row_anchors[-1]) > row_tolerance: | |
| rows.append([line]) | |
| row_anchors.append(anchor) | |
| continue | |
| rows[-1].append(line) | |
| row_anchors[-1] = sum(item.bbox.y for item in rows[-1]) / len(rows[-1]) | |
| rows.sort(key=lambda row: min(line.bbox.y for line in row)) | |
| sorted_lines: list[TextLine] = [] | |
| reverse_x = mode == "Rows RTL Top" | |
| for row in rows: | |
| sorted_lines.extend( | |
| sorted( | |
| row, | |
| key=lambda line: ( | |
| -(line.bbox.x + line.bbox.w / 2.0) if reverse_x else (line.bbox.x + line.bbox.w / 2.0), | |
| line.bbox.y, | |
| line.line_id or "", | |
| ), | |
| ) | |
| ) | |
| return sorted_lines | |
| def _resolved_mode(mode: str, lines: list[TextLine], image_width: float | None) -> str: | |
| if mode != "Auto": | |
| return mode | |
| return "Columns LTR" if _has_two_column_shape(lines, image_width) else "Rows LTR" | |
| def _block_key(block: Block, line_positions: dict[str, int]) -> tuple[int, int, str]: | |
| if block.line_ids: | |
| positions = [line_positions[line_id] for line_id in block.line_ids if line_id in line_positions] | |
| if positions: | |
| return (min(positions), block.order, block.block_id) | |
| return (10**9, block.order, block.block_id) | |
| def _apply_to_document(document: Document, mode: str) -> tuple[Document, list[dict[str, Any]]]: | |
| reports: list[dict[str, Any]] = [] | |
| for page in document.pages: | |
| lines = list(page.text_lines) | |
| heights = [line.bbox.h for line in lines if line.bbox.h > 0] | |
| row_height = max(12.0, float(median(heights)) if heights else 24.0) | |
| actual_mode = _resolved_mode(mode, lines, page.width) | |
| if actual_mode == "Column Major": | |
| order = column_major_indices( | |
| [(line.bbox.x, line.bbox.y, line.bbox.w, line.bbox.h) for line in lines] | |
| ) | |
| sorted_lines = [lines[index] for index in order] | |
| elif actual_mode in {"Rows LTR Top", "Rows RTL Top"}: | |
| sorted_lines = _sort_top_anchored_rows(lines, mode=actual_mode, row_height=row_height) | |
| else: | |
| sorted_lines = sorted( | |
| lines, | |
| key=lambda line: _line_key(line, mode=actual_mode, image_width=page.width, row_height=row_height), | |
| ) | |
| page.text_lines = sorted_lines | |
| lines_by_id = {line.line_id or "": line for line in sorted_lines} | |
| line_positions = {line.line_id or "": index for index, line in enumerate(sorted_lines)} | |
| for block in page.blocks: | |
| if not block.line_ids: | |
| continue | |
| block.line_ids = sorted( | |
| block.line_ids, | |
| key=lambda line_id: line_positions.get(line_id, 10**9), | |
| ) | |
| block_lines = [lines_by_id[line_id] for line_id in block.line_ids if line_id in lines_by_id] | |
| if block_lines: | |
| block.text = "\n".join(line.text for line in block_lines) | |
| block.bbox = union_bbox([line.bbox for line in block_lines]) | |
| page.blocks = sorted(page.blocks, key=lambda block: _block_key(block, line_positions)) | |
| for order, block in enumerate(page.blocks, start=1): | |
| block.order = order | |
| page.metadata = dict(page.metadata or {}) | |
| page.metadata["reading_order_strategy"] = { | |
| "requested_mode": mode, | |
| "resolved_mode": actual_mode, | |
| "row_height": row_height, | |
| "line_count": len(sorted_lines), | |
| } | |
| reports.append( | |
| { | |
| "page_index": page.page_index, | |
| "requested_mode": mode, | |
| "resolved_mode": actual_mode, | |
| "row_height": row_height, | |
| "line_count": len(sorted_lines), | |
| } | |
| ) | |
| document.metadata = dict(document.metadata or {}) | |
| document.metadata["reading_order_strategy"] = {"requested_mode": mode} | |
| return document, reports | |
| def _write_document_artifacts(document_dir: Path, document: Document) -> None: | |
| document_dir.mkdir(parents=True, exist_ok=True) | |
| (document_dir / "document.json").write_text(json.dumps(document.to_dict(), ensure_ascii=False, indent=2) + "\n", encoding="utf-8") | |
| (document_dir / "document.md").write_text(render_document_markdown(document), encoding="utf-8") | |
| (document_dir / "document.body.md").write_text(render_document_markdown(document, include_structural_roles=False), encoding="utf-8") | |
| def apply_strategy(args: argparse.Namespace) -> dict[str, Any]: | |
| source_documents_dir = args.source_documents_dir | |
| out = args.out | |
| if source_documents_dir.resolve() == out.resolve(): | |
| raise ValueError("--out must not be the same directory as --source-documents-dir") | |
| if out.exists() and any(out.iterdir()): | |
| if not args.force: | |
| raise FileExistsError(f"output directory exists and is not empty: {out}") | |
| if out.resolve() in {Path.cwd().resolve(), Path("/")}: | |
| raise ValueError(f"refusing to remove unsafe output directory: {out}") | |
| shutil.rmtree(out) | |
| out.mkdir(parents=True, exist_ok=True) | |
| source_paths = _document_paths(source_documents_dir) | |
| pages: list[dict[str, Any]] = [] | |
| for sample_id, document_path in sorted(source_paths.items()): | |
| document = Document.from_dict(_read_json_object(document_path)) | |
| document, page_reports = _apply_to_document(document, args.mode) | |
| target_dir = out / sample_id | |
| _write_document_artifacts(target_dir, document) | |
| pages.append( | |
| { | |
| "sample_id": sample_id, | |
| "source_document": str(document_path), | |
| "source_document_sha256": sha256_file(document_path), | |
| "document_json": str(target_dir / "document.json"), | |
| "document_json_sha256": sha256_file(target_dir / "document.json"), | |
| "page_reports": page_reports, | |
| } | |
| ) | |
| report = { | |
| "tool": "tools/apply_page_reading_order_strategy.py", | |
| "created_at_utc": datetime.now(timezone.utc).isoformat(), | |
| "source_documents_dir": str(source_documents_dir), | |
| "out": str(out), | |
| "mode": args.mode, | |
| "document_count": len(source_paths), | |
| "pages": pages, | |
| } | |
| report_path = out / REPORT_JSON | |
| report["report_json"] = str(report_path) | |
| report_path.write_text(json.dumps(report, ensure_ascii=False, indent=2) + "\n", encoding="utf-8") | |
| return report | |
| def main() -> int: | |
| parser = argparse.ArgumentParser(description=__doc__) | |
| parser.add_argument("--source-documents-dir", type=Path, required=True) | |
| parser.add_argument("--out", type=Path, required=True) | |
| parser.add_argument( | |
| "--mode", | |
| choices=["Auto", "Rows LTR", "Rows RTL", "Rows LTR Top", "Rows RTL Top", "Columns LTR", "Columns RTL", "Column Major"], | |
| required=True, | |
| ) | |
| parser.add_argument("--force", action="store_true") | |
| args = parser.parse_args() | |
| report = apply_strategy(args) | |
| print(report["report_json"]) | |
| print(f"documents={report['document_count']} mode={report['mode']}") | |
| return 0 | |
| if __name__ == "__main__": | |
| raise SystemExit(main()) | |