Add 1000 files (OCR_INDRA/.gitignore ...)
Browse filesThis view is limited to 50 files because it contains too many changes. See raw diff
- .gitattributes +59 -0
- OCR_INDRA/.gitignore +5 -0
- OCR_INDRA/INDRA_OCR_NEPALI_COMPLETE_REPORT.docx +0 -0
- OCR_INDRA/INDRA_OCR_NEPALI_FEASIBILITY.md +340 -0
- OCR_INDRA/README_BUNDLE.md +14 -0
- OCR_INDRA/docs/INDRA_EXPERIMENTS.md +253 -0
- OCR_INDRA/docs/INDRA_INVESTIGATION.md +292 -0
- OCR_INDRA/docs/REMOTE_RUN_SHEET.md +181 -0
- OCR_INDRA/indra/.gitignore +7 -0
- OCR_INDRA/indra/CHANGELOG.md +97 -0
- OCR_INDRA/indra/EXAMPLES.md +84 -0
- OCR_INDRA/indra/LICENSE +202 -0
- OCR_INDRA/indra/README.md +296 -0
- OCR_INDRA/indra/assets/benchmarks/speed_score.png +3 -0
- OCR_INDRA/indra/assets/benchmarks/throughput.png +3 -0
- OCR_INDRA/indra/assets/datalab-logo.png +0 -0
- OCR_INDRA/indra/assets/examples/chinese_essay.png +3 -0
- OCR_INDRA/indra/assets/examples/multi_column.png +3 -0
- OCR_INDRA/indra/assets/examples/rotated_letter.png +3 -0
- OCR_INDRA/indra/assets/examples/table_pmc.png +3 -0
- OCR_INDRA/indra/bench/opb_indra.py +180 -0
- OCR_INDRA/indra/indra_ocr/__init__.py +51 -0
- OCR_INDRA/indra/indra_ocr/__main__.py +4 -0
- OCR_INDRA/indra/indra_ocr/cli.py +175 -0
- OCR_INDRA/indra/indra_ocr/det_post.py +155 -0
- OCR_INDRA/indra/indra_ocr/document.py +291 -0
- OCR_INDRA/indra/indra_ocr/layout_post.py +385 -0
- OCR_INDRA/indra/indra_ocr/line_mask.py +129 -0
- OCR_INDRA/indra/indra_ocr/math.py +145 -0
- OCR_INDRA/indra/indra_ocr/models.py +106 -0
- OCR_INDRA/indra/indra_ocr/order_post.py +307 -0
- OCR_INDRA/indra/indra_ocr/parallel.py +210 -0
- OCR_INDRA/indra/indra_ocr/pipeline.py +604 -0
- OCR_INDRA/indra/indra_ocr/rec_preproc.py +97 -0
- OCR_INDRA/indra/indra_ocr/textlayer.py +253 -0
- OCR_INDRA/indra/indra_ocr/viewer.py +649 -0
- OCR_INDRA/indra/indra_ocr/vocab.py +289 -0
- OCR_INDRA/indra/pyproject.toml +37 -0
- OCR_INDRA/indra/tests/test_inputs.py +201 -0
- OCR_INDRA/indra/tests/test_textlayer.py +307 -0
- OCR_INDRA/logs/indra_crop_fp32_6000_20261023.log +16 -0
- OCR_INDRA/logs/investigation.md +168 -0
- OCR_INDRA/make_contact_sheet.py +31 -0
- OCR_INDRA/results/manifests_crops_20261023/dev.jsonl +0 -0
- OCR_INDRA/results/manifests_crops_20261023/dev_smoke.jsonl +64 -0
- OCR_INDRA/results/manifests_crops_20261023/train.jsonl +3 -0
- OCR_INDRA/results/padding_variants/predictions.jsonl +0 -0
- OCR_INDRA/results/probe.json +17 -0
- OCR_INDRA/results/smoke/cer_full.json +18 -0
- OCR_INDRA/results/smoke/cer_rec_only.json +18 -0
.gitattributes
CHANGED
|
@@ -33,3 +33,62 @@ saved_model/**/* filter=lfs diff=lfs merge=lfs -text
|
|
| 33 |
*.zip filter=lfs diff=lfs merge=lfs -text
|
| 34 |
*.zst filter=lfs diff=lfs merge=lfs -text
|
| 35 |
*tfevents* filter=lfs diff=lfs merge=lfs -text
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 33 |
*.zip filter=lfs diff=lfs merge=lfs -text
|
| 34 |
*.zst filter=lfs diff=lfs merge=lfs -text
|
| 35 |
*tfevents* filter=lfs diff=lfs merge=lfs -text
|
| 36 |
+
OCR_INDRA/indra/assets/benchmarks/speed_score.png filter=lfs diff=lfs merge=lfs -text
|
| 37 |
+
OCR_INDRA/indra/assets/benchmarks/throughput.png filter=lfs diff=lfs merge=lfs -text
|
| 38 |
+
OCR_INDRA/indra/assets/examples/chinese_essay.png filter=lfs diff=lfs merge=lfs -text
|
| 39 |
+
OCR_INDRA/indra/assets/examples/multi_column.png filter=lfs diff=lfs merge=lfs -text
|
| 40 |
+
OCR_INDRA/indra/assets/examples/rotated_letter.png filter=lfs diff=lfs merge=lfs -text
|
| 41 |
+
OCR_INDRA/indra/assets/examples/table_pmc.png filter=lfs diff=lfs merge=lfs -text
|
| 42 |
+
OCR_INDRA/results/manifests_crops_20261023/train.jsonl filter=lfs diff=lfs merge=lfs -text
|
| 43 |
+
OCR_INDRA/tools/indra_eval/testdata/NotoSansDevanagari.ttf filter=lfs diff=lfs merge=lfs -text
|
| 44 |
+
ocr-tech/build/download-arxiv-limbuocr/draft.pdf filter=lfs diff=lfs merge=lfs -text
|
| 45 |
+
ocr-tech/outputs/newari-ranjana-render-conformance-audit-2026-07-16-r52/NithyaRanjanaNU-Regular.otf filter=lfs diff=lfs merge=lfs -text
|
| 46 |
+
ocr-tech/outputs/newari-ranjana-review-font-supplement-2026-07-15-r39/NithyaRanjanaNU-Regular.otf filter=lfs diff=lfs merge=lfs -text
|
| 47 |
+
ocr-tech/outputs/sikkim-herald-gurung-khema-two-edition-source-review-2026-07-15-r13/contact-sheets/gurung-khema-2022-03-01.png filter=lfs diff=lfs merge=lfs -text
|
| 48 |
+
ocr-tech/outputs/sikkim-herald-gurung-khema-two-edition-source-review-2026-07-15-r13/contact-sheets/gurung-khema-2022-04-22.png filter=lfs diff=lfs merge=lfs -text
|
| 49 |
+
ocr-tech/outputs/sikkim-herald-gurung-khema-two-edition-source-review-2026-07-15-r13/renders/gurung-khema-2022-03-01/page-001.png filter=lfs diff=lfs merge=lfs -text
|
| 50 |
+
ocr-tech/outputs/sikkim-herald-gurung-khema-two-edition-source-review-2026-07-15-r13/renders/gurung-khema-2022-03-01/page-002.png filter=lfs diff=lfs merge=lfs -text
|
| 51 |
+
ocr-tech/outputs/sikkim-herald-gurung-khema-two-edition-source-review-2026-07-15-r13/renders/gurung-khema-2022-03-01/page-003.png filter=lfs diff=lfs merge=lfs -text
|
| 52 |
+
ocr-tech/outputs/sikkim-herald-gurung-khema-two-edition-source-review-2026-07-15-r13/renders/gurung-khema-2022-03-01/page-004.png filter=lfs diff=lfs merge=lfs -text
|
| 53 |
+
ocr-tech/outputs/sikkim-herald-gurung-khema-two-edition-source-review-2026-07-15-r13/renders/gurung-khema-2022-04-22/page-001.png filter=lfs diff=lfs merge=lfs -text
|
| 54 |
+
ocr-tech/outputs/sikkim-herald-gurung-khema-two-edition-source-review-2026-07-15-r13/renders/gurung-khema-2022-04-22/page-002.png filter=lfs diff=lfs merge=lfs -text
|
| 55 |
+
ocr-tech/outputs/sikkim-herald-gurung-khema-two-edition-source-review-2026-07-15-r13/renders/gurung-khema-2022-04-22/page-003.png filter=lfs diff=lfs merge=lfs -text
|
| 56 |
+
ocr-tech/outputs/sikkim-herald-gurung-khema-two-edition-source-review-2026-07-15-r13/renders/gurung-khema-2022-04-22/page-004.png filter=lfs diff=lfs merge=lfs -text
|
| 57 |
+
ocr-tech/outputs/sikkim-herald-six-edition-source-review-2026-07-15-r11/contact-sheets/kirat-rai-2024-09-26.png filter=lfs diff=lfs merge=lfs -text
|
| 58 |
+
ocr-tech/outputs/sikkim-herald-six-edition-source-review-2026-07-15-r11/contact-sheets/kirat-rai-2025-01-13.png filter=lfs diff=lfs merge=lfs -text
|
| 59 |
+
ocr-tech/outputs/sikkim-herald-six-edition-source-review-2026-07-15-r11/contact-sheets/lepcha-2024-07-26.png filter=lfs diff=lfs merge=lfs -text
|
| 60 |
+
ocr-tech/outputs/sikkim-herald-six-edition-source-review-2026-07-15-r11/contact-sheets/lepcha-2024-10-04.png filter=lfs diff=lfs merge=lfs -text
|
| 61 |
+
ocr-tech/outputs/sikkim-herald-six-edition-source-review-2026-07-15-r11/contact-sheets/sunuwar-jenticha-2024-04-12.png filter=lfs diff=lfs merge=lfs -text
|
| 62 |
+
ocr-tech/outputs/sikkim-herald-six-edition-source-review-2026-07-15-r11/contact-sheets/sunuwar-jenticha-2025-01-26.png filter=lfs diff=lfs merge=lfs -text
|
| 63 |
+
ocr-tech/outputs/sikkim-herald-six-edition-source-review-2026-07-15-r11/renders/kirat-rai-2024-09-26/page-001.png filter=lfs diff=lfs merge=lfs -text
|
| 64 |
+
ocr-tech/outputs/sikkim-herald-six-edition-source-review-2026-07-15-r11/renders/kirat-rai-2024-09-26/page-002.png filter=lfs diff=lfs merge=lfs -text
|
| 65 |
+
ocr-tech/outputs/sikkim-herald-six-edition-source-review-2026-07-15-r11/renders/kirat-rai-2024-09-26/page-003.png filter=lfs diff=lfs merge=lfs -text
|
| 66 |
+
ocr-tech/outputs/sikkim-herald-six-edition-source-review-2026-07-15-r11/renders/kirat-rai-2024-09-26/page-004.png filter=lfs diff=lfs merge=lfs -text
|
| 67 |
+
ocr-tech/outputs/sikkim-herald-six-edition-source-review-2026-07-15-r11/renders/kirat-rai-2025-01-13/page-001.png filter=lfs diff=lfs merge=lfs -text
|
| 68 |
+
ocr-tech/outputs/sikkim-herald-six-edition-source-review-2026-07-15-r11/renders/kirat-rai-2025-01-13/page-002.png filter=lfs diff=lfs merge=lfs -text
|
| 69 |
+
ocr-tech/outputs/sikkim-herald-six-edition-source-review-2026-07-15-r11/renders/kirat-rai-2025-01-13/page-003.png filter=lfs diff=lfs merge=lfs -text
|
| 70 |
+
ocr-tech/outputs/sikkim-herald-six-edition-source-review-2026-07-15-r11/renders/kirat-rai-2025-01-13/page-004.png filter=lfs diff=lfs merge=lfs -text
|
| 71 |
+
ocr-tech/outputs/sikkim-herald-six-edition-source-review-2026-07-15-r11/renders/lepcha-2024-07-26/page-001.png filter=lfs diff=lfs merge=lfs -text
|
| 72 |
+
ocr-tech/outputs/sikkim-herald-six-edition-source-review-2026-07-15-r11/renders/lepcha-2024-07-26/page-002.png filter=lfs diff=lfs merge=lfs -text
|
| 73 |
+
ocr-tech/outputs/sikkim-herald-six-edition-source-review-2026-07-15-r11/renders/lepcha-2024-07-26/page-003.png filter=lfs diff=lfs merge=lfs -text
|
| 74 |
+
ocr-tech/outputs/sikkim-herald-six-edition-source-review-2026-07-15-r11/renders/lepcha-2024-07-26/page-004.png filter=lfs diff=lfs merge=lfs -text
|
| 75 |
+
ocr-tech/outputs/sikkim-herald-six-edition-source-review-2026-07-15-r11/renders/lepcha-2024-10-04/page-001.png filter=lfs diff=lfs merge=lfs -text
|
| 76 |
+
ocr-tech/outputs/sikkim-herald-six-edition-source-review-2026-07-15-r11/renders/lepcha-2024-10-04/page-002.png filter=lfs diff=lfs merge=lfs -text
|
| 77 |
+
ocr-tech/outputs/sikkim-herald-six-edition-source-review-2026-07-15-r11/renders/lepcha-2024-10-04/page-003.png filter=lfs diff=lfs merge=lfs -text
|
| 78 |
+
ocr-tech/outputs/sikkim-herald-six-edition-source-review-2026-07-15-r11/renders/lepcha-2024-10-04/page-004.png filter=lfs diff=lfs merge=lfs -text
|
| 79 |
+
ocr-tech/outputs/sikkim-herald-six-edition-source-review-2026-07-15-r11/renders/sunuwar-jenticha-2024-04-12/page-001.png filter=lfs diff=lfs merge=lfs -text
|
| 80 |
+
ocr-tech/outputs/sikkim-herald-six-edition-source-review-2026-07-15-r11/renders/sunuwar-jenticha-2024-04-12/page-002.png filter=lfs diff=lfs merge=lfs -text
|
| 81 |
+
ocr-tech/outputs/sikkim-herald-six-edition-source-review-2026-07-15-r11/renders/sunuwar-jenticha-2024-04-12/page-003.png filter=lfs diff=lfs merge=lfs -text
|
| 82 |
+
ocr-tech/outputs/sikkim-herald-six-edition-source-review-2026-07-15-r11/renders/sunuwar-jenticha-2024-04-12/page-004.png filter=lfs diff=lfs merge=lfs -text
|
| 83 |
+
ocr-tech/outputs/sikkim-herald-six-edition-source-review-2026-07-15-r11/renders/sunuwar-jenticha-2025-01-26/page-001.png filter=lfs diff=lfs merge=lfs -text
|
| 84 |
+
ocr-tech/outputs/sikkim-herald-six-edition-source-review-2026-07-15-r11/renders/sunuwar-jenticha-2025-01-26/page-002.png filter=lfs diff=lfs merge=lfs -text
|
| 85 |
+
ocr-tech/outputs/sikkim-herald-six-edition-source-review-2026-07-15-r11/renders/sunuwar-jenticha-2025-01-26/page-003.png filter=lfs diff=lfs merge=lfs -text
|
| 86 |
+
ocr-tech/outputs/sikkim-herald-six-edition-source-review-2026-07-15-r11/renders/sunuwar-jenticha-2025-01-26/page-004.png filter=lfs diff=lfs merge=lfs -text
|
| 87 |
+
ocr-tech/paper/draft.pdf filter=lfs diff=lfs merge=lfs -text
|
| 88 |
+
ocr-tech/paper/figures/language-pack-status.png filter=lfs diff=lfs merge=lfs -text
|
| 89 |
+
ocr-tech/paper/figures/limbu-linearization-diagnostic.png filter=lfs diff=lfs merge=lfs -text
|
| 90 |
+
ocr-tech/paper/figures/limbu-page-results.png filter=lfs diff=lfs merge=lfs -text
|
| 91 |
+
ocr-tech/paper/figures/limbu-recognizer-results.png filter=lfs diff=lfs merge=lfs -text
|
| 92 |
+
ocr-tech/paper/figures/limbu-source-evidence.png filter=lfs diff=lfs merge=lfs -text
|
| 93 |
+
ocr-tech/paper/figures/research-program.png filter=lfs diff=lfs merge=lfs -text
|
| 94 |
+
ocr-tech/paper/figures/system-architecture.png filter=lfs diff=lfs merge=lfs -text
|
OCR_INDRA/.gitignore
ADDED
|
@@ -0,0 +1,5 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
__pycache__/
|
| 2 |
+
*.pyc
|
| 3 |
+
.deps/
|
| 4 |
+
runs/
|
| 5 |
+
*.bundle
|
OCR_INDRA/INDRA_OCR_NEPALI_COMPLETE_REPORT.docx
ADDED
|
Binary file (70.3 kB). View file
|
|
|
OCR_INDRA/INDRA_OCR_NEPALI_FEASIBILITY.md
ADDED
|
@@ -0,0 +1,340 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
# Indra OCR × Nepali — Source-Level Feasibility Report
|
| 2 |
+
|
| 3 |
+
Scope: whether Datalab's **Indra OCR** (`datalab-to/indra`, v0.3.2, models HF tag `v3.1.1`) can be adapted
|
| 4 |
+
(fine-tuned) to improve our existing Nepali OCR system, and the lowest-risk path to evaluate it.
|
| 5 |
+
|
| 6 |
+
Verdict (short): **Investigation B outcome = B — Feasible with moderate engineering.**
|
| 7 |
+
The Devanagari recognizer's *complete* float weights are published (`model.safetensors`, 1,236,366 params,
|
| 8 |
+
verified complete against the ONNX class count), the Devanagari vocabulary already covers Nepali 100 %,
|
| 9 |
+
and the exact training-time crop/CenterMask preprocessing is shipped in the repo — but **no training code,
|
| 10 |
+
losses, optimizers, data loaders, or export scripts are published**, and the architecture's op-level details
|
| 11 |
+
(strides, activations, block order, custom decode export) must be recovered from the ONNX graph.
|
| 12 |
+
Recommended first step is a zero-training benchmark of the released model on our Nepali corpus.
|
| 13 |
+
|
| 14 |
+
Evidence tags: **[V]** verified from source/artifact, **[I]** reasonable inference, **[U]** unknown, needs a check.
|
| 15 |
+
|
| 16 |
+
---
|
| 17 |
+
|
| 18 |
+
## Investigation A — Reconstructed architecture
|
| 19 |
+
|
| 20 |
+
### A.0 Component inventory (what exists, where)
|
| 21 |
+
|
| 22 |
+
| Component | Model file | Source (file:line) | Type |
|
| 23 |
+
|---|---|---|---|
|
| 24 |
+
| Page rasterization (PDF) | — | `indra_ocr/pipeline.py:589` `load_pages` (pypdfium2, default 200 dpi, grayscale); `indra_ocr/document.py:136` `page_scale` | deterministic |
|
| 25 |
+
| Text-layer extraction + trust checks | — | `indra_ocr/textlayer.py:200` `read_text_layer`, `:80` `extract_runs`, `:171` `invisible_text_frac`, `:186` `no_ink_frac` | deterministic |
|
| 26 |
+
| Orientation (0/90/180/270) | `orient.onnx` | `pipeline.py:431` `orientation`, `:114` `orient_input` (768 px thumbnail, 4-way argmax, threshold 0.95 at `:54`) | CNN classifier, 4 classes |
|
| 27 |
+
| Deskew (±5°) | — | `pipeline.py:66` `estimate_skew` (ink-profile sharpness search, no ML) | deterministic |
|
| 28 |
+
| Line detection | `det.onnx` | `pipeline.py:240` `detect`; `indra_ocr/det_post.py:25` `page_to_input`, `:105` `decode`; adaptive 2nd pass `pipeline.py:242-256` | CNN "LineNet", 3-channel output (core logit + top/bottom distances) |
|
| 29 |
+
| Line masking ("CenterMask") | `mask.onnx` (optional) | `indra_ocr/line_mask.py:60` `mask_windows`, `:110` `crop_lines` | small segmentation CNN |
|
| 30 |
+
| Script routing (hanzi) | `script.onnx` (optional) | `pipeline.py:332` `hanzi_lines` (>0.5 → CJK recognizer) | line classifier |
|
| 31 |
+
| **Line recognition (main)** | **`rec.onnx`** | `pipeline.py:264` `recognize` | **CNN + CTC, 1,236,366 params** |
|
| 32 |
+
| Line recognition (CJK) | `rec_cjk.onnx` (optional) | `pipeline.py:290-320` | CNN + CTC, own vocab (9,967 tokens **[V]**, from `meta.json`) |
|
| 33 |
+
| Math recognition (opt-in) | `mathcls/math_enc/math_dec_init/math_dec_step.onnx` | `indra_ocr/math.py:52` `MathReader`, `:89` `read` (greedy, KV-cached attention decoder) | classifier + encoder-decoder (autoregressive) |
|
| 34 |
+
| Reading order + layout heads | `order.onnx` | `pipeline.py:409` `reading_order`; `indra_ocr/order_post.py:18` `line_features`, `:62` `decode_order`, `:273` `decode_blocks_order`, `:220` `_cluster` | pairwise-logit "OrderNet"; optional heads: block class (10), same-block, same-row/col/cell |
|
| 35 |
+
| Markdown/document assembly | — | `indra_ocr/layout_post.py:332` `page_markdown`, `:304` `page_blocks` (headings `##`, lists, HTML tables) | deterministic + geometry |
|
| 36 |
+
| Page/document orchestration, parallelism | — | `indra_ocr/document.py:117` `Converter`, `:248` `convert`; `indra_ocr/parallel.py:99` `convert_many` (1 process/core, 1 page/task) | — |
|
| 37 |
+
|
| 38 |
+
No separate layout detector or region model exists. Line detection is the only detection stage; "layout"
|
| 39 |
+
is the order model's classification heads plus geometry. **[V]**
|
| 40 |
+
|
| 41 |
+
### A.1 Recognizer architecture (from the published float weights)
|
| 42 |
+
|
| 43 |
+
Verified from `model.safetensors` header (HF `v3.1.1`, 107 tensors, all F32) — **[V]**:
|
| 44 |
+
|
| 45 |
+
```
|
| 46 |
+
Input [B, 1, 32, W] float32 ink map (0 = paper, 1 = ink), W multiple of 8
|
| 47 |
+
│
|
| 48 |
+
├─ rec.stem.weight [128,1,32,4] + rec.stem_bn.* → Conv2d(1→128, k=32×4) + BatchNorm(128)
|
| 49 |
+
│
|
| 50 |
+
├─ 10 × residual block rec.blocks.{0..9}.*
|
| 51 |
+
│ rec.blocks.N.expand.weight [256,128,1] 1×1 conv 128→256
|
| 52 |
+
│ rec.blocks.N.project.weight [128,256,1] 1×1 conv 256→128
|
| 53 |
+
│ rec.blocks.N.se.fc1/fc2 [64,256,1]/[256,64,1] squeeze-excitation on 256 ch
|
| 54 |
+
│ onnx::Conv_NNN.* [128,1,9] depthwise 1×9 conv (128 groups)
|
| 55 |
+
│ (exact intra-block order, strides, activations, pooling: read from rec.onnx graph) [U]
|
| 56 |
+
│
|
| 57 |
+
└─ rec.head.weight [1678,128,1] + bias [1678] 1×1 conv → 1678 CTC classes
|
| 58 |
+
```
|
| 59 |
+
|
| 60 |
+
* Parameter accounting sums **exactly** to 1,236,366 = the Hub-reported count
|
| 61 |
+
(stem 16,512 + BN 512 + 10 blocks × 100,288 + head 216,462) — every parameter of the recognizer is present. **[V]**
|
| 62 |
+
* Class count 1,678 = 1 CTC blank + 1,677 tokens, which **exactly equals** `Vocab.multi()`
|
| 63 |
+
(`vocab.py:138-156`, incl. Devanagari U+0900–U+097F at line 147). The shipped `meta.json` vocabulary
|
| 64 |
+
string parsed to the same 1,677-token list (byte-for-byte equal). **[V]** (The HF `config.json` calling
|
| 65 |
+
`model.safetensors` the "Latin-script (English) recognizer" is loose wording; the head size and the
|
| 66 |
+
README's "OCR reads … Devanagari, Bengali" settle it.)
|
| 67 |
+
* The 10 depthwise convs have ONNX-derived names (`onnx::Conv_369…396`) while block tensors keep original
|
| 68 |
+
PyTorch FQNs with a `rec.` module prefix — the safetensors was assembled from the ONNX initializer set,
|
| 69 |
+
and 10 weights were not direct `nn.Parameter`s. Mapping them is by graph order (see B.3). **[I]**
|
| 70 |
+
* No RNN/attention weights exist in `rec.onnx`: it is a **pure convolutional CTC recognizer**. **[V]**
|
| 71 |
+
* Width reduction: `WIDTH_MULTIPLE = 8` is documented as the "timestep stride"
|
| 72 |
+
(`rec_preproc.py:24`) → overall width downsampling 8×. **[I]**, confirmable from conv strides in the graph.
|
| 73 |
+
|
| 74 |
+
### A.2 Preprocessing (recognition path)
|
| 75 |
+
|
| 76 |
+
* Crop rule is a **shared training/inference** function, `rec_preproc.py:27` `crop_line`: expand box by
|
| 77 |
+
`PAD_Y=0.2` × its height vertically and `PAD_X=0.3` × height horizontally (`:22-23`), crop with white
|
| 78 |
+
outside the page, resize to `LINE_HEIGHT=32` (`:21`, = `meta["rec_height"]=32` **[V]**), width capped
|
| 79 |
+
4096, grayscale→ink `1 − gray/255`, right-pad widths to a multiple of 8 (`pad_batch` `:56`). Single
|
| 80 |
+
channel, NCHW, no mean/std normalization beyond the ink inversion. **[V]**
|
| 81 |
+
* If `mask.onnx` is present, a 96 px-high window around each line is masked first
|
| 82 |
+
(`line_mask.py:15,20,60,92`): pixels below 0.5 center-line probability become paper level (90th
|
| 83 |
+
percentile gray). Docstring: "the recognizer must only ever see crops that went through the same
|
| 84 |
+
exported mask model" — **same step as in training**. **[V]**
|
| 85 |
+
* Batching: lines grouped by similar width (`width_batches`, `rec_preproc.py:69`; max 64, ratio 1.3,
|
| 86 |
+
area cap) and right-padded (`pad_batch`). Inference-only tuning. **[V]**
|
| 87 |
+
* Detector input: page resized so long side = `meta["det_long_side"] = 1024` **[V]**, ink map, padded to a
|
| 88 |
+
multiple of 32 (`det_post.py:25`); adaptive second pass up to 2048 px if median line height < 9 px
|
| 89 |
+
(`pipeline.py:37-48,242-256`). Output at ½ resolution with 3 channels (`det_post.py:16,105`). **[V]**
|
| 90 |
+
* Orientation input: 768 px square ink thumbnail (`pipeline.py:49,114`). **[V]**
|
| 91 |
+
|
| 92 |
+
### A.3 Decoding, confidence, outputs
|
| 93 |
+
|
| 94 |
+
* `rec.onnx` I/O contract: inputs `x` [B,1,32,W] and `widths` [B]; outputs `best` (argmax path per
|
| 95 |
+
timestep), `pmax` (peak probability per timestep), `lengths` (valid timesteps) — `pipeline.py:294`. **[V]**
|
| 96 |
+
* Decoding: standard **greedy CTC** collapse, `pipeline.py:299`; text via `Vocab.decode`
|
| 97 |
+
(`vocab.py:222`); blank = 0. No beam search. **[V]**
|
| 98 |
+
* Confidence: **mean peak probability of kept tokens** (`pipeline.py:315`); a line under
|
| 99 |
+
`MIN_CONF = 0.5` gets its box but **no text** (`pipeline.py:60,316-320`). Math path uses its own 0.9
|
| 100 |
+
mean rule (`math.py:26`). **[V]**
|
| 101 |
+
* Per-token x positions from timestep indices, mapped back through crop scale (`pipeline.py:300-307`);
|
| 102 |
+
line boxes are the detector's ink boxes scaled to page px (`pipeline.py:257-261`), split further at
|
| 103 |
+
wide character gaps (`SPLIT_GAP=3.0`, cell `1.8`, gutter `1.6` line heights —
|
| 104 |
+
`pipeline.py:44-47`, `split_line :373`, `gutter_cuts :342`). **[V]**
|
| 105 |
+
* Text normalization: NFC + look-alike canonicalization (`vocab.py:22-45` `canonical`); CJK lines join
|
| 106 |
+
without spaces, hyphen line-breaks merged (`layout_post.py:83` `join_lines`). **[V]**
|
| 107 |
+
* Unknown characters: **no UNK class**. Training labels with a token outside the vocab are *dropped*
|
| 108 |
+
(`vocab.py:6,212-217` `encode`); inference can only ever emit the fixed 1,677 tokens. Devanagari
|
| 109 |
+
syllable-structure checks (`vocab.py:260,283`) flag broken font maps; `document.py:40` `readable` gates
|
| 110 |
+
the text-layer fallback. **[V]**
|
| 111 |
+
|
| 112 |
+
### A.4 Digital-PDF text extraction
|
| 113 |
+
|
| 114 |
+
Yes — it bypasses OCR for pages whose layer passes checks (`document.py:143` `pdf_page` →
|
| 115 |
+
`textlayer.py:200`): reject if >5 % invisible text (an OCR layer), >0.5 % junk codepoints (broken font
|
| 116 |
+
map), or >3 % text runs without ink. A trusted layer is used directly: pdfium characters are *assigned to
|
| 117 |
+
detected line boxes* (`document.py:65` `assign_runs`), recognizer runs only on empty boxes
|
| 118 |
+
("hybrid"), and PDF content order is kept when coherent (`textlayer.py:239`). Text mode is script-agnostic
|
| 119 |
+
— it already returns Nepali text from any sound digital PDF with **zero ML work**. **[V]**
|
| 120 |
+
|
| 121 |
+
### A.5 Reading order & document structure
|
| 122 |
+
|
| 123 |
+
`order.onnx` scores every line pair ("i before j") plus optional heads (block class over 10 labels,
|
| 124 |
+
same-block, same-row/column/cell). Blocks come from average-linkage clustering; in-block order by XY-cut
|
| 125 |
+
(tables/forms row-major); blocks ordered by mean pairwise logit (`order_post.py:273`); markdown built in
|
| 126 |
+
`layout_post.py:332`. Fallback without the model: pure-geometry XY-cut (`order_post.py:113`). **[V]**
|
| 127 |
+
|
| 128 |
+
### A.6 Compact flow diagram
|
| 129 |
+
|
| 130 |
+
```
|
| 131 |
+
PDF ──pypdfium2 @dpi──┐ ┌── digital text layer: pdfium chars →
|
| 132 |
+
image file ──cv2──► grayscale page ──┤ │ trust checks → assign to line boxes
|
| 133 |
+
▼ │ (NO recognizer; OCR only empty boxes)
|
| 134 |
+
orient.onnx (4-way, p>0.95) → deskew (geom) [document.py]
|
| 135 |
+
▼
|
| 136 |
+
det.onnx (long side 1024 → ink map, ÷32 pad) → [N,5] line boxes [det_post.py]
|
| 137 |
+
▼
|
| 138 |
+
mask.onnx CenterMask (opt) → crop_line: pad .2h/.3h → h=32 → ink [line_mask.py, rec_preproc.py]
|
| 139 |
+
▼
|
| 140 |
+
script.onnx? ─ hanzi ──► rec_cjk.onnx (CTC, 9,968 classes)
|
| 141 |
+
└── text ────► rec.onnx (CNN+SE, 1,678 CTC classes) [pipeline.py:264]
|
| 142 |
+
▼
|
| 143 |
+
greedy CTC collapse → per-line text + mean-peak confidence (MIN 0.5)
|
| 144 |
+
▼
|
| 145 |
+
split at gaps/gutters → order.onnx (pairwise + layout heads) or XY-cut → page_markdown
|
| 146 |
+
▼ [order_post.py, layout_post.py]
|
| 147 |
+
Page(lines, blocks, markdown, rotation, skew, timings) → CLI / convert / convert_many
|
| 148 |
+
```
|
| 149 |
+
|
| 150 |
+
Recognizer detail (verified from weights): `[B,1,32,W] → Conv(1→128, k32×4)+BN → 10×[dw 1×9 + SE(256→64→256)
|
| 151 |
+
+ expand 1×1 128→256 + project 1×1 256→128] → 1×1 head → 1678 classes`.
|
| 152 |
+
|
| 153 |
+
---
|
| 154 |
+
|
| 155 |
+
## Investigation B — Fine-tuning feasibility (highest priority)
|
| 156 |
+
|
| 157 |
+
### B.0 What was searched
|
| 158 |
+
|
| 159 |
+
Training scripts, configs, model definitions, checkpoints/checkpoint downloads, losses, optimizers, LR
|
| 160 |
+
schedules, checkpoint-saving logic, data loaders/labels/augmentation/batching/eval, vocab creation,
|
| 161 |
+
export/conversion scripts, training tests — across: all source files (3,850 lines), both tests, `bench/`,
|
| 162 |
+
`.github/workflows/`, all git history/branches, and the official HF model repo + its file list, config.json,
|
| 163 |
+
meta.json, license, and the GitHub org's public repos.
|
| 164 |
+
|
| 165 |
+
**Result: no training code of any kind is published. None ever was (git history: no deleted files).** **[V]**
|
| 166 |
+
|
| 167 |
+
The repo *references* an unpublished training codebase by name — strong evidence a closed-source training
|
| 168 |
+
repo exists and shares these exact torch-free modules:
|
| 169 |
+
|
| 170 |
+
* `det_post.py:1-4`: "…the PyTorch model in **det_model.py**, which re-exports them" — file absent.
|
| 171 |
+
* `order_post.py:1-5`: "**order_model.py** (the PyTorch OrderNet) re-exports all of these" — file absent.
|
| 172 |
+
* `rec_preproc.py:1-13`: "Line crop rule **shared by recognition training and inference**… Training and
|
| 173 |
+
inference must both call `crop_line`."
|
| 174 |
+
* `line_mask.py:1-9`: "One function for training and inference… same exported mask model (mask.onnx)".
|
| 175 |
+
* `vocab.py:6`: "A **training line** with a token outside the vocabulary is dropped".
|
| 176 |
+
* `textlayer.py:1-15`: the trust checks "follow the rules that **select the recognizer's own training labels**".
|
| 177 |
+
* `order_post.py:194` `label_order`, `:146/159` `kendall_tau`/`successor_acc`, `:100/113` baselines —
|
| 178 |
+
training/eval utilities with no inference consumer.
|
| 179 |
+
* `det_post.py:48` `encode_targets` — training-target builder, unused at inference.
|
| 180 |
+
* `line_mask.py:123` `session_for` docstring: "One CPU session per process (**dataloader workers** call this lazily)".
|
| 181 |
+
|
| 182 |
+
### B.1 Answers to the explicit questions
|
| 183 |
+
|
| 184 |
+
1. **Supported training/fine-tuning pipeline present?** **No.** Inference-only package (`onnxruntime`,
|
| 185 |
+
no torch). No training entry points, losses, optimizers, data loaders, configs, or export scripts.
|
| 186 |
+
CI (`.github/workflows/publish.yml`) only builds/tests/publishes the PyPI package.
|
| 187 |
+
|
| 188 |
+
2. **Pretrained trainable weights available, or only ONNX?** For the **main recognizer: both**. HF
|
| 189 |
+
`model.safetensors` (revision `v3.1.1`, 4,955,608 bytes) is the **float32 weight set of `rec.onnx`**
|
| 190 |
+
(HF `config.json` states this explicitly). Verified complete: 107 tensors; bytes = 4,945,464 (=
|
| 191 |
+
1,236,366 × 4) + 10,144 header/offsets exactly; names preserve the original module tree
|
| 192 |
+
(`rec.stem.*`, `rec.blocks.0..9.*`, `rec.head.*`). For *all other components* (`det/order/orient/
|
| 193 |
+
script/rec_cjk/math*`): **ONNX only** — no trainable-format copies, no optimizer states anywhere.
|
| 194 |
+
|
| 195 |
+
3. **Can ONNX be mapped back to trainable parameters?** Not needed for `rec` — the mapping already
|
| 196 |
+
exists in reverse (safetensors → ONNX). For the other models it is *technically* possible (ONNX
|
| 197 |
+
initializers are readable) but the module definitions are absent. Specific reconstruction hazards for
|
| 198 |
+
the recognizer, to be handled deliberately:
|
| 199 |
+
* 10 depthwise convs carry `onnx::Conv_369…396` names — they were derived weights (no direct PyTorch
|
| 200 |
+
parameter), so association is by **graph topological order**, not by name; requires reading
|
| 201 |
+
`rec.onnx` node order. **[I]**
|
| 202 |
+
* Strides, padding, activations, SE pooling type, residual ordering, block internals: **not encodable
|
| 203 |
+
in a state dict** → must be read from ONNX node attributes (5-minute task with `onnx`, once the
|
| 204 |
+
model folder is downloaded for evaluation). **[U]**
|
| 205 |
+
* BN: the safetensors keeps `stem_bn` running stats separately, so the published weights are the
|
| 206 |
+
pre-folding PyTorch values — good for direct `load_state_dict`, but the ONNX graph may still fold
|
| 207 |
+
BN; parity tests must compare post-softmax greedy decode, not raw intermediate tensors. **[I]**
|
| 208 |
+
* The exported graph's outputs (`best`, `pmax`, `lengths`) are **post-processing baked into the ONNX
|
| 209 |
+
graph**, not raw logits. Training needs raw logits from a reconstructed module; the re-export must
|
| 210 |
+
recreate the same contract (or the 20-line `recognize()` consumer must be adapted). **[V]**
|
| 211 |
+
* Width stride 8 is documented but not weight-inferable (`rec_preproc.py:24`) — confirm from conv strides. **[I]**
|
| 212 |
+
|
| 213 |
+
4. **Original architecture in a trainable framework?** No code, but the architecture is fully
|
| 214 |
+
**reconstructible**: a 1.24M-param CNN (stem + 10 SE residual blocks + 1×1 CTC head), all parameters
|
| 215 |
+
present, no recurrence/attention, and the exact input preprocessing function is shipped. Effort: a
|
| 216 |
+
one-file PyTorch implementation + weight-load + numeric-parity test. **[V/I]**
|
| 217 |
+
|
| 218 |
+
5. **Vocab represent all Nepali characters?** **Yes — verified against the shipped artifact.** HF
|
| 219 |
+
`meta.json` (v3.1.1) `vocab` == `Vocab.multi()` (`vocab.py:138-156`): 1,677 tokens, 1,678 classes
|
| 220 |
+
(1,236,366-param head matches). All 128 assigned Devanagari codepoints (U+0900–U+097F) are present —
|
| 221 |
+
letters, vowel signs, virama, nukta, anusvara/candrabindu/visarga, danda `।`, double danda `॥`, digits
|
| 222 |
+
`०-९`, plus `₹`. Nepali is entirely within that block → **no vocabulary change and no output-layer
|
| 223 |
+
surgery needed**. (HF language tags list `hi`, `bn`, `ru`… but not `ne`; the script coverage is what
|
| 224 |
+
matters and it is byte-verified.) **[V]**
|
| 225 |
+
|
| 226 |
+
6. **If the vocabulary must change, what happens to the head/weights?** The head is a 1×1 conv
|
| 227 |
+
`[1678,128,1]+[1678]`. Extending/replacing classes = resizing that one tensor (append rows →
|
| 228 |
+
random init, blank id 0 must stay 0; replacing a token's row loses its pretrained row); the other 105
|
| 229 |
+
tensors are untouched, `vocab.py` builders and `meta.json` regenerated. Cost is trivial — but
|
| 230 |
+
**unnecessary for Nepali**. **[V/I]**
|
| 231 |
+
|
| 232 |
+
7. **Can training reproduce published preprocessing/decoding?** **Largely yes, by construction**: the
|
| 233 |
+
crop rule (`crop_line`), CenterMask step, height 32, pad rules, greedy CTC decode, confidence
|
| 234 |
+
definition and NFC/label canonicalization are all in the repo and documented as shared with training.
|
| 235 |
+
Gaps: the exact data mixture, augmentation policy, optimizer/schedule, and label-selection recipe
|
| 236 |
+
(partially described at `textlayer.py:1-15`) are **not published**. **[V/U]**
|
| 237 |
+
|
| 238 |
+
8. **Engineering work if no pipeline exists** (bounded plan):
|
| 239 |
+
1. Reconstruct `RecModel` (PyTorch) from safetensors + ONNX graph attributes; weight-load assert.
|
| 240 |
+
2. Parity harness: same crops through reconstructed torch module vs `rec.onnx` — require identical
|
| 241 |
+
greedy decode and logits within fp tolerance on a few hundred lines.
|
| 242 |
+
3. CTC fine-tune loop: AdamW + cosine (standard recipe), labels via `vocab.encode`
|
| 243 |
+
(OOV lines dropped — automatically excludes any char Nepali needs that's missing, which is none),
|
| 244 |
+
`rec_preproc.crop_line` (+ CenterMask session) for inputs, light augmentation.
|
| 245 |
+
4. Re-export ONNX with the same `{x, widths} → {best, pmax, lengths}` contract (or run torch in-house).
|
| 246 |
+
5. Eval: held-out Nepali scans — line CER + full-page; A/B against our current system.
|
| 247 |
+
|
| 248 |
+
### B.2 Classification: **B — Feasible with moderate engineering**
|
| 249 |
+
|
| 250 |
+
* Not **A**: no training code/configs shipped; no checkpoint/optimizer state; the pipeline must be built.
|
| 251 |
+
* Not **C**: for the component we need to adapt, the *compatible pretrained trainable weights exist and
|
| 252 |
+
are complete* (verified against the model's own class count and parameter count), and the architecture
|
| 253 |
+
is a small reconstructible CNN — not "arch or weights missing".
|
| 254 |
+
* Not **D**: no technical blocker; the license **explicitly permits fine-tuning/training/derivatives**
|
| 255 |
+
(see B.4); no gated access (repo is public, not gated).
|
| 256 |
+
* The "moderate" qualifier: op-level architecture recovery from the ONNX graph, parity testing, and
|
| 257 |
+
building the training/eval harness around closed-source training recipes.
|
| 258 |
+
|
| 259 |
+
### B.3 Accuracy risk (the real decision driver)
|
| 260 |
+
|
| 261 |
+
The model is **speed-optimized**: 1.2M params, CPU-first, ~230 ms/page. Its own README reports text
|
| 262 |
+
accuracy on OmniParseBench **behind PaddleOCR's full pipeline (41.3 vs 55.6)** and ~20 % on old-scan
|
| 263 |
+
tests, with handwriting unsupported. Nepali fine-tuning can shift the recognizer toward our domain, but
|
| 264 |
+
whether a 1.2M-param CNN + our fine-tuning dataset beats our existing Nepali system is **empirical and
|
| 265 |
+
not answerable from source**. **[I]** This is why evaluation must start with a zero-training benchmark.
|
| 266 |
+
|
| 267 |
+
### B.4 Licensing (model weights ≠ code — treated separately)
|
| 268 |
+
|
| 269 |
+
* Repo **code**: Apache-2.0 (`indra/LICENSE`).
|
| 270 |
+
* **Model weights**: modified OpenRAIL-M (`HF LICENSE`, verified text). Key terms:
|
| 271 |
+
* Use, **fine-tuning, training, evaluation, re-parametrizing are explicitly permitted** (§5).
|
| 272 |
+
* Derivatives must carry the same license (share-alike §8) + attribution + state changes (§7); use
|
| 273 |
+
restrictions must be propagated (§4a).
|
| 274 |
+
* Commercial limits (Attachment A §2): no use if >$20M prior-year revenue or >$20M raised funding
|
| 275 |
+
(except personal/research use), and **no competing product/service vs Datalab's offerings**.
|
| 276 |
+
* `model.safetensors` is published "for the Hub's parameter count; the package does not use it"
|
| 277 |
+
(HF `config.json`) — usable, but not a supported training artifact (pin HF revision `v3.1.1`).
|
| 278 |
+
|
| 279 |
+
### B.5 Lowest-risk, evidence-backed evaluation path
|
| 280 |
+
|
| 281 |
+
**Phase 0 — zero-training benchmark (~1–2 days; no code changes to our system).** This is the decisive,
|
| 282 |
+
lowest-risk step and it de-risks everything downstream:
|
| 283 |
+
1. Isolated env: `pip install indra-ocr` (downloads ~18 MB of models on first run — needs your approval,
|
| 284 |
+
since nothing is cached locally).
|
| 285 |
+
2. Run our Nepali eval corpus (scans/images) through the released model: `indra-ocr --mode ocr` and
|
| 286 |
+
default `auto`; parse `--format json` for per-line text + confidence.
|
| 287 |
+
3. **Recognizer-only isolation**: feed *our* line boxes to `OrtOCR.recognize(gray, boxes)` (public API)
|
| 288 |
+
to separate recognizer quality from detector quality.
|
| 289 |
+
4. Born-digital Nepali PDFs: test `mode=auto` — the text-layer path is script-agnostic and needs no ML;
|
| 290 |
+
if our sources include digital PDFs this may be an immediate standalone win.
|
| 291 |
+
5. **Decision gate** with our test set: line-level CER (NFC-normalized, using our labels) vs current
|
| 292 |
+
system, plus per-confidence-bucket breakdown. Continue only if Indra's recognizer is competitive or
|
| 293 |
+
the fine-tune headroom is credible.
|
| 294 |
+
|
| 295 |
+
**Phase 1 — fine-tune (2–6 weeks, only if the gate passes).** Exactly the B.1/8 plan: reconstruct
|
| 296 |
+
`RecModel` → parity test vs `rec.onnx` → CTC fine-tune on Nepali line crops (no vocab/head changes) →
|
| 297 |
+
re-export same ONNX contract → A/B vs current system on held-out pages. Guard rails: pin HF revision
|
| 298 |
+
`v3.1.1`; keep the reconstruction parity test in CI; treat the fine-tuned weights as an OpenRAIL
|
| 299 |
+
derivative.
|
| 300 |
+
|
| 301 |
+
**Fallback if reconstruction stalls (bounds the risk):** keep Indra's detector/order/markdown pipeline
|
| 302 |
+
and swap in *any* CTC recognizer we train ourselves, exported to the same `{x, widths} → {best, pmax,
|
| 303 |
+
lengths}` contract — the repo contains everything needed to run it as a drop-in.
|
| 304 |
+
|
| 305 |
+
**Phase 2 — productionize** only after Phase 1 wins on our data: license review (commercial terms,
|
| 306 |
+
share-alike, attribution), throughput check (they claim 7–49 pages/s on 4–32 cores; verifiable on our
|
| 307 |
+
hardware).
|
| 308 |
+
|
| 309 |
+
### B.6 Key unknowns to close during Phase 0/1
|
| 310 |
+
|
| 311 |
+
* Op-level details of `rec.onnx` (strides/activations/SE pooling/skip order) — read graph attributes
|
| 312 |
+
locally once the model folder exists. **[U]**
|
| 313 |
+
* Whether the 8× width stride and the folder's `mask.onnx` presence hold for the trained checkpoint. **[U]**
|
| 314 |
+
* Empirical: does fine-tuning beat our current Nepali system? **[U — benchmark-gated]**
|
| 315 |
+
* Whether Datalab intends `model.safetensors` as a stable artifact (it was added for Hub metadata). **[U]**
|
| 316 |
+
|
| 317 |
+
---
|
| 318 |
+
|
| 319 |
+
## Appendix — Primary evidence
|
| 320 |
+
|
| 321 |
+
Local repo (`indra/`, v0.3.2, commit `3d773ae`): source files and line references throughout this report;
|
| 322 |
+
git history (2 relevant commits + 3 remote-branch commits) contains **no** training files ever.
|
| 323 |
+
|
| 324 |
+
Official online sources (read-only; no weights downloaded except the safetensors *header* metadata via
|
| 325 |
+
HTTP range request, and the small JSON config files):
|
| 326 |
+
* `https://github.com/datalab-to/indra` — inference package (Apache-2.0); no training code.
|
| 327 |
+
* `https://api.github.com/orgs/datalab-to/repos` — 13 public repos; none is an Indra training repo.
|
| 328 |
+
* `https://huggingface.co/api/models/datalab-to/indra-ocr/tree/v3.1.1` — file list: `det.onnx`,
|
| 329 |
+
`rec.onnx`, `rec_cjk.onnx`, `order.onnx`, `orient.onnx`, `script.onnx`, `math*.onnx`, `meta.json`,
|
| 330 |
+
**`model.safetensors`** (4,955,608 B).
|
| 331 |
+
* `https://huggingface.co/api/models/datalab-to/indra-ocr` — safetensors param count 1,236,366 (F32);
|
| 332 |
+
license `openrail`.
|
| 333 |
+
* `https://huggingface.co/datalab-to/indra-ocr/resolve/v3.1.1/config.json` — "model.safetensors holds the
|
| 334 |
+
float weights of rec.onnx … for the Hub parameter count; the package does not use it."
|
| 335 |
+
* `https://huggingface.co/datalab-to/indra-ocr/resolve/v3.1.1/meta.json` — `det_long_side=1024`,
|
| 336 |
+
`rec_height=32`, `vocab` (1,677 tokens, == `Vocab.multi()`), `vocab_cjk` (9,967 tokens).
|
| 337 |
+
* `model.safetensors` header (first 16 KB) — 107 tensors; tensor names/shapes ⇒ architecture; parameter
|
| 338 |
+
sum equals 1,236,366 exactly.
|
| 339 |
+
* `https://huggingface.co/datalab-to/indra-ocr/resolve/v3.1.1/LICENSE` — modified OpenRAIL-M text.
|
| 340 |
+
* README (GitHub/HF) — capability claims, limitations (old scans, handwriting), benchmarks.
|
OCR_INDRA/README_BUNDLE.md
ADDED
|
@@ -0,0 +1,14 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
# OCR_INDRA experiment bundle
|
| 2 |
+
|
| 3 |
+
Remote run companion for `docs/REMOTE_RUN_SHEET.md` (start there).
|
| 4 |
+
Layout: indra/ = cloned indra-ocr v0.3.2 (commit 3d773ae); models/indra-ocr-v3.1.1/
|
| 5 |
+
= Hub artifacts @ v3.1.1 (OpenRAIL-M — fine-tuning explicitly licensed §II ¶36);
|
| 6 |
+
tools/indra_eval = verified inspector/eval harness; tools/exp = experiment code;
|
| 7 |
+
INDRA_OCR_NEPALI_COMPLETE_REPORT.docx = full investigation report (read-only).
|
| 8 |
+
|
| 9 |
+
Setup: `bash tools/exp/remote_setup.sh` (deps + probe + offline verification gate),
|
| 10 |
+
then follow the run sheet. `bash tools/exp/verify_bundle.sh` alone re-checks the
|
| 11 |
+
toolchain with no data: torch re-implementation → ONNX export → exactness vs the
|
| 12 |
+
shipped rec.onnx → rec-lines smoke CER. Offline-verified 2026-10-20 on torch
|
| 13 |
+
2.14.1+cpu / onnxruntime 1.31.0 (export uses the legacy torch exporter on purpose).
|
| 14 |
+
Never copy datasets off the remote; only results files leave.
|
OCR_INDRA/docs/INDRA_EXPERIMENTS.md
ADDED
|
@@ -0,0 +1,253 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
# Indra × Nepali — Experiment Plan (A–E)
|
| 2 |
+
|
| 3 |
+
Companion to `docs/INDRA_INVESTIGATION.md`. Everything runnable is designed
|
| 4 |
+
around `tools/indra_eval/eval_indra.py` (works on CPU; the remote T4 simply
|
| 5 |
+
makes batches fast). Remote-dependent sections are marked **[B]** and are
|
| 6 |
+
blocked until SSH access to `khalti-t4` (202.51.0.7) is restored; every
|
| 7 |
+
command to run there is written out so the set can be executed as-is.
|
| 8 |
+
|
| 9 |
+
Fairness principle: **recognizer comparisons use identical line crops**
|
| 10 |
+
(rec-lines mode); **pipeline comparisons use page-level text**. Both modes are
|
| 11 |
+
built and smoke-tested (`results/smoke/`: CER 3.49% rec-only / 2.62% full on a
|
| 12 |
+
synthetic page — indicational only).
|
| 13 |
+
|
| 14 |
+
---
|
| 15 |
+
|
| 16 |
+
## Experiment A — zero-shot Indra vs deployed PP-OCRv5 (do FIRST) [B with A.1 runnable]
|
| 17 |
+
|
| 18 |
+
**Goal:** close "does the ~1.24 M-param Indra recognizer beat the deployed
|
| 19 |
+
PP-OCRv5 Mobile Devanagari recognizer?" without training anything.
|
| 20 |
+
|
| 21 |
+
**Materials (all remote):** frozen eval sets `~/ocr/ocr-eval-20260924*/` +
|
| 22 |
+
`docs/DATA_LICENSING.md`, the deployed paddle pipeline
|
| 23 |
+
(`~/ocr/ocr-deva-ft`, `tools/deva_rec_torch`, `tools/eval_metrics.py`), the
|
| 24 |
+
CLONE of this repo + `models/indra-ocr-v3.1.1/` (copy `indra/` and `models/`
|
| 25 |
+
up: ~46 MB — no dataset leaves the machine).
|
| 26 |
+
|
| 27 |
+
**A.1 runnable from here already indep. of remote sets:**
|
| 28 |
+
- `make_synth_page.py` renders pages at **matching font metrics** for a sanity
|
| 29 |
+
envelope (no migration of datasets needed: it was built here, scored here).
|
| 30 |
+
|
| 31 |
+
**A.2 (remote) recognizer head-to-head on identical crops:**
|
| 32 |
+
1. Export the deployed pipeline's GT line boxes for an eval set once:
|
| 33 |
+
PP-OCRv5 det results on the set → boxes jsonl
|
| 34 |
+
(`{"file", lines: [{i, text: GT, box}]}`).
|
| 35 |
+
2. Run our adapter on the same crops:
|
| 36 |
+
```
|
| 37 |
+
PYTHONPATH=$HOME/ocr/OCR_INDRA/indra:$HOME/ocr/dep python3 \
|
| 38 |
+
tools/indra_eval/eval_indra.py predict <page.jpg> \
|
| 39 |
+
--boxes <gt_boxes.jsonl> --jsonl results/indra_rec/<set>.jsonl
|
| 40 |
+
```
|
| 41 |
+
and score with the identical-crop baseline (PP-OCRv5 rec on the same cropped
|
| 42 |
+
images saved as png), then `cer` both against GT.
|
| 43 |
+
3. Report: micro/macro CER + WER + per-line detail + conf histogram;(any
|
| 44 |
+
`MIN_CONF`-dropped lines are reported separately (they hit the full-pipeline
|
| 45 |
+
number but not rec-lines).
|
| 46 |
+
4. Also submit **full-pipeline** Indra (`predict` without `--boxes`) on the
|
| 47 |
+
same set for the det+rec+order end-to-end number.
|
| 48 |
+
|
| 49 |
+
**Pass/fail:** Indra rec-lines CER < deployed rec CER on ≥ 2 of 3 eval sets,
|
| 50 |
+
with per-set mean confidence reported. Also record det recall (lines found /
|
| 51 |
+
GT lines) for the full-pipeline run.
|
| 52 |
+
|
| 53 |
+
## Experiment B — fine-tune the recognizer (the supported path)
|
| 54 |
+
|
| 55 |
+
**Goal:** Adapt `rec.onnx`'s weights to Nepali-specific distribution
|
| 56 |
+
(conjunct-heavy printing, nukta, ₹, Devanagari digits in running text), retrain
|
| 57 |
+
with `model.safetensors` as init, export an ONNX with the *identical*
|
| 58 |
+
interface, drop-in replace.
|
| 59 |
+
|
| 60 |
+
**Architecture requirements to reproduce byte-compatible behavior (from
|
| 61 |
+
Investigation §1):**
|
| 62 |
+
- module: stem Conv(1→128, k32×4, s32×4) + BN + HardSwish → mask Mul →
|
| 63 |
+
10 × [dw 9-k1 g128 pad4 → expand 1×1 128→256 → HS → SE(masked-mean→256→64→256
|
| 64 |
+
sigmoid gate) → project 1×1 256→128 → +residual → mask Mul] → head 1×1
|
| 65 |
+
128→1678 → softmax → argmax.
|
| 66 |
+
- Padding mask `(arange(W/4) < lengths).float()` with `lengths =
|
| 67 |
+
ceil(widths/4)` (clip W/4); apply post-stem, at SE pools (mean over VALID
|
| 68 |
+
steps only), and after each block's residual add. **The mask is functional,
|
| 69 |
+
not cosmetic: it keeps SE statistics from being diluted by padding.**
|
| 70 |
+
- Parameter mapping table: `model.safetensors` keys `rec/<name>` —
|
| 71 |
+
`stem.weight` [128,1,32,4], `stem.bias` [128], `stem_bn.{weight,bias,
|
| 72 |
+
running_mean,running_var}` [128], `blocks.<i>.dw.weight` [128,1,9] /
|
| 73 |
+
`dw.bias` [128] (ONNX names `onnx::Conv_{369,370};372,373,…396,397`),
|
| 74 |
+
`blocks.<i>.expand.{weight,bias}` [256,128], `blocks.<i>.se.fc1/fc2`,
|
| 75 |
+
`blocks.<i>.project.{weight,bias}` [128,256], `head.weight` [1678,128,1],
|
| 76 |
+
`head.bias` [1678].
|
| 77 |
+
- Loss: CTC (log_softmax over [B,T,1678]; blank id 0; input_lengths = lengths,
|
| 78 |
+
target = `vocab.encode(canonical(text))` IDs list; NFC-normalize exactly as
|
| 79 |
+
`vocab.py:42`).
|
| 80 |
+
- Optimizer: AdamW, lr 3e-4 cosine, ~10–20k steps, batch 32 lines by
|
| 81 |
+
width-bucket, crop 32px — the pipeline above runs on 1× A100/40 GB in
|
| 82 |
+
minutes per epoch for our line counts ([B] exact data volume from
|
| 83 |
+
`~/ocr/ocr-deva-ft/data`).
|
| 84 |
+
|
| 85 |
+
**Unique risk (verified):** the trainer's crop augmentation recipe is closed;
|
| 86 |
+
starting too aggressively (`rotate`, `tone`) would shift the input
|
| 87 |
+
distribution away from `crop_line`'s exact ops. Mitigation: augmentations
|
| 88 |
+
limited to elastic/scale jitter of ±5% consistent with `PAD_Y/PAD_X` bend,
|
| 89 |
+
train/eval identical crop code (shared module imported from `rec_preproc.py`
|
| 90 |
+
— the DOC that matters).
|
| 91 |
+
|
| 92 |
+
**Export/eval:** torch → ONNX with same outputs (`ids,conf,lengths`) or a
|
| 93 |
+
logits-only export + a 15-line Python decode; either way validate against
|
| 94 |
+
`rec.onnx` outputs on 512 in-crops before scoring (**exactness test**:
|
| 95 |
+
max |Δconf| < 1e-3 on identical inputs).
|
| 96 |
+
|
| 97 |
+
**Verified offline 2026-10-20** (torch 2.14.1+cpu / onnxruntime 1.31.0,
|
| 98 |
+
`tools/exp/verify_bundle.sh`): `model.safetensors` → IndraRec (1,236,366
|
| 99 |
+
tensors) → ONNX export matches the shipped `rec.onnx` with **0 id mismatches,
|
| 100 |
+
0 lengths mismatches, max |Δconf| 7.1e-6** on the 1,590-step probe battery;
|
| 101 |
+
the post-train gate (torch .pt vs its own export) passes at the same tolerance;
|
| 102 |
+
`run_rec.py` reproduces the recorded smoke CER (3.49 %) bit-for-bit. The
|
| 103 |
+
exporter forces the legacy TorchScript path (`dynamo=False`) because torch ≥ 2.6
|
| 104 |
+
defaults to the torch.export exporter, which needs the `onnxscript` package.
|
| 105 |
+
|
| 106 |
+
**Acceptance:** fine-tuned rec-lines CER < zero-shot rec-lines CER on all
|
| 107 |
+
3 frozen sets by ≥ 2 points relative, on Devanagari digit-heavy and
|
| 108 |
+
conjunct-heavy slices specifically (those are where the smoke run showed
|
| 109 |
+
weakness: `सङ्केत→सड्लेत`, `₹→र`, `ऋतु→तऋहतु`).
|
| 110 |
+
|
| 111 |
+
## Experiment C — fine-tune the line detector / order model
|
| 112 |
+
|
| 113 |
+
**Verdict from Investigation §3: blocked by shipped-artifact reality** —
|
| 114 |
+
`det.onnx`/`order.onnx` come with NO state dict and NO recipe. Two paths:
|
| 115 |
+
1. **Torch re-implementation from scratch** using the target specs that ARE
|
| 116 |
+
in the repo (`det_post.encode_targets` → line core logit + top/bottom
|
| 117 |
+
distance regression; `order_post.label_order` → 10 block classes +
|
| 118 |
+
pairwise salience). Cost: reimplementing the backbone (which is unknown —
|
| 119 |
+
the ONNX graphs would need the same node-by-node reverse archaeology as
|
| 120 |
+
rec.onnx: feasible, we did it, but heavy) + a data pipeline + synth label
|
| 121 |
+
generator. **Not recommended as the first move** — zero-shot + rec-only
|
| 122 |
+
fine-tuning captures most value: the det model already detects 10/10
|
| 123 |
+
synthetic lines and machine-printed Devanagari well.
|
| 124 |
+
2. **Skip det/order entirely**; if detection quality on real scan material
|
| 125 |
+
turns out to be the bottleneck in A-full, locally replace with our known
|
| 126 |
+
corresponding PP-OCR det model and keep only the Indra rec — that is
|
| 127 |
+
Experiment D.
|
| 128 |
+
|
| 129 |
+
## Experiment D — hybrid: keep PP-OCR det, swap rec → Indra
|
| 130 |
+
|
| 131 |
+
**Goal:** combine a known-good Devanagari line finder (PP-OCRv5 det) with the
|
| 132 |
+
Indra recognizer under identical crops, and vice-versa. Outputs line boxes
|
| 133 |
+
jsonl exactly like A.2-step 1. This is a 1-day experiment **with zero training**
|
| 134 |
+
and makes the fair-comparison claim airtight: if `PP-det + Indra-rec` beats
|
| 135 |
+
`PP-det + PP-rec` on the same crops, the recognizer itself is the win; if not,
|
| 136 |
+
Indra's advantage (if any) lives elsewhere (orientation/order/layout).
|
| 137 |
+
|
| 138 |
+
## Experiment E — productionization / distill for WSL CPU box
|
| 139 |
+
|
| 140 |
+
**Goal:** a deployable artifact for the 8-core/3 GB (or T4) runtime:
|
| 141 |
+
- Indra package supports `rec.int8.onnx` automatically (`pipeline.py:195–200`)
|
| 142 |
+
— quantize the rec model once (no retraining), then re-run Experiment A with
|
| 143 |
+
the int8 model: acceptance: CER regression ≤ +0.3 pts absolute and
|
| 144 |
+
≥ 2× throughput.
|
| 145 |
+
- **Measured local guidance (2026-10-20, WSL CPU, threads=4, smoke crops;
|
| 146 |
+
indicative, not a benchmark):** ORT *dynamic* (weights-only) quantization is
|
| 147 |
+
a **trap** for this Conv+SE model — 0.69–0.27× fp32 throughput (ConvInteger +
|
| 148 |
+
per-activation quant/dequant dominates) though greedy ids stay identical and
|
| 149 |
+
max |Δconf| = 0.077. *Static* QDQ uint8 (per-tensor, MinMax, 64 calib crops)
|
| 150 |
+
built 3.5× smaller (4.98 → 1.43 MB) and ran **~1.4× faster** than fp32 — on a
|
| 151 |
+
stronger box the gap should widen; the ≥2× acceptance must be re-measured
|
| 152 |
+
there. Tool: `tools/exp/e_int8_quantize.py` (static, calibration through the
|
| 153 |
+
shared `crop_line`; `--check-smoke` reports id/conf drift; writes a JSON report).
|
| 154 |
+
Use real line crops for calibration (the local test used noise → id drift 56
|
| 155 |
+
steps on the smoke page; real calibration should cut that).
|
| 156 |
+
- Latency budget on our box is meaningless (WSL starved) — measure only on
|
| 157 |
+
the T4: lines/s at long-side 1200/1600/2048 (det scales with page size).
|
| 158 |
+
- Memory: with `enable_cpu_mem_arena=False` (already set in the package),
|
| 159 |
+
line-batched rec is flat per page.
|
| 160 |
+
|
| 161 |
+
---
|
| 162 |
+
|
| 163 |
+
## §11 Data plan for fine-tuning (Experiment B)
|
| 164 |
+
|
| 165 |
+
- Source: the same frozen train corpus that produced the deployed fine-tuned
|
| 166 |
+
PP-OCR model (`~/ocr/ocr-deva-ft/data` — [B] inventory is on remote;
|
| 167 |
+
`docs/DEVANAGARI_RESOURCES_INVENTORY.md` documents it). **No dataset is
|
| 168 |
+
copied off the remote machine** — training artifacts/results files transfer,
|
| 169 |
+
not eval sets.
|
| 170 |
+
- Emphasis slices for Nepali: conjunct-heavy book/print pages, Devanagari
|
| 171 |
+
digit blocks, ₹-annotated amounts, मध्य-मस useful — determined by the
|
| 172 |
+
per-line CER profile of A-zero (error concentration, not vibes).
|
| 173 |
+
- Same-crop exactness: train the augmented pages through the exact
|
| 174 |
+
`crop_line`/`pad_batch` code (import from the cloned package), so zero-shot
|
| 175 |
+
and fine-tuned are measured through one code path.
|
| 176 |
+
- Leakage (§12): any *train* page rendered by the deployed pipeline must be
|
| 177 |
+
excluded from ALL three frozen eval sets; confirm set boundaries by
|
| 178 |
+
re-running `tools/eval_metrics.py`'s pairing logic [B].
|
| 179 |
+
|
| 180 |
+
## §12 Leakage review [B]
|
| 181 |
+
|
| 182 |
+
1. **Set purity:** frozen eval sets must not overlap the rec fine-tune
|
| 183 |
+
corpus. Verify by page-hash (sha256 of rendered page images), not
|
| 184 |
+
filenames ([B] run on remote).
|
| 185 |
+
2. **Synth contamination:** if synthetic rendered pages were part of
|
| 186 |
+
`ocr-deva-ft` training and the eval sets contain natural scans only, no
|
| 187 |
+
leakage; confirm the eval-set provenance doc. Rendered-by-us
|
| 188 |
+
(`make_synth_page.py`) pages are NEVER used for training or reporting.
|
| 189 |
+
3. **Adaptation leakage:** Experiment D swaps components inside a pipeline
|
| 190 |
+
built on eval sets — acceptable for comparison because GT boxes come from
|
| 191 |
+
the DETECTOR of the deployed pipeline on EVAL images only, and neither
|
| 192 |
+
component is trained on them.
|
| 193 |
+
4. **Font leakage:** if fonts in the training corpus are also in the eval
|
| 194 |
+
sets' print source (same family), CER estimates are optimistic for BOTH
|
| 195 |
+
systems equally — acceptable for A/B ranking, not for absolute claims;
|
| 196 |
+
state it in the report.
|
| 197 |
+
|
| 198 |
+
## §13 Licensing matrix [B for dataset rows]
|
| 199 |
+
|
| 200 |
+
| Item | License | Fine-tune allowed? | Deploy allowed? | Notes |
|
| 201 |
+
|---|---|---|---|---|
|
| 202 |
+
| Indra weights v3.1.1 | AI PUBS Open RAIL-M (modified) | **YES, explicit** (§II ¶36) | YES (restrictions = Attachment A misuse list; internal OCR unaffected) | Derivative weights → same license + notice on redistribution |
|
| 203 |
+
| `indra_ocr` code | Apache-2.0 | n/a | YES | |
|
| 204 |
+
| PaddleOCR (baseline) | Apache-2.0 | YES (we already fine-tune it) | YES | current deployed |
|
| 205 |
+
| PP-OCRv5 Mobile Devanagari weights | Apache-2.0 | YES | YES | deployed |
|
| 206 |
+
| Nepali train corpus | per `docs/DATA_LICENSING.md` [B] | pending review | — | no removal of datasets from remote |
|
| 207 |
+
| Frozen eval sets | per `docs/DATA_LICENSING.md` [B] | evaluation only | — | |
|
| 208 |
+
| Noto Sans Devanagari (testrender only) | OFL 1.1 | YES (fonts embedding fine) | test asset only | `tools/indra_eval/testdata/` |
|
| 209 |
+
|
| 210 |
+
## §14 Deployment comparison
|
| 211 |
+
|
| 212 |
+
| Axis | Indra v3.1.1 (rec-only swap / full) | PP-OCRv5 Mobile Devanagari deployed |
|
| 213 |
+
|---|---|---|
|
| 214 |
+
| Recognizer params | **1.236 M** (verified) | rec ~similar class (Mobile range; exact count from remote bench) |
|
| 215 |
+
| Runtime | onnxruntime CPU (already installed here) | paddle (remote) |
|
| 216 |
+
| Extra machinery needed | **none** — single ONNX + meta.json vocab JSON | paddle + FineTuned assets + converter glue |
|
| 217 |
+
| Latency (lines/s, T4 CPU) | measure (A/D/E) | measure |
|
| 218 |
+
| Deterministic CPU memory | flat batch (mask) | paddle per-page buffers |
|
| 219 |
+
| Order/reading model | included (order.onnx; optional) | not used for plain OCR flow |
|
| 220 |
+
| Honest staffing: fine-tune effort | reimplement trainer (~1–2 days) | already exists for Paddle |
|
| 221 |
+
| Dependency risk | pinned HF revision + local files | paddle version drift |
|
| 222 |
+
|
| 223 |
+
## §17 Decision tree
|
| 224 |
+
|
| 225 |
+
1. **Run A (zero-shot rec-lines + full) on the frozen sets.**
|
| 226 |
+
- If Indra rec CER < deployed rec CER **on 2 of 3 sets**: proceed to the
|
| 227 |
+
hybrid experiment D (PP-det + Indra-rec); if D holds the win, plan a
|
| 228 |
+
**rec-swap deployment trial** alongside the existing PP pipeline(1 week).
|
| 229 |
+
- If Indra rec CER ≥ deployed rec CER: record the gap per slice. If the
|
| 230 |
+
gap is train-material (conjuncts/₹/digits): run **B** (fine-tune rec
|
| 231 |
+
from safetensors, ~10–20k steps); expected to close a
|
| 232 |
+
distribution-gap-shaped loss; decide by ΔCER on the eval sets.
|
| 233 |
+
2. **Regardless of A's outcome,** the graceful fallback stands: bringing Indra
|
| 234 |
+
recognizer into our PP-OCR det pipeline is a **drop-in component** (same
|
| 235 |
+
crops→ink-map contract), so nothing about A/B/D commits us.
|
| 236 |
+
3. **Det/order fine-tuning (C) stays out of scope** until det-on-real-scan
|
| 237 |
+
failure is demonstrated in A-full (`det recall < line recall of deployed`).
|
| 238 |
+
4. **Never** touch: datasets on the remote, the deployed pipeline's recipes,
|
| 239 |
+
its eval scripts, or `~/.ssh`/`~/.bashrc`. All experiments run in a
|
| 240 |
+
separate directory (`~/ocr/OCR_INDRA/`), copying only code/weights in.
|
| 241 |
+
|
| 242 |
+
## §18 Status summary
|
| 243 |
+
|
| 244 |
+
- [V] runnable here end-to-end: smoke chain (synth page → rec-only → full →
|
| 245 |
+
CER), all inputs/outputs/paths in Investigation §8.
|
| 246 |
+
- [B] everything requiring `khalti-t4`: A.2 (frozen sets), B data, C skip
|
| 247 |
+
decision input, licensing rows, E throughput on deployment hardware.
|
| 248 |
+
- [V] fine-tune toolkit verified offline 2026-10-20 (`tools/exp/verify_bundle.sh`:
|
| 249 |
+
export exactness pass; mechanics training + post-train gate pass; A/B
|
| 250 |
+
orchestrations complete on a local stand-in set); int8 static/dynamic
|
| 251 |
+
behaviour probed (dynamic = slower trap, static = ~1.4× faster, 3.5× smaller).
|
| 252 |
+
- Open: WP det recall numbers, paddle latency comparison, int8 CER on real
|
| 253 |
+
calibration + deployment-box throughput.
|
OCR_INDRA/docs/INDRA_INVESTIGATION.md
ADDED
|
@@ -0,0 +1,292 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
# Indra OCR — Engineering Investigation (v3.1.1, package 0.3.2)
|
| 2 |
+
|
| 3 |
+
Investigator: OCR_INDRA workspace, 2026-10-17.
|
| 4 |
+
Method: every claim below is backed by (a) model artifacts inspected bit-for-bit
|
| 5 |
+
(`models/indra-ocr-v3.1.1/`, Hub revision `v3.1.1`) or (b) source at a cited
|
| 6 |
+
file:line in the cloned repo (`indra/`, commit `3d773ae`, tag v0.3.2).
|
| 7 |
+
Verification scripts live in `tools/indra_eval/` and are re-runnable.
|
| 8 |
+
|
| 9 |
+
Status legend: **[V]** verified with evidence here, **[B]** blocked (needs the
|
| 10 |
+
remote machine `khalti-t4`), **[X]** closed as refuted/inaccurate vs the
|
| 11 |
+
earlier report `INDRA_OCR_NEPALI_FEASIBILITY.md`.
|
| 12 |
+
|
| 13 |
+
---
|
| 14 |
+
|
| 15 |
+
## 0. Verdict
|
| 16 |
+
|
| 17 |
+
**The recognizer is trainable in principle and Nepali-ready in vocabulary, but
|
| 18 |
+
fine-tuning is a reimplementation project: the repo ships no training code and
|
| 19 |
+
no training weights for anything but the recognizer.**
|
| 20 |
+
|
| 21 |
+
- Zero-shot evaluation (Experiment A) is *ready today* — the adapter
|
| 22 |
+
(`tools/indra_eval/eval_indra.py`) runs Indra end-to-end on CPU and scores
|
| 23 |
+
CER; it needs only the frozen Nepali eval sets from the remote machine [B].
|
| 24 |
+
- The CTC recognizer (`rec.onnx`) covers 100% of Devanagari (all 128 assigned
|
| 25 |
+
codepoints U+0900–U+097F verified in the vocab) — no head surgery needed [V].
|
| 26 |
+
- In a same-crop head-to-head under our environment, the ~1.24 M-param
|
| 27 |
+
Indra recognizer scored **CER 3.49%** on a synthetic Nepali page vs the
|
| 28 |
+
deployed PP-OCRv5-Mobile Devanagari recognizer's published behavior on
|
| 29 |
+
similar synthetic material (to be re-measured on identical crops) [V smoke / B].
|
| 30 |
+
- Fine-tuning only the recognizer with `model.safetensors` as init is the
|
| 31 |
+
*only* supported adaptation path: the artifacts ship trainable weights for
|
| 32 |
+
`rec/` exclusively; det/order/orient/math exist only as ONNX graphs with no
|
| 33 |
+
accompanying state dict [V].
|
| 34 |
+
|
| 35 |
+
---
|
| 36 |
+
|
| 37 |
+
## 1. What exactly is the ~1M-param recognizer? [V]
|
| 38 |
+
|
| 39 |
+
**Answer: a MobileNetV3-style 1-D CNN trunk with padding-masked SE blocks and
|
| 40 |
+
a flat CTC head — 1,236,366 parameters exactly, no RNN, no attention.**
|
| 41 |
+
|
| 42 |
+
Evidence: `tools/indra_eval/inspect_rec_nodes.py` (201-node dump of
|
| 43 |
+
`rec.onnx`), `tools/indra_eval/inspect_rec_mask.py` (intermediate-tensor run),
|
| 44 |
+
`tools/indra_eval/inspect_models.py` (safetensors ↔ ONNX initializer
|
| 45 |
+
cross-check: 108 tensors, 1,236,366 F32 params, byte-identical set).
|
| 46 |
+
|
| 47 |
+
### 1.1 Tensor flow (all verified from the graph)
|
| 48 |
+
|
| 49 |
+
```
|
| 50 |
+
input x [B,1,32,W] f32 ink map (0=paper, 1=ink; see §1.3)
|
| 51 |
+
input widths [B] int64 (true crop widths BEFORE padding)
|
| 52 |
+
|
| 53 |
+
0 Conv(k=32x4, stride=32x4, 1→128, pad 0) "collapse height, stride time 4"
|
| 54 |
+
2 Squeeze axis 2 → [B,128,T] with T = W/4
|
| 55 |
+
1 BN(eps 1e-5, stem-only BatchNormalization) → HardSwish
|
| 56 |
+
43 Mul × Cast_4 (padding mask, see §1.2) → trunk input
|
| 57 |
+
|
| 58 |
+
10 × { dw Conv [128,1,9] g=128 pad 4/4
|
| 59 |
+
expand Conv 1×1 128→256 → HardSwish
|
| 60 |
+
SE: Mul(mask) → ReduceSum(t) ←/→ ReduceSum(mask) = valid count
|
| 61 |
+
Div (mean over VALID timesteps, Clip min 1)
|
| 62 |
+
fc1 256→64 → Relu → fc2 64→256 → Sigmoid → gate Mul
|
| 63 |
+
project Conv 1×1 256→128
|
| 64 |
+
Add residual → Mul(mask) } ← mask re-applied per block
|
| 65 |
+
|
| 66 |
+
188 Conv 1×1 128→1678 (head) → Transpose [B,T,1678]
|
| 67 |
+
191 ArgMax → ids [B,T]
|
| 68 |
+
190/193..200 softmax-max (logsumexp-free: ReduceMax, Exp, ReduceSum, Recip)
|
| 69 |
+
→ conf [B,T]
|
| 70 |
+
27 Clip(lengths) → lengths [B]
|
| 71 |
+
```
|
| 72 |
+
|
| 73 |
+
Op census (from the node dump): 52 Conv, 11 HardSwish, 10 Relu, 10 Sigmoid
|
| 74 |
+
(SE gates), 32 Mul, 12 ReduceSum, 1 BatchNormalization, 1 ArgMax — plus
|
| 75 |
+
glue. The parameters: 10 identical residual blocks (dw 128×1×9 + bias,
|
| 76 |
+
expand 256×128, project 128×256, SE 64×256 + 256×64) ≈ 118 k/block + stem
|
| 77 |
+
128×32×4 + BN 4×128 + head 1678×128 ≈ 1,236,366 total [V].
|
| 78 |
+
|
| 79 |
+
### 1.2 The padding mask `Cast_4` — semantics now fully pinned [V, closes a [U]]
|
| 80 |
+
|
| 81 |
+
`Cast_4 = (arange(W/4) < lengths).float()` of shape `[B,1,W']`, built in-graph
|
| 82 |
+
from:
|
| 83 |
+
- nodes 3–27: `lengths = floor((widths+3)/4) = ceil(widths/4)`, clipped to
|
| 84 |
+
`W/4` (Clip max = `Shape(x)[2] / 4`... clipped to the feature length);
|
| 85 |
+
numeric table: `widths 1,2,3,4 → 1; 5 → 2; 9 → 3; 65 → 17; 129 → 32 (= W/4,
|
| 86 |
+
clipped); 200 → 32`.
|
| 87 |
+
- nodes 31–40: `Range(0, W/4) < lengths` (broadcast `[1,W'] < [B,1]`) →
|
| 88 |
+
`Cast` f32 twice (PyTorch export artifact of `unsqueeze().to(f32)`).
|
| 89 |
+
|
| 90 |
+
The mask is consumed **three ways** (this closes the earlier "[U] what does
|
| 91 |
+
Cast_4 do?"):
|
| 92 |
+
1. **post-stem zeroing** (node 43) and **per-block output re-zeroing**
|
| 93 |
+
(nodes 43, 61, 75, …, 187): the trunk never carries energy into padded
|
| 94 |
+
timesteps — proven numerically: with ink drawn across the whole 128-column
|
| 95 |
+
input but `widths=64`, final-block features at t ≥ 16 are *exactly 0.0*
|
| 96 |
+
(max |f| = 0.0) while the same pixels unmasked (widths=128) reach |f|=15.6;
|
| 97 |
+
2. **padding-aware SE pooling** (each block: `ReduceSum(HW × mask) /
|
| 98 |
+
max(ReduceSum(mask), 1)` → channel means over VALID timesteps only; the
|
| 99 |
+
mask sum is the divisor, so padded columns cannot dilute the squeeze stats);
|
| 100 |
+
3. the **head reads masked features** (node 187→188), so at padded timesteps
|
| 101 |
+
the graph emits whatever the head computes on a zero vector — measured:
|
| 102 |
+
argmax = 0 (blank) with conf ≈ 0.78. **`ids/conf` at t ≥ lengths are NOT
|
| 103 |
+
special-cased; callers must truncate by `lengths`.** `pipeline.py:297`
|
| 104 |
+
does exactly this (`n = int(lengths[k])`).
|
| 105 |
+
|
| 106 |
+
### 1.3 Time stride is 4, not 8 [X — closes a wrong claim]
|
| 107 |
+
|
| 108 |
+
`rec_preproc.py:24` says `WIDTH_MULTIPLE = 8 # widths are padded to a
|
| 109 |
+
multiple of this (timestep stride)`. The graph contradicts the parenthetical:
|
| 110 |
+
the stem Conv strides are `[32,4]`, giving **one timestep per 4 input
|
| 111 |
+
columns** (`T = W/4`, `lengths = ceil(widths/4)`), and padding to a multiple
|
| 112 |
+
of 8 is about the W dimension of the input only (it makes every 2 timesteps
|
| 113 |
+
aligned, a vectorization nicety). Verified numerically: `T = 32` for
|
| 114 |
+
`W = 128`, and `lengths` equals `ceil(widths/4)` at every measured value.
|
| 115 |
+
The comment is wrong as written; the code is right.
|
| 116 |
+
|
| 117 |
+
### 1.4 Output contract [V — closes a [U]]
|
| 118 |
+
|
| 119 |
+
Graph outputs are exactly `ids [B,T] int64`, `conf [B,T] f32`,
|
| 120 |
+
`lengths [B] int64` (argmax + softmax max prob + valid count computed
|
| 121 |
+
in-graph). `pipeline.py:294` consumes them positionally as
|
| 122 |
+
`best, pmax, lengths`, then:
|
| 123 |
+
- truncates to `n = int(lengths[k])` (:297),
|
| 124 |
+
- CTC-greedy collapse: keep ≠ blank(0) and skip repeats (:299),
|
| 125 |
+
- confidence = mean `pmax` over kept tokens (:315),
|
| 126 |
+
- **drops the whole line if conf < `MIN_CONF = 0.5`** (:60, :316–319).
|
| 127 |
+
|
| 128 |
+
Any retrained recognizer exported to ONNX must reproduce this contract
|
| 129 |
+
(argmax + softmax-max + lengths on the same output names/order), or
|
| 130 |
+
`pipeline.py:294` and the x-positions step (which maps CTC peak timesteps
|
| 131 |
+
back to page x via the crop scale, :295–306) must be updated.
|
| 132 |
+
|
| 133 |
+
### 1.5 Recognizer input preprocessing (the shard that fine-tuning must clone)
|
| 134 |
+
|
| 135 |
+
`rec_preproc.py` — `crop_line` (:27): expand box by `PAD_Y=0.2 × h` vertically
|
| 136 |
+
and `PAD_X=0.3 × h` horizontally (:22–23), restrict to page, resize so
|
| 137 |
+
height = `LINE_HEIGHT = 32` keeping aspect (`out_w = max(WIDTH_MULTIPLE,
|
| 138 |
+
min(max_width=4096, round(cw*32/ch)))`), producing an ink map where paper = 0.
|
| 139 |
+
`pad_batch` (:56) right-pads W to a multiple of 8 in the batch; `width_batches`
|
| 140 |
+
(:69) groups crops so W-dim batch padding is bounded. The TRAINER uses the
|
| 141 |
+
same crop per its docstring (:5–10 "expand … resize … ink map") — the docstring
|
| 142 |
+
states train/infer share this shard, which is why zero-shot works at all and
|
| 143 |
+
why fine-tuning must reimplement `crop_line` + `pad_batch` bit-exactly.
|
| 144 |
+
|
| 145 |
+
## 2. What handles Devanagari? [V]
|
| 146 |
+
|
| 147 |
+
**Only `rec.onnx` reads Devanagari — and its vocabulary already includes
|
| 148 |
+
every assigned Devanagari codepoint.**
|
| 149 |
+
|
| 150 |
+
- `vocab.py:170` `Vocab.multi()` → 1,677 tokens + blank class 0 (:3, :207),
|
| 151 |
+
head = 1,678 classes. The `meta.json` vocab string is exactly this vocab
|
| 152 |
+
(`tools/indra_eval/inspect_vocab.py`).
|
| 153 |
+
- Coverage: **128/128 assigned codepoints in U+0900–U+097F** — 80 Lo letters,
|
| 154 |
+
11 Mc vowel signs, 23 Mn marks incl. ़ (U+093C nukta) and the conjunct
|
| 155 |
+
virama क्, plus ॉ (U+0950), ँ । (U+0964), ॥ (U+0965), digits ०–९
|
| 156 |
+
(U+0966–096F), ₹ (U+20B9). Bengali block also present (96 codepoints).
|
| 157 |
+
- The recognizer is routed by two *fixed* guards, not the network:
|
| 158 |
+
- `script.onnx` (hanzi binary classifier, sigmoid scalar; `pipeline.py:274–289`)
|
| 159 |
+
sends hanzi lines to `rec_cjk.onnx` — irrelevant for Nepali pages
|
| 160 |
+
(all-Devanagari pages never trigger it; the classifier is line-level and
|
| 161 |
+
trained on hanzi).
|
| 162 |
+
- `vocab.canonical` (:42) NFC-normalizes; `Vocab.encode` (:212) drops
|
| 163 |
+
out-of-vocab codepoints — fine-tuning data must be pre-normalized the
|
| 164 |
+
same way or the loss silently degrades (the trainer's data pipeline is
|
| 165 |
+
closed; safest is to clone `canonical`/`encode`).
|
| 166 |
+
- Confirmed counter-evidence collected in the smoke run: ़ (nukta) is often
|
| 167 |
+
*dropped contextually* at 32 px height (ऱ and र differ only by a 1–2 px
|
| 168 |
+
dot); ₹ was read as र. Both are training-distribution issues, not capability
|
| 169 |
+
gaps — exactly the kind of thing fine-tuning would fix (Experiment B).
|
| 170 |
+
|
| 171 |
+
## 3. Component map: trainable / fixed-but-replaceable / fixed [V]
|
| 172 |
+
|
| 173 |
+
| Component | Where weights come from | Trainable via shipped artifacts? | Replaceability |
|
| 174 |
+
|---|---|---|---|
|
| 175 |
+
| `rec.onnx` recognizer | `model.safetensors` (108 tensors, all `rec/`) | **YES** — the only shipped state dict; init for a torch CTC retrain (Experiment B) | head/vocab identical → drop-in for +1,678 classes |
|
| 176 |
+
| `det.onnx` line detector | ONNX only | NO state dict shipped | could retrain in torch (targets spec exists: `det_post.encode_targets`, `det_post.py:48` — `core logit + top/bottom distances` per line, `CORE_FRAC=0.4`, `CORE_X_SHRINK=0.15`) — but training code is closed → high effort |
|
| 177 |
+
| `order.onnx` reading order/layout | ONNX only | NO | same: `order_post.label_order`/`decode_*` reveal supervision (10 block classes) but no weights |
|
| 178 |
+
| `orient.onnx` page rotation | ONNX only | NO | trivial to replace with a torch/tlclassifier or heuristic |
|
| 179 |
+
| `mask.onnx` CenterMask | NOT SHIPPED in v3.1.1 (`pipeline.py:201–205` optional) | — | absent here; `crop_lines(gray, boxes, 32, None)` runs w/o it (verified) |
|
| 180 |
+
| `script.onnx` + `rec_cjk.onnx` | ONNX only | NO | optional; would be excluded from a Devanagari build |
|
| 181 |
+
| `math*.onnx` (4 files) | NOT shipped unless `math=True` (`models.py:22–44`) | — | out of scope |
|
| 182 |
+
| geometry/deskew/layout postproc | pure Python (`pipeline.py:37–149`, `layout_post.py`) | fixed code, no training needed | replaceable with own code paths |
|
| 183 |
+
| text-layer (PDF) naive path | `textlayer.py` | fixed code | irrelevant to scans/Nepali notes |
|
| 184 |
+
|
| 185 |
+
**The decisive asymmetry:** Google ships a *recognizer-only* training starting
|
| 186 |
+
point. Fine-tuning anything else means either (a) training from scratch (det
|
| 187 |
+
targets spec'd, but no reference weights/recipe) or (b) leaving det/order
|
| 188 |
+
frozen ONNX and only exercising the rec head — that's Experiment B; both are
|
| 189 |
+
documented in `INDRA_EXPERIMENTS.md`.
|
| 190 |
+
|
| 191 |
+
## 4. Integration architecture (how a page becomes text) [V]
|
| 192 |
+
|
| 193 |
+
`pipeline.py:456–470` (`OrtOCR.__call__`):
|
| 194 |
+
1. `upright()`: optional `orient.onnx` turn above `ORIENT_MIN = 0.95` (:54) +
|
| 195 |
+
skew estimate/deskew up to 5° (:62–63).
|
| 196 |
+
2. `detect()`: `det.onnx` on a page resized long-side to detector input,
|
| 197 |
+
threshold → line boxes (uses `det_post.page_to_input` / `decode`;
|
| 198 |
+
`ADAPTIVE_MIN_H=9px` re-detect pass at a larger scale, :37–40).
|
| 199 |
+
3. `recognize()` (:264–320): CenterMask if present (not shipped); per-line
|
| 200 |
+
`crop_line` → `pad_batch` → rec session, CTC collapse & conf gating
|
| 201 |
+
(§1.2/§1.4).
|
| 202 |
+
4. line splitting at wide gaps (`SPLIT_GAP=3.0`, `CELL_GAP=1.8`,
|
| 203 |
+
`GUTTER_GAP=1.6`, :44–47) and gutter detection (:342).
|
| 204 |
+
5. `reading_order()` on `order.onnx` (`feats [1,N,16]`, outputs pairwise salience
|
| 205 |
+
`a`, block class `cls` [10 classes], same-block `same`, table t-heads `tab`)
|
| 206 |
+
then layout (`layout_post.py: page_blocks`, `page_markdown`).
|
| 207 |
+
|
| 208 |
+
MIN_CONF=0.5 means **a busy page with low-quality Devanagari can silently
|
| 209 |
+
emit empty lines** in Experiment A — compare-on-GT-boxes (rec-lines mode)
|
| 210 |
+
sidesteps this entirely, which is why Experiment A does both full-pipeline
|
| 211 |
+
and rec-lines scoring.
|
| 212 |
+
|
| 213 |
+
## 5. Licensing [V]
|
| 214 |
+
|
| 215 |
+
- **Model weights** (`models/indra-ocr-v3.1.1/LICENSE`): **AI PUBS OPEN RAIL-M
|
| 216 |
+
(MODIFIED), v0.1, 2023-03-02**. Section II ¶36 states verbatim: *"Use may
|
| 217 |
+
include creating any content with, fine-tuning, updating, running, training,
|
| 218 |
+
evaluating and/or re-parametrizing the Model"* — **fine-tuning is explicitly
|
| 219 |
+
licensed**. ¶5 attaches Attachment A use-based restrictions (standard
|
| 220 |
+
OpenRAIL misuse list — weapons/illegal/defamation; OCR and document
|
| 221 |
+
processing is untouched) and requires passing the same restrictions to
|
| 222 |
+
distributees; ¶4 permits hosting/redistribution with notice. Derivative
|
| 223 |
+
weights (fine-tuned checkpoints, distillations) fall under the same license.
|
| 224 |
+
- **Code** (`indra/LICENSE`): **Apache-2.0** — reuse, modification,
|
| 225 |
+
redistribution of the inference package are standard.
|
| 226 |
+
- Practical consequence: our fine-tuned recognizer (Experiment B) is legally a
|
| 227 |
+
"Derivative of the Model" under OpenRAIL-M — internal deployment and
|
| 228 |
+
evaluation are unrestricted; redistribution requires keeping the license +
|
| 229 |
+
restriction notice. Full matrix incl. Nepali datasets: `§12` of
|
| 230 |
+
`INDRA_EXPERIMENTS.md` (dataset side **[B]** — needs `docs/DATA_LICENSING.md`
|
| 231 |
+
on the remote machine).
|
| 232 |
+
|
| 233 |
+
## 6. Closing the prior report's [U] items
|
| 234 |
+
|
| 235 |
+
| Prior unknown | Status | Evidence |
|
| 236 |
+
|---|---|---|
|
| 237 |
+
| exact recognizer arch (RNN? CNN?) | **[V] + [X]**: pure CNN+CTC, NO RNN/attention | §1.1, node dump |
|
| 238 |
+
| timestep stride | **[X]**: 4, not 8 | §1.3 (graph constants + inc.timings) |
|
| 239 |
+
| output contract (best/pmax docstrings vs solo head) | **[V]**: `ids`,`conf`,`lengths` names; truncation mandatory | §1.4 |
|
| 240 |
+
| safetensors ↔ ONNX mapping (esp. depthwise `onnx::Conv_369…397`) | **[V]**: 108 tensors = 108 initializers, 1,236,366 params both ways; dw weights are `rec/onnx::Conv_369` [128,1,9] + `…370` [128] bias pairs | `inspect_models.py` |
|
| 241 |
+
| Devanagari coverage claim | **[V]**: 128/128 codepoints in vocab | `inspect_vocab.py` |
|
| 242 |
+
| What re-export must reproduce | **[V]**: mask + per-block re-zeroing + SE-valid-norm + decode tail; `lengths = ceil(widths/4)` clip W/4 | §1.2, `inspect_rec_mask.py` |
|
| 243 |
+
| Confidence semantics of padded steps | **[X]**: head-on-zeros ≈ blank conf 0.78; caller truncates | §1.2(3) |
|
| 244 |
+
| vocab[token i+1] ↔ blank 0 | **[V]** | `vocab.py:3,207,241–242` |
|
| 245 |
+
| rec input normalization | **[V]**: ink map only (0 paper, 1 ink) per `crop_line`, no mean/std | `rec_preproc.py:1–10` |
|
| 246 |
+
| enables fine-tune? | **[V] yes for rec only** (weights exist; no training code; CTC contract reconstructed) | §3, `§B` doc |
|
| 247 |
+
| full pipeline runs on Nepali test image | **[V] smoke**: synthetic shaped page, 10/10 lines, CER 2.6–3.5% | `results/smoke/` |
|
| 248 |
+
| what differs in real (non-synth) Nepali | **[B]** frozen remote sets + baseline | Experiment A |
|
| 249 |
+
| trained det/order weights for fine-tune | **[V] not shipped** (ONNX only) | §3 |
|
| 250 |
+
|
| 251 |
+
## 7. Open items (not closed by this investigation)
|
| 252 |
+
|
| 253 |
+
1. **[B]** Zero-shot numbers on the frozen Nepali eval sets (needs `khalti-t4`).
|
| 254 |
+
2. **[B]** Head-to-head vs PP-OCRv5 Mobile Devanagari *on the same crops* — the
|
| 255 |
+
remote machine holds the paddle recognizer + eval harness. The local
|
| 256 |
+
smoke CER of 2.6–3.5% is on a synthetic page with a 32 px font and is
|
| 257 |
+
*not* a benchmark.
|
| 258 |
+
3. **[B]** Licensing matrix final values need `docs/DATA_LICENSING.md` and
|
| 259 |
+
`docs/DEVANAGARI_RESOURCES_INVENTORY.md` (remote).
|
| 260 |
+
4. FX-detector quality on Devanagari (qubee endpoints, aggressively slanted
|
| 261 |
+
conjuncts) is unknowable without the det model being exercised on real
|
| 262 |
+
scans — that's part of Experiment A full-pipeline.
|
| 263 |
+
5. The *root* training repo (referenced by name in the closed code) is not
|
| 264 |
+
accessible; Archived recipe unknowns: batch sizes, LR schedule, data mix.
|
| 265 |
+
|
| 266 |
+
## 8. Reproduce everything
|
| 267 |
+
|
| 268 |
+
```bash
|
| 269 |
+
cd /mnt/c/Users/Dell/Desktop/OCR_INDRA # or wherever the repo is
|
| 270 |
+
PY=/tmp/opencode/indra-lib # pip --target deps
|
| 271 |
+
export PYTHONPATH="$PWD/indra:$PY"
|
| 272 |
+
|
| 273 |
+
python3 tools/indra_eval/inspect_vocab.py # vocab + Devanagari coverage
|
| 274 |
+
python3 tools/indra_eval/inspect_models.py # graph I/O + param count
|
| 275 |
+
python3 tools/indra_eval/inspect_rec_nodes.py # full node dump + smoke run
|
| 276 |
+
python3 tools/indra_eval/inspect_rec_mask.py # mask semantics + lengths table
|
| 277 |
+
|
| 278 |
+
python3 tools/indra_eval/make_synth_page.py results/smoke # synth page + GT
|
| 279 |
+
python3 tools/indra_eval/eval_indra.py predict results/smoke/synth_nepali_page.png \
|
| 280 |
+
--boxes results/smoke/synth_nepali_gt.jsonl --jsonl results/smoke/rec_only.jsonl
|
| 281 |
+
python3 tools/indra_eval/eval_indra.py predict results/smoke/synth_nepali_page.png \
|
| 282 |
+
--jsonl results/smoke/full_run.jsonl --print-text
|
| 283 |
+
python3 tools/indra_eval/eval_indra.py cer results/smoke/synth_nepali_record.jsonl \
|
| 284 |
+
results/smoke/rec_only.jsonl --detail results/smoke/cer_rec_only.json --rec-only
|
| 285 |
+
python3 tools/indra_eval/eval_indra.py cer results/smoke/synth_nepali_record.jsonl \
|
| 286 |
+
results/smoke/full_run.jsonl --detail results/smoke/cer_full.json
|
| 287 |
+
```
|
| 288 |
+
|
| 289 |
+
Observed (this machine, WSL, CPU, 8 cores, 3 GB RAM): rec-only CER 3.49% /
|
| 290 |
+
WER 15.79%; full-pipeline CER 2.62% / WER 13.16%; 10/10 lines found;
|
| 291 |
+
mean_conf 0.962–0.963. Timings on this box are meaningless (det ≈ 18.6 min
|
| 292 |
+
— machine-starved; bench on the T4).
|
OCR_INDRA/docs/REMOTE_RUN_SHEET.md
ADDED
|
@@ -0,0 +1,181 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
# Remote run sheet — khalti-t4 (202.51.0.7)
|
| 2 |
+
|
| 3 |
+
Companion to `docs/INDRA_EXPERIMENTS.md` (the plan) and
|
| 4 |
+
`docs/INDRA_INVESTIGATION.md` (the evidence). This sheet is the *operational*
|
| 5 |
+
sequence: copy code+weights IN (never data OUT), run A, run B, decide.
|
| 6 |
+
|
| 7 |
+
**Status 2026-10-20:** SSH to `khalti-t4` is still rejected for key
|
| 8 |
+
`khalti-t4-ocr-team` (`Permission denied (publickey,password)`, port 22 open) —
|
| 9 |
+
authorize the key's public half on the box or hand the bundle to someone with
|
| 10 |
+
access. Bundle + toolchain themselves are verified offline (section 0).
|
| 11 |
+
|
| 12 |
+
Ground rules (from §17 of the plan): never modify anything in `~/ocr/` except
|
| 13 |
+
new files under `~/ocr/OCR_INDRA/`; only results files travel back; synthetic
|
| 14 |
+
pages from `make_synth_page.py` are never used for training or reporting.
|
| 15 |
+
|
| 16 |
+
---
|
| 17 |
+
|
| 18 |
+
## 0. Transfer + setup (one-time)
|
| 19 |
+
|
| 20 |
+
```bash
|
| 21 |
+
# local
|
| 22 |
+
bash tools/exp/make_bundle.sh # -> indra_nepali_exp.bundle (~50 MB)
|
| 23 |
+
scp indra_nepali_exp.bundle user@khalti-t4:~/
|
| 24 |
+
|
| 25 |
+
# remote
|
| 26 |
+
git clone ~/indra_nepali_exp.bundle ~/ocr/OCR_INDRA
|
| 27 |
+
cd ~/ocr/OCR_INDRA
|
| 28 |
+
bash tools/exp/remote_setup.sh # deps + env probe -> results/probe.json
|
| 29 |
+
```
|
| 30 |
+
|
| 31 |
+
**Probe 0 — send back / record:** `results/probe.json` (torch+cuda, ORT version,
|
| 32 |
+
RAM, cores) + `nvidia-smi`. Two facts decide variants: CUDA availability
|
| 33 |
+
(fp16 AMP vs fp32 training bands) and ORT version parity (exactness gate
|
| 34 |
+
tolerance 1e-3 assumes same-version ORT locally/remotely).
|
| 35 |
+
|
| 36 |
+
Sanity gate (no data needed) — one command, runs the whole offline chain
|
| 37 |
+
(param invariant → export model.safetensors to ONNX → exactness vs the
|
| 38 |
+
shipped rec.onnx → rec-lines smoke CER re-check):
|
| 39 |
+
|
| 40 |
+
```bash
|
| 41 |
+
bash tools/exp/verify_bundle.sh
|
| 42 |
+
# PASS = results/verify/summary.json has "pass": true.
|
| 43 |
+
# Exactness expectations (measured locally 2026-10-20, torch 2.14.1+cpu,
|
| 44 |
+
# onnxruntime 1.31.0): id_mismatches 0, lengths_mismatches 0,
|
| 45 |
+
# max|Δconf| ~7e-6 (gate is 1e-3); smoke CER micro == results/smoke 0.0349.
|
| 46 |
+
# Any FAIL = wrong init / broken export — STOP, do not run A/B.
|
| 47 |
+
```
|
| 48 |
+
|
| 49 |
+
Notes:
|
| 50 |
+
* `export_rec_onnx.py` forces the legacy TorchScript exporter (`dynamo=False`):
|
| 51 |
+
torch ≥ 2.6 defaults to the new torch.export exporter, which requires the
|
| 52 |
+
`onnxscript` package and changes the dynamic-shape API. The legacy path is
|
| 53 |
+
the verified one; older torch without the `dynamo` kwarg falls back automatically.
|
| 54 |
+
* `remote_setup.sh` writes `results/probe.json` regardless of the cwd it is run from
|
| 55 |
+
and then runs the same chain; stop if it fails.
|
| 56 |
+
|
| 57 |
+
## 1. Inventory probes (answer before running adapters)
|
| 58 |
+
|
| 59 |
+
These are the five unknowns the adapters in `tools/exp/` have plugs for.
|
| 60 |
+
Record everything; do NOT guess formats into the code.
|
| 61 |
+
|
| 62 |
+
- **Probe 1 — frozen eval sets**
|
| 63 |
+
```bash
|
| 64 |
+
ls ~/ocr/ocr-eval-20260924*/ | head -40
|
| 65 |
+
head -3 ~/ocr/ocr-eval-20260924*/<any .jsonl> # GT schema?
|
| 66 |
+
cat ~/ocr/ocr-eval-20260924*/docs/DATA_LICENSING.md 2>/dev/null | head -30
|
| 67 |
+
```
|
| 68 |
+
- **Probe 2 — deployed pipeline surface**
|
| 69 |
+
```bash
|
| 70 |
+
ls ~/ocr/ocr-deva-ft/ ~/ocr/ocr-deva-ft/tools 2>/dev/null
|
| 71 |
+
head -60 ~/ocr/ocr-deva-ft/tools/eval_metrics.py 2>/dev/null # pairing logic to REUSE
|
| 72 |
+
```
|
| 73 |
+
- **Probe 3 — deployed recognizer invocation** (baseline wiring, `_load_engine`
|
| 74 |
+
in `tools/exp/a_ppocr_baseline.py`)
|
| 75 |
+
```bash
|
| 76 |
+
head -80 ~/ocr/ocr-deva-ft/tools/deva_rec_torch/*( 2>/dev/null || \
|
| 77 |
+
ls ~/ocr/ocr-deva-ft/tools/deva_rec_torch 2>/dev/null
|
| 78 |
+
ls ~/ocr/ocr-deva-ft/models 2>/dev/null
|
| 79 |
+
```
|
| 80 |
+
- **Probe 4 — detector output format** (for `a_export_gt_boxes.py`)
|
| 81 |
+
Any existing det-results file from the deployed pipeline run on eval pages —
|
| 82 |
+
else run its det once on one page and show the output.
|
| 83 |
+
- **Probe 5 — training corpus layout**
|
| 84 |
+
```bash
|
| 85 |
+
ls ~/ocr/ocr-deva-ft/data | head -20
|
| 86 |
+
head -3 ~/ocr/ocr-deva-ft/data/<manifest/transcript .jsonl/txt> 2>/dev/null
|
| 87 |
+
du -sh ~/ocr/ocr-deva-ft/data
|
| 88 |
+
```
|
| 89 |
+
|
| 90 |
+
## 2. Experiment A — zero-shot Indra + same-crop baseline
|
| 91 |
+
|
| 92 |
+
```bash
|
| 93 |
+
export PYTHONPATH=.deps:$PWD/indra:$PWD/tools
|
| 94 |
+
SET=~/ocr/ocr-eval-20260924_set1 # repeat per set
|
| 95 |
+
|
| 96 |
+
# 2.1 GT boxes jsonl in eval schema (validate → fix → merge, per probe 1/4)
|
| 97 |
+
python3 tools/exp/a_export_gt_boxes.py validate "$SET/boxes_raw.jsonl"
|
| 98 |
+
python3 tools/exp/a_export_gt_boxes.py normalize "$SET/boxes_raw.jsonl" --out "$SET/boxes.eval.jsonl"
|
| 99 |
+
python3 tools/exp/a_export_gt_boxes.py merge "$SET/boxes.eval.jsonl" \
|
| 100 |
+
--text "$SET/texts.jsonl" --out "$SET/gt_boxes.jsonl"
|
| 101 |
+
|
| 102 |
+
# 2.2 Indra recognizer-only on the SET's boxes (one process per set)
|
| 103 |
+
python3 tools/exp/run_rec.py \
|
| 104 |
+
--onnx models/indra-ocr-v3.1.1/rec.onnx \
|
| 105 |
+
--boxes "$SET/gt_boxes.jsonl" \
|
| 106 |
+
--out results/a/indra_zero/rec_lines.jsonl
|
| 107 |
+
|
| 108 |
+
# 2.3 identical crops for the baseline (same crop geometry as the recognizer)
|
| 109 |
+
python3 tools/exp/a_ppocr_baseline.py crops \
|
| 110 |
+
--boxes "$SET/gt_boxes.jsonl" --out-dir results/a/crops_set1
|
| 111 |
+
|
| 112 |
+
# 2.4 deployed baseline on those crops ← PLUG fills after probe 3
|
| 113 |
+
python3 tools/exp/a_ppocr_baseline.py baseline \
|
| 114 |
+
--manifest results/a/crops_set1/crops_manifest.jsonl \
|
| 115 |
+
--out results/a/ppocrv5/rec_lines.jsonl
|
| 116 |
+
|
| 117 |
+
# 2.5 full-page Indra (det + orient + rec + order) — per page
|
| 118 |
+
python3 tools/indra_eval/eval_indra.py predict "$SET/<page>.png" \
|
| 119 |
+
--jsonl results/a/indra_zero/full/<page>.jsonl
|
| 120 |
+
|
| 121 |
+
# 2.6 score everything (GT records jsonl, one record per page, same pairing)
|
| 122 |
+
python3 tools/indra_eval/eval_indra.py cer "$SET/gt_records.jsonl" \
|
| 123 |
+
results/a/indra_zero/rec_lines.jsonl --detail results/a/cer_indra_zero.json --rec-only
|
| 124 |
+
python3 tools/indra_eval/eval_indra.py cer "$SET/gt_records.jsonl" \
|
| 125 |
+
results/a/ppocrv5/rec_lines.jsonl --detail results/a/cer_ppocrv5.json
|
| 126 |
+
python3 tools/indra_eval/eval_indra.py cer "$SET/gt_records.jsonl" \
|
| 127 |
+
results/a/indra_zero/full.jsonl --detail results/a/cer_indra_full.json
|
| 128 |
+
```
|
| 129 |
+
|
| 130 |
+
Record per set: micro/macro CER, WER, conf histograms, det recall (full-run
|
| 131 |
+
lines found / GT lines), dropped-line count, and the error-typing hand-sort
|
| 132 |
+
(distribution-shaped vs contextual — this decides whether B will help).
|
| 133 |
+
|
| 134 |
+
## 3. Experiment B — recognize fine-tuning (needs probe 5)
|
| 135 |
+
|
| 136 |
+
```bash
|
| 137 |
+
# 3.1 build manifests from the corpus — subcommands adapt to the layout:
|
| 138 |
+
# line-level crops + .txt transcripts -> data_prep.py linepairs
|
| 139 |
+
# pages + boxes jsonl with text -> data_prep.py pages
|
| 140 |
+
# leakage-safe split by PAGE hash + deployed-train exclusion (§12)
|
| 141 |
+
python3 tools/exp/data_prep.py linepairs --dir ~/ocr/ocr-deva-ft/data/lines \
|
| 142 |
+
--out data/manifest_all.jsonl # (or: pages --input ... --out ...)
|
| 143 |
+
python3 tools/exp/data_prep.py split --manifest data/manifest_all.jsonl \
|
| 144 |
+
--train-out data/train_manifest.jsonl --val-out data/val_manifest.jsonl
|
| 145 |
+
# after listing deployed-train pages into deployed_train_pages.txt:
|
| 146 |
+
python3 tools/exp/data_prep.py exclude --manifest data/val_manifest.jsonl \
|
| 147 |
+
--deployed-pages deployed_train_pages.txt --out data/val_manifest.jsonl
|
| 148 |
+
|
| 149 |
+
# 3.2 train (fp16 autocast on T4; a few hours ceiling for 10–20k steps)
|
| 150 |
+
PYTHONPATH=$PWD/indra:$PWD/tools python3 tools/exp/train_rec.py \
|
| 151 |
+
--train data/train_manifest.jsonl --val data/val_manifest.jsonl \
|
| 152 |
+
--data-root ~/ocr/ocr-deva-ft/data \
|
| 153 |
+
--out runs/rec_ft --steps 12000 --batch 32
|
| 154 |
+
|
| 155 |
+
# 3.3 export + gate
|
| 156 |
+
python3 tools/exp/export_rec_onnx.py --weights runs/rec_ft/best.pt \
|
| 157 |
+
--out runs/rec_ft/rec_ft.onnx --check
|
| 158 |
+
PYTHONPATH=$PWD/indra:$PWD/tools python3 tools/exp/exactness_check.py \
|
| 159 |
+
--onnx runs/rec_ft/rec_ft.onnx --against torch --ref-torch runs/rec_ft/best.pt
|
| 160 |
+
|
| 161 |
+
# 3.4 re-run A with the fine-tuned recognizer (rec-lines, same boxes)
|
| 162 |
+
python3 tools/exp/run_rec.py --onnx runs/rec_ft/rec_ft.onnx \
|
| 163 |
+
--boxes "$SET/gt_boxes.jsonl" --out results/b/indra_ft/rec_lines.jsonl
|
| 164 |
+
python3 tools/indra_eval/eval_indra.py cer "$SET/gt_records.jsonl" \
|
| 165 |
+
results/b/indra_ft/rec_lines.jsonl --detail results/b/cer_indra_ft.json --rec-only
|
| 166 |
+
|
| 167 |
+
# 3.5 (optional) full-pipeline with swapped rec:
|
| 168 |
+
cp -r models/indra-ocr-v3.1.1 models/indra-ft-swap
|
| 169 |
+
cp runs/rec_ft/rec_ft.onnx models/indra-ft-swap/rec.onnx
|
| 170 |
+
export INDRA_OCR_MODEL_DIR=$PWD/models/indra-ft-swap # then eval_indra predict
|
| 171 |
+
```
|
| 172 |
+
|
| 173 |
+
**B acceptance (unchanged from the plan):** fine-tuned rec-lines CER <
|
| 174 |
+
zero-shot rec-lines CER on all 3 sets by ≥2 points relative, with the
|
| 175 |
+
digit-heavy and conjunct-heavy slices specifically improving (they were the
|
| 176 |
+
smoke-run weak spots: `सङ्केत→सड्लेत`, `₹→र`, `ऋतु→तऋहतु`, ऱ nukta).
|
| 177 |
+
|
| 178 |
+
## 4. What travels back
|
| 179 |
+
|
| 180 |
+
Only `results/` (JSON/JSONL summaries + per-line detail), `runs/rec_ft/best.pt`
|
| 181 |
+
(+ safetensors) and `results/probe.json`. Everything else stays on the box.
|
OCR_INDRA/indra/.gitignore
ADDED
|
@@ -0,0 +1,7 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
__pycache__/
|
| 2 |
+
*.pyc
|
| 3 |
+
*.egg-info/
|
| 4 |
+
.venv/
|
| 5 |
+
dist/
|
| 6 |
+
build/
|
| 7 |
+
uv.lock
|
OCR_INDRA/indra/CHANGELOG.md
ADDED
|
@@ -0,0 +1,97 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
# Changelog
|
| 2 |
+
|
| 3 |
+
## 0.3.2
|
| 4 |
+
|
| 5 |
+
- Models from Hugging Face tag `v3.1.1`: the same models as `v3.1`, plus
|
| 6 |
+
`config.json` (the file the Hub counts downloads by) and `model.safetensors`
|
| 7 |
+
(the recognizer's weights, for the Hub's parameter count; never downloaded).
|
| 8 |
+
- A cached model folder is checked against the Hub at most once a day
|
| 9 |
+
(`CHECK_EVERY_S`); the other runs use the cache with no network, and a
|
| 10 |
+
failed check (offline) keeps the cached models.
|
| 11 |
+
|
| 12 |
+
## 0.3.1
|
| 13 |
+
|
| 14 |
+
- A model download that fails (no network, no access) or a model folder with
|
| 15 |
+
missing files raises `indra_ocr.ModelError` with one clear message; the CLI
|
| 16 |
+
and the viewer print it as one line and exit with 1.
|
| 17 |
+
- `--version` for `indra-ocr` and `indra-ocr-viewer`.
|
| 18 |
+
- `page.skew` is rounded to 0.01 degree.
|
| 19 |
+
- Project links on PyPI, and absolute image URLs in the PyPI description, so
|
| 20 |
+
its images show there.
|
| 21 |
+
- README: install from PyPI (`pip install indra-ocr`); a Limitations section
|
| 22 |
+
that leads with complex tables, handwriting and scripts outside the OCR list.
|
| 23 |
+
|
| 24 |
+
## 0.3.0
|
| 25 |
+
|
| 26 |
+
Models v3 (Hugging Face `datalab-to/indra-ocr`, tag `v3`; `v3.1` adds the math models).
|
| 27 |
+
|
| 28 |
+
- **Viewer.** A Datalab-styled page with the pipeline options in the page
|
| 29 |
+
(source, math, headers and footers, reading order, OCR inside images, dpi,
|
| 30 |
+
page limit), show/hide for lines, blocks, block labels and reading order, and
|
| 31 |
+
tabs for the rendered markdown, its source, the lines and the JSON.
|
| 32 |
+
- `convert_many` no longer hangs when its worker processes cannot start (with
|
| 33 |
+
the spawn start method, code not under `if __name__ == "__main__":`, or code
|
| 34 |
+
from stdin): it warns and reads the remaining inputs in the calling process.
|
| 35 |
+
- **Math (opt-in).** `--math` / `load(math=True)`: a line classifier
|
| 36 |
+
(`mathcls.onnx`) and a math recognizer (`math_enc` / `math_dec_init` /
|
| 37 |
+
`math_dec_step.onnx`, attention decoder with cached keys and values) write
|
| 38 |
+
lines with math as text with inline LaTeX. A reading replaces a line only
|
| 39 |
+
with math in it and a mean token probability of 0.9 or more. Omni Parse
|
| 40 |
+
Bench: headline 62.1 -> 66.8, text 41.4 -> 57.5, math tests 14.3 -> 56.5,
|
| 41 |
+
tables 56.4 -> 54.8; median page ~210 -> ~360 ms (x86, 4 cores, auto mode).
|
| 42 |
+
Off by default; the math models download only when it is on.
|
| 43 |
+
|
| 44 |
+
- **Orientation.** `orient.onnx` turns a page that is turned by a quarter or
|
| 45 |
+
half turn upright before detection (`page.rotation`). A turn needs at least
|
| 46 |
+
95% from the model. Omni Parse Bench: rotated content 3.5% -> 24.3%.
|
| 47 |
+
- **All cores.** The CLI and the new `indra_ocr.convert_many` run one
|
| 48 |
+
single-threaded worker process per usable core, one page per task (a single
|
| 49 |
+
PDF too); `--cores` limits them. 40 documents: 30.0 s on 1 core, 8.7 s on 4,
|
| 50 |
+
2.8 s on 16 (one process with ORT threads gained at most 1.2x beyond 4
|
| 51 |
+
cores). The CLI's `--threads` is now an alias of `--cores`.
|
| 52 |
+
- **Deskew.** A page tilted by up to 5 degrees is straightened before
|
| 53 |
+
detection (`page.skew`, about 8 ms a page); boxes are mapped back to the
|
| 54 |
+
page as turned. Omni Parse Bench rotation_skew suite: 60.0 -> 76.4.
|
| 55 |
+
- **Low-confidence lines get no text.** A recognized line under 0.5 mean
|
| 56 |
+
confidence (empty form fields, checkboxes, rules read as "-", "I", "1")
|
| 57 |
+
keeps its box but no text.
|
| 58 |
+
- **Orientation from a broken text layer.** A PDF page OCR'd for a broken font
|
| 59 |
+
map is not turned when its glyphs run upright (orient.onnx turns about half
|
| 60 |
+
of upright Arabic and Hebrew pages).
|
| 61 |
+
- **CJK markdown.** Lines of Chinese or Japanese join without a space; a
|
| 62 |
+
bullet detected as its own line starts a list item.
|
| 63 |
+
- Table cells escape only `&`, `<` and `>` (no `'`).
|
| 64 |
+
- **Text inside diagrams and charts.** The detector finds 51% of the text
|
| 65 |
+
inside pictures (12.5% before).
|
| 66 |
+
- **Right-to-left text layers.** Runs in a right-to-left line are ordered right
|
| 67 |
+
to left.
|
| 68 |
+
- **OCR only where it can read.** A text layer in a script the recognizer
|
| 69 |
+
cannot read is kept even when a trust check fails (not for broken font
|
| 70 |
+
maps), and empty lines on such pages are not OCR'd into junk.
|
| 71 |
+
- **Layout blocks.** `page.blocks` lists the layout blocks (label, box, text)
|
| 72 |
+
in reading order; the CLI's JSON output includes them.
|
| 73 |
+
- **`--keep-furniture`** keeps page headers and footers in the markdown.
|
| 74 |
+
- **Optional models:** `script.onnx` + `rec_cjk.onnx` route lines with Chinese
|
| 75 |
+
characters to a Chinese recognizer; `mask.onnx` (CenterMask) masks
|
| 76 |
+
neighbouring lines before recognition. Both run only when present.
|
| 77 |
+
- `bench/opb_indra.py`: Omni Parse Bench runner.
|
| 78 |
+
- **Robust inputs** (release test: 4,340 benchmark pages, 1,177 documents with
|
| 79 |
+
16,017 pages, 37 edge cases): the file type comes from the file's bytes;
|
| 80 |
+
multi-page TIFFs give every page; pages above 100 MP render at a lower dpi;
|
| 81 |
+
unreadable, corrupt or password-protected files raise `InputError`
|
| 82 |
+
(`password=` / `--password`); a page that fails gives an empty page with
|
| 83 |
+
`source == "error"` instead of ending the document; the CLI continues past
|
| 84 |
+
failed files and exits 1. Text outside a PDF's crop box is not read.
|
| 85 |
+
- **Faster on dense pages:** reading-order clustering is O(n^2) instead of
|
| 86 |
+
O(n^3) (a 1,984-line page: 53 s -> 0.45 s).
|
| 87 |
+
- **Bounded memory:** ONNX Runtime's CPU arena is off (it grew without bound
|
| 88 |
+
on changing batch shapes; a 40-page large scan: 4.9 GB -> 1.4 GB).
|
| 89 |
+
|
| 90 |
+
## 0.2.0
|
| 91 |
+
|
| 92 |
+
Models v2. Blocks and table heads from average-linkage
|
| 93 |
+
clustering.
|
| 94 |
+
|
| 95 |
+
## 0.1.0
|
| 96 |
+
|
| 97 |
+
First package: ONNX pipeline, PDF text-layer mode, CLI, viewer.
|
OCR_INDRA/indra/EXAMPLES.md
ADDED
|
@@ -0,0 +1,84 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
# Indra OCR examples
|
| 2 |
+
|
| 3 |
+
Real Indra OCR outputs on [OmniParseBench](https://github.com/datalab-to/omni_parse_bench) pages, from the v3 models with default settings (`--mode auto`, page headers and footers kept). The boxes are drawn from the JSON output: colored boxes are layout blocks with their label, faint blue boxes are lines. The outputs are not edited. The markdown excerpts are cut at "...". Back to the [README](README.md).
|
| 4 |
+
|
| 5 |
+
## Scanned upside down
|
| 6 |
+
|
| 7 |
+
<img src="assets/examples/rotated_letter.png" width="600px"/>
|
| 8 |
+
|
| 9 |
+
A two-column annual report page, scanned upside down. Indra OCR turns the page upright (`rotation: 180`), then reads the columns in order and finds the headings. Source: OCR. OmniParseBench page `rotation_skew-rse11a9ca2_rot180-p1`.
|
| 10 |
+
|
| 11 |
+
```markdown
|
| 12 |
+
...
|
| 13 |
+
|
| 14 |
+
In 2021, we took what we had learned from both programs to launch our latest ini ...
|
| 15 |
+
|
| 16 |
+
## Sustainable Finance
|
| 17 |
+
|
| 18 |
+
Another area where we've long been focused is sustainability. We have been a lea ...
|
| 19 |
+
```
|
| 20 |
+
|
| 21 |
+
## Table
|
| 22 |
+
|
| 23 |
+
<img src="assets/examples/table_pmc.png" width="600px"/>
|
| 24 |
+
|
| 25 |
+
A journal page from a digital PDF with two tables. Indra OCR reads the text layer and outputs each table as HTML, with its caption. Source: PDF text layer. OmniParseBench page `tables_derived-PMC2564913.1_doc-p4`.
|
| 26 |
+
|
| 27 |
+
```markdown
|
| 28 |
+
...
|
| 29 |
+
|
| 30 |
+
Table 1: Background data for the 75-year old study participants (n = 154).
|
| 31 |
+
|
| 32 |
+
<table>
|
| 33 |
+
<tr><th></th><th>Atrial fibrillation</th><th>No atrial fibriallation</th><th>Chi-square/df/p or t/d.f/p</th></tr>
|
| 34 |
+
<tr><td>Gender (male/female)</td><td>34 (31.8%)/12 (25.5%)</td><td>73 (68.2%)/35 (74.5%)</td><td>0.608/1/0.44</td></tr>
|
| 35 |
+
<tr><td>BMI (kg/m2)</td><td>25.3 ± 3.7</td><td>25.4 ± 3.2</td><td>-0.176/145/0.86</td></tr>
|
| 36 |
+
<tr><td>Waist (cm)</td><td>93.5 ± 11.7</td><td>91.6 ± 10.9</td><td>0.936/151/0.35</td></tr>
|
| 37 |
+
<tr><td>SBP (mmHg)</td><td>143 ± 21</td><td>147 ± 16</td><td>-1.169/152/0.24</td></tr>
|
| 38 |
+
<tr><td>DBP (mmHg)</td><td>82 ± 10</td><td>81 ± 9</td><td>1.069/152/0.29</td></tr>
|
| 39 |
+
...
|
| 40 |
+
```
|
| 41 |
+
|
| 42 |
+
## Two columns and headings
|
| 43 |
+
|
| 44 |
+
<img src="assets/examples/multi_column.png" width="600px"/>
|
| 45 |
+
|
| 46 |
+
A two-column journal page from a digital PDF. Indra OCR reads the text layer, orders the columns, and finds the section headings. Source: PDF text layer. OmniParseBench page `olmo_multi_column-0845863d27c84b14011e17a388821525e783_page_4_pg1-p1`.
|
| 47 |
+
|
| 48 |
+
```markdown
|
| 49 |
+
sand content and maturity of turbidite deposits. Such predictions requ ...
|
| 50 |
+
|
| 51 |
+
## 2. Geological setting
|
| 52 |
+
|
| 53 |
+
The Precambrian Borborema Province in northeastern Brazil (Fig. 1) con ...
|
| 54 |
+
|
| 55 |
+
The fragmentation of Gondwana in the Mesozoic is recorded in northeast ...
|
| 56 |
+
|
| 57 |
+
The structuring of the Eastern and Equatorial Margins started with int ...
|
| 58 |
+
|
| 59 |
+
vicinity of the Borborema Plateau. This trend encompasses small Neocom ...
|
| 60 |
+
|
| 61 |
+
...
|
| 62 |
+
```
|
| 63 |
+
|
| 64 |
+
## Chinese
|
| 65 |
+
|
| 66 |
+
<img src="assets/examples/chinese_essay.png" width="600px"/>
|
| 67 |
+
|
| 68 |
+
A typeset Chinese essay, read by OCR with the Chinese recognizer, with its headings. Source: OCR. OmniParseBench page `multiling_finepdfs-images_cmn_Hani_54743253-p1`.
|
| 69 |
+
|
| 70 |
+
```markdown
|
| 71 |
+
## CVPR 之感想
|
| 72 |
+
|
| 73 |
+
田渊栋我的主页
|
| 74 |
+
|
| 75 |
+
## 章零 引子
|
| 76 |
+
|
| 77 |
+
结束了,终于结束了,这该死的 CVPR。不得不承认,写 paper 是一件相当麻烦的事情, ...
|
| 78 |
+
|
| 79 |
+
## 章一 黑夜中,我见到了黎明
|
| 80 |
+
|
| 81 |
+
整整一个月前,11 月 6 日,周一。
|
| 82 |
+
|
| 83 |
+
...
|
| 84 |
+
```
|
OCR_INDRA/indra/LICENSE
ADDED
|
@@ -0,0 +1,202 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
|
| 2 |
+
Apache License
|
| 3 |
+
Version 2.0, January 2004
|
| 4 |
+
http://www.apache.org/licenses/
|
| 5 |
+
|
| 6 |
+
TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION
|
| 7 |
+
|
| 8 |
+
1. Definitions.
|
| 9 |
+
|
| 10 |
+
"License" shall mean the terms and conditions for use, reproduction,
|
| 11 |
+
and distribution as defined by Sections 1 through 9 of this document.
|
| 12 |
+
|
| 13 |
+
"Licensor" shall mean the copyright owner or entity authorized by
|
| 14 |
+
the copyright owner that is granting the License.
|
| 15 |
+
|
| 16 |
+
"Legal Entity" shall mean the union of the acting entity and all
|
| 17 |
+
other entities that control, are controlled by, or are under common
|
| 18 |
+
control with that entity. For the purposes of this definition,
|
| 19 |
+
"control" means (i) the power, direct or indirect, to cause the
|
| 20 |
+
direction or management of such entity, whether by contract or
|
| 21 |
+
otherwise, or (ii) ownership of fifty percent (50%) or more of the
|
| 22 |
+
outstanding shares, or (iii) beneficial ownership of such entity.
|
| 23 |
+
|
| 24 |
+
"You" (or "Your") shall mean an individual or Legal Entity
|
| 25 |
+
exercising permissions granted by this License.
|
| 26 |
+
|
| 27 |
+
"Source" form shall mean the preferred form for making modifications,
|
| 28 |
+
including but not limited to software source code, documentation
|
| 29 |
+
source, and configuration files.
|
| 30 |
+
|
| 31 |
+
"Object" form shall mean any form resulting from mechanical
|
| 32 |
+
transformation or translation of a Source form, including but
|
| 33 |
+
not limited to compiled object code, generated documentation,
|
| 34 |
+
and conversions to other media types.
|
| 35 |
+
|
| 36 |
+
"Work" shall mean the work of authorship, whether in Source or
|
| 37 |
+
Object form, made available under the License, as indicated by a
|
| 38 |
+
copyright notice that is included in or attached to the work
|
| 39 |
+
(an example is provided in the Appendix below).
|
| 40 |
+
|
| 41 |
+
"Derivative Works" shall mean any work, whether in Source or Object
|
| 42 |
+
form, that is based on (or derived from) the Work and for which the
|
| 43 |
+
editorial revisions, annotations, elaborations, or other modifications
|
| 44 |
+
represent, as a whole, an original work of authorship. For the purposes
|
| 45 |
+
of this License, Derivative Works shall not include works that remain
|
| 46 |
+
separable from, or merely link (or bind by name) to the interfaces of,
|
| 47 |
+
the Work and Derivative Works thereof.
|
| 48 |
+
|
| 49 |
+
"Contribution" shall mean any work of authorship, including
|
| 50 |
+
the original version of the Work and any modifications or additions
|
| 51 |
+
to that Work or Derivative Works thereof, that is intentionally
|
| 52 |
+
submitted to Licensor for inclusion in the Work by the copyright owner
|
| 53 |
+
or by an individual or Legal Entity authorized to submit on behalf of
|
| 54 |
+
the copyright owner. For the purposes of this definition, "submitted"
|
| 55 |
+
means any form of electronic, verbal, or written communication sent
|
| 56 |
+
to the Licensor or its representatives, including but not limited to
|
| 57 |
+
communication on electronic mailing lists, source code control systems,
|
| 58 |
+
and issue tracking systems that are managed by, or on behalf of, the
|
| 59 |
+
Licensor for the purpose of discussing and improving the Work, but
|
| 60 |
+
excluding communication that is conspicuously marked or otherwise
|
| 61 |
+
designated in writing by the copyright owner as "Not a Contribution."
|
| 62 |
+
|
| 63 |
+
"Contributor" shall mean Licensor and any individual or Legal Entity
|
| 64 |
+
on behalf of whom a Contribution has been received by Licensor and
|
| 65 |
+
subsequently incorporated within the Work.
|
| 66 |
+
|
| 67 |
+
2. Grant of Copyright License. Subject to the terms and conditions of
|
| 68 |
+
this License, each Contributor hereby grants to You a perpetual,
|
| 69 |
+
worldwide, non-exclusive, no-charge, royalty-free, irrevocable
|
| 70 |
+
copyright license to reproduce, prepare Derivative Works of,
|
| 71 |
+
publicly display, publicly perform, sublicense, and distribute the
|
| 72 |
+
Work and such Derivative Works in Source or Object form.
|
| 73 |
+
|
| 74 |
+
3. Grant of Patent License. Subject to the terms and conditions of
|
| 75 |
+
this License, each Contributor hereby grants to You a perpetual,
|
| 76 |
+
worldwide, non-exclusive, no-charge, royalty-free, irrevocable
|
| 77 |
+
(except as stated in this section) patent license to make, have made,
|
| 78 |
+
use, offer to sell, sell, import, and otherwise transfer the Work,
|
| 79 |
+
where such license applies only to those patent claims licensable
|
| 80 |
+
by such Contributor that are necessarily infringed by their
|
| 81 |
+
Contribution(s) alone or by combination of their Contribution(s)
|
| 82 |
+
with the Work to which such Contribution(s) was submitted. If You
|
| 83 |
+
institute patent litigation against any entity (including a
|
| 84 |
+
cross-claim or counterclaim in a lawsuit) alleging that the Work
|
| 85 |
+
or a Contribution incorporated within the Work constitutes direct
|
| 86 |
+
or contributory patent infringement, then any patent licenses
|
| 87 |
+
granted to You under this License for that Work shall terminate
|
| 88 |
+
as of the date such litigation is filed.
|
| 89 |
+
|
| 90 |
+
4. Redistribution. You may reproduce and distribute copies of the
|
| 91 |
+
Work or Derivative Works thereof in any medium, with or without
|
| 92 |
+
modifications, and in Source or Object form, provided that You
|
| 93 |
+
meet the following conditions:
|
| 94 |
+
|
| 95 |
+
(a) You must give any other recipients of the Work or
|
| 96 |
+
Derivative Works a copy of this License; and
|
| 97 |
+
|
| 98 |
+
(b) You must cause any modified files to carry prominent notices
|
| 99 |
+
stating that You changed the files; and
|
| 100 |
+
|
| 101 |
+
(c) You must retain, in the Source form of any Derivative Works
|
| 102 |
+
that You distribute, all copyright, patent, trademark, and
|
| 103 |
+
attribution notices from the Source form of the Work,
|
| 104 |
+
excluding those notices that do not pertain to any part of
|
| 105 |
+
the Derivative Works; and
|
| 106 |
+
|
| 107 |
+
(d) If the Work includes a "NOTICE" text file as part of its
|
| 108 |
+
distribution, then any Derivative Works that You distribute must
|
| 109 |
+
include a readable copy of the attribution notices contained
|
| 110 |
+
within such NOTICE file, excluding those notices that do not
|
| 111 |
+
pertain to any part of the Derivative Works, in at least one
|
| 112 |
+
of the following places: within a NOTICE text file distributed
|
| 113 |
+
as part of the Derivative Works; within the Source form or
|
| 114 |
+
documentation, if provided along with the Derivative Works; or,
|
| 115 |
+
within a display generated by the Derivative Works, if and
|
| 116 |
+
wherever such third-party notices normally appear. The contents
|
| 117 |
+
of the NOTICE file are for informational purposes only and
|
| 118 |
+
do not modify the License. You may add Your own attribution
|
| 119 |
+
notices within Derivative Works that You distribute, alongside
|
| 120 |
+
or as an addendum to the NOTICE text from the Work, provided
|
| 121 |
+
that such additional attribution notices cannot be construed
|
| 122 |
+
as modifying the License.
|
| 123 |
+
|
| 124 |
+
You may add Your own copyright statement to Your modifications and
|
| 125 |
+
may provide additional or different license terms and conditions
|
| 126 |
+
for use, reproduction, or distribution of Your modifications, or
|
| 127 |
+
for any such Derivative Works as a whole, provided Your use,
|
| 128 |
+
reproduction, and distribution of the Work otherwise complies with
|
| 129 |
+
the conditions stated in this License.
|
| 130 |
+
|
| 131 |
+
5. Submission of Contributions. Unless You explicitly state otherwise,
|
| 132 |
+
any Contribution intentionally submitted for inclusion in the Work
|
| 133 |
+
by You to the Licensor shall be under the terms and conditions of
|
| 134 |
+
this License, without any additional terms or conditions.
|
| 135 |
+
Notwithstanding the above, nothing herein shall supersede or modify
|
| 136 |
+
the terms of any separate license agreement you may have executed
|
| 137 |
+
with Licensor regarding such Contributions.
|
| 138 |
+
|
| 139 |
+
6. Trademarks. This License does not grant permission to use the trade
|
| 140 |
+
names, trademarks, service marks, or product names of the Licensor,
|
| 141 |
+
except as required for reasonable and customary use in describing the
|
| 142 |
+
origin of the Work and reproducing the content of the NOTICE file.
|
| 143 |
+
|
| 144 |
+
7. Disclaimer of Warranty. Unless required by applicable law or
|
| 145 |
+
agreed to in writing, Licensor provides the Work (and each
|
| 146 |
+
Contributor provides its Contributions) on an "AS IS" BASIS,
|
| 147 |
+
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
|
| 148 |
+
implied, including, without limitation, any warranties or conditions
|
| 149 |
+
of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A
|
| 150 |
+
PARTICULAR PURPOSE. You are solely responsible for determining the
|
| 151 |
+
appropriateness of using or redistributing the Work and assume any
|
| 152 |
+
risks associated with Your exercise of permissions under this License.
|
| 153 |
+
|
| 154 |
+
8. Limitation of Liability. In no event and under no legal theory,
|
| 155 |
+
whether in tort (including negligence), contract, or otherwise,
|
| 156 |
+
unless required by applicable law (such as deliberate and grossly
|
| 157 |
+
negligent acts) or agreed to in writing, shall any Contributor be
|
| 158 |
+
liable to You for damages, including any direct, indirect, special,
|
| 159 |
+
incidental, or consequential damages of any character arising as a
|
| 160 |
+
result of this License or out of the use or inability to use the
|
| 161 |
+
Work (including but not limited to damages for loss of goodwill,
|
| 162 |
+
work stoppage, computer failure or malfunction, or any and all
|
| 163 |
+
other commercial damages or losses), even if such Contributor
|
| 164 |
+
has been advised of the possibility of such damages.
|
| 165 |
+
|
| 166 |
+
9. Accepting Warranty or Additional Liability. While redistributing
|
| 167 |
+
the Work or Derivative Works thereof, You may choose to offer,
|
| 168 |
+
and charge a fee for, acceptance of support, warranty, indemnity,
|
| 169 |
+
or other liability obligations and/or rights consistent with this
|
| 170 |
+
License. However, in accepting such obligations, You may act only
|
| 171 |
+
on Your own behalf and on Your sole responsibility, not on behalf
|
| 172 |
+
of any other Contributor, and only if You agree to indemnify,
|
| 173 |
+
defend, and hold each Contributor harmless for any liability
|
| 174 |
+
incurred by, or claims asserted against, such Contributor by reason
|
| 175 |
+
of your accepting any such warranty or additional liability.
|
| 176 |
+
|
| 177 |
+
END OF TERMS AND CONDITIONS
|
| 178 |
+
|
| 179 |
+
APPENDIX: How to apply the Apache License to your work.
|
| 180 |
+
|
| 181 |
+
To apply the Apache License to your work, attach the following
|
| 182 |
+
boilerplate notice, with the fields enclosed by brackets "[]"
|
| 183 |
+
replaced with your own identifying information. (Don't include
|
| 184 |
+
the brackets!) The text should be enclosed in the appropriate
|
| 185 |
+
comment syntax for the file format. We also recommend that a
|
| 186 |
+
file or class name and description of purpose be included on the
|
| 187 |
+
same "printed page" as the copyright notice for easier
|
| 188 |
+
identification within third-party archives.
|
| 189 |
+
|
| 190 |
+
Copyright [yyyy] [name of copyright owner]
|
| 191 |
+
|
| 192 |
+
Licensed under the Apache License, Version 2.0 (the "License");
|
| 193 |
+
you may not use this file except in compliance with the License.
|
| 194 |
+
You may obtain a copy of the License at
|
| 195 |
+
|
| 196 |
+
http://www.apache.org/licenses/LICENSE-2.0
|
| 197 |
+
|
| 198 |
+
Unless required by applicable law or agreed to in writing, software
|
| 199 |
+
distributed under the License is distributed on an "AS IS" BASIS,
|
| 200 |
+
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
| 201 |
+
See the License for the specific language governing permissions and
|
| 202 |
+
limitations under the License.
|
OCR_INDRA/indra/README.md
ADDED
|
@@ -0,0 +1,296 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
<p align="center">
|
| 2 |
+
<img src="assets/datalab-logo.png" alt="Datalab Logo" width="150"/>
|
| 3 |
+
</p>
|
| 4 |
+
<h1 align="center">Datalab</h1>
|
| 5 |
+
<p align="center">
|
| 6 |
+
<strong>State of the Art models for Document Intelligence</strong>
|
| 7 |
+
</p>
|
| 8 |
+
<p align="center">
|
| 9 |
+
<a href="https://opensource.org/licenses/Apache-2.0"><img src="https://img.shields.io/badge/Code%20License-Apache_2.0-green.svg" alt="Code License"></a>
|
| 10 |
+
<a href="https://www.datalab.to/pricing"><img src="https://img.shields.io/badge/Model%20License-OpenRAIL--M-blue.svg" alt="Model License"></a>
|
| 11 |
+
<a href="https://discord.gg/KuZwXNGnfH"><img src="https://img.shields.io/badge/Discord-Join%20us-5865F2?logo=discord&logoColor=white" alt="Discord"></a>
|
| 12 |
+
<a href="https://huggingface.co/datalab-to/indra-ocr"><img src="https://img.shields.io/badge/Models-Hugging%20Face-yellow" alt="Models"></a>
|
| 13 |
+
</p>
|
| 14 |
+
<p align="center">
|
| 15 |
+
<a href="https://www.datalab.to"><img src="https://img.shields.io/badge/Homepage-datalab.to-blue" alt="Homepage"></a>
|
| 16 |
+
<a href="https://documentation.datalab.to"><img src="https://img.shields.io/badge/Docs-Read%20the%20docs-blue" alt="Docs"></a>
|
| 17 |
+
<a href="https://www.datalab.to/playground"><img src="https://img.shields.io/badge/Datalab%20Playground-Try%20it-orange" alt="Datalab Playground"></a>
|
| 18 |
+
</p>
|
| 19 |
+
|
| 20 |
+
<hr/>
|
| 21 |
+
|
| 22 |
+
# Indra OCR
|
| 23 |
+
|
| 24 |
+
Indra OCR is a 1M-parameter OCR model that reads up to 49 pages per second on CPU. It turns PDFs and images into markdown, plain text, or JSON with boxes, and it has these features:
|
| 25 |
+
|
| 26 |
+
- Throughput - 49 pages/s, 11-17x faster than Tesseract and PaddleOCR on the same machine and pages ([table](#throughput))
|
| 27 |
+
- Latency - ~230 ms per page on 4 x86 vCPUs, 12-24x faster than Tesseract and PaddleOCR
|
| 28 |
+
- Accuracy - scores 41.3 on [OmniParseBench](https://github.com/datalab-to/omni_parse_bench) text, ahead of PaddleOCR's PP-OCR text models (35.8) and Tesseract (19.3). Its headline score (62.1) beats PaddleOCR's full PP-StructureV3 pipeline (58.4).
|
| 29 |
+
- Layout and boxes - a bounding box for every line and every layout block, with block labels (headings, lists, tables, captions, footnotes, page headers) and reading order
|
| 30 |
+
- Digital PDFs - reads the PDF's own text layer in any script, and OCRs only what is missing
|
| 31 |
+
- Small - about 18 MB of models on ONNX Runtime, no PyTorch
|
| 32 |
+
|
| 33 |
+
The main recognizer, `rec.onnx`, has about 1M parameters. See [benchmarks](#benchmarks) for the full results and [limitations](#limitations) for what it does not do yet.
|
| 34 |
+
|
| 35 |
+
Indra OCR is named for [Indra](https://en.wikipedia.org/wiki/Indra).
|
| 36 |
+
|
| 37 |
+
<img src="assets/benchmarks/throughput.png" width="700px"/>
|
| 38 |
+
|
| 39 |
+
## Try Datalab's Managed Platform
|
| 40 |
+
|
| 41 |
+
Indra OCR is built for speed on CPUs. When you need the highest accuracy, on handwriting, math, complex tables and degraded scans, our managed platform runs [Chandra](https://github.com/datalab-to/chandra), our state of the art OCR model, with zero data retention by default, SOC 2 Type 2, and custom BAAs.
|
| 42 |
+
|
| 43 |
+
If you have high volume workloads, we offer a batch processing service that can handle 1B+ pages a day.
|
| 44 |
+
|
| 45 |
+
Get started with **$5 in free credits** - [sign up](https://www.datalab.to/?utm_source=gh-indra) - or try Chandra in our [public playground](https://www.datalab.to/playground?utm_source=gh-indra).
|
| 46 |
+
|
| 47 |
+
## Quickstart
|
| 48 |
+
|
| 49 |
+
```shell
|
| 50 |
+
pip install indra-ocr
|
| 51 |
+
|
| 52 |
+
indra-ocr paper.pdf # markdown to stdout
|
| 53 |
+
indra-ocr *.pdf --out results/ # one .md per input, all cores
|
| 54 |
+
indra-ocr scan.png --format json # lines, boxes, layout blocks, confidences
|
| 55 |
+
|
| 56 |
+
indra-ocr-viewer # local web viewer on http://localhost:8765
|
| 57 |
+
```
|
| 58 |
+
|
| 59 |
+
The first run downloads the models (about 18 MB) from [Hugging Face](https://huggingface.co/datalab-to/indra-ocr).
|
| 60 |
+
|
| 61 |
+
## Examples
|
| 62 |
+
|
| 63 |
+
Real Indra OCR outputs on OmniParseBench pages. Colored boxes are layout blocks with their label, faint blue boxes are lines, drawn from the JSON output with no edits.
|
| 64 |
+
|
| 65 |
+
<table>
|
| 66 |
+
<tr>
|
| 67 |
+
<td width="50%"><img src="assets/examples/rotated_letter.png"/><br/><sub>Scanned upside down: Indra OCR turns the page upright, reads both columns in order and finds the headings.</sub></td>
|
| 68 |
+
<td width="50%"><img src="assets/examples/table_pmc.png"/><br/><sub>A journal page with two tables, output as HTML with their captions.</sub></td>
|
| 69 |
+
</tr>
|
| 70 |
+
</table>
|
| 71 |
+
|
| 72 |
+
**[See more examples, with the markdown output for each →](https://github.com/datalab-to/indra/blob/main/EXAMPLES.md)**
|
| 73 |
+
|
| 74 |
+
|
| 75 |
+
# License
|
| 76 |
+
|
| 77 |
+
The code is [Apache 2.0](https://github.com/datalab-to/indra/blob/main/LICENSE). The model weights use a modified OpenRAIL-M license ([LICENSE](https://huggingface.co/datalab-to/indra-ocr/blob/main/LICENSE)).
|
| 78 |
+
|
| 79 |
+
# Commercial usage
|
| 80 |
+
|
| 81 |
+
This code is Apache 2.0, and our model weights use a modified OpenRAIL-M license (free for research, personal use, and startups under $20M funding/revenue, cannot be used competitively with our API). To remove the OpenRAIL license requirements, or for broader commercial licensing, visit our pricing page [here](https://www.datalab.to/pricing?utm_source=gh-indra).
|
| 82 |
+
|
| 83 |
+
# Installation
|
| 84 |
+
|
| 85 |
+
```shell
|
| 86 |
+
pip install indra-ocr
|
| 87 |
+
```
|
| 88 |
+
|
| 89 |
+
For the latest code from GitHub instead: `pip install git+https://github.com/datalab-to/indra.git`. To let the viewer compare with Tesseract, install `pip install "indra-ocr[tesseract]"` and a Tesseract binary.
|
| 90 |
+
|
| 91 |
+
Python 3.10 or newer, tested on Linux (x86_64 and aarch64, Python 3.10 to 3.13) and macOS (Apple silicon). It installs ONNX Runtime, OpenCV and pypdfium2, and no PyTorch. The models download from Hugging Face on the first run and are cached. To use a local copy, set `INDRA_OCR_MODEL_DIR` or pass `--model-dir`.
|
| 92 |
+
|
| 93 |
+
# Usage
|
| 94 |
+
|
| 95 |
+
## CLI
|
| 96 |
+
|
| 97 |
+
```shell
|
| 98 |
+
indra-ocr paper.pdf # markdown to stdout
|
| 99 |
+
indra-ocr *.pdf --out results/ # one .md per input
|
| 100 |
+
indra-ocr scan.png --format json # lines, boxes, layout blocks, confidences, timings
|
| 101 |
+
indra-ocr paper.pdf --format text # plain text, one line per detected line
|
| 102 |
+
indra-ocr paper.pdf --mode ocr --timings # ignore the text layer, OCR every page; timings on stderr
|
| 103 |
+
indra-ocr paper.pdf --keep-furniture # keep page headers and footers in the markdown
|
| 104 |
+
indra-ocr *.pdf --out results/ --cores 8 # limit the cores (default: all usable cores)
|
| 105 |
+
indra-ocr paper.pdf --math # write math as LaTeX (off by default; slower on math pages)
|
| 106 |
+
```
|
| 107 |
+
|
| 108 |
+
**CLI Options:**
|
| 109 |
+
- `--out DIR`: write one file per input (`.md`, `.txt` or `.jsonl`)
|
| 110 |
+
- `--format [md|text|json]`: output format (default: md)
|
| 111 |
+
- `--mode [auto|ocr|text]`: `auto` reads the text layer where it passes the checks and OCRs elsewhere (default). `ocr` never reads the text layer. `text` uses the text layer even when the checks fail
|
| 112 |
+
- `--model-order`: order text-layer lines with the order model, not the PDF content order
|
| 113 |
+
- `--no-image-ocr`: on text-layer pages, do not OCR text inside images
|
| 114 |
+
- `--cores N`: cores to use, one worker process each (default: all usable cores)
|
| 115 |
+
- `--dpi`: render resolution for PDFs (default: 200)
|
| 116 |
+
- `--max-pages N`: read at most N pages per file
|
| 117 |
+
- `--password`: password for encrypted PDFs
|
| 118 |
+
- `--keep-furniture`: keep page headers and footers in the markdown
|
| 119 |
+
- `--math`: write math as LaTeX (`$...$`); off by default, see [Math](#math)
|
| 120 |
+
- `--timings`: print per-page source and time to stderr
|
| 121 |
+
- `--model-dir`: folder with the models (default: download)
|
| 122 |
+
|
| 123 |
+
The CLI continues past files it cannot read, and exits with 1 if any failed.
|
| 124 |
+
|
| 125 |
+
## From python
|
| 126 |
+
|
| 127 |
+
```python
|
| 128 |
+
import indra_ocr
|
| 129 |
+
|
| 130 |
+
conv = indra_ocr.load(threads=4) # mode="auto" | "ocr" | "text"
|
| 131 |
+
for i, page in conv.convert("paper.pdf"):
|
| 132 |
+
print(page.source, page.timings["total_ms"])
|
| 133 |
+
print(page.markdown) # headings, lists, tables as HTML
|
| 134 |
+
page.blocks # [{"label", "bbox", "text"}] in reading order
|
| 135 |
+
page.lines # [Line(box, text, conf)] in reading order
|
| 136 |
+
```
|
| 137 |
+
|
| 138 |
+
For many pages or files, `convert_many` runs one single-threaded worker process per core, one page per task, and returns results in input order:
|
| 139 |
+
|
| 140 |
+
```python
|
| 141 |
+
import indra_ocr
|
| 142 |
+
|
| 143 |
+
if __name__ == "__main__": # worker processes re-import this file
|
| 144 |
+
for path, result in indra_ocr.convert_many(["a.pdf", "b.pdf"], cores=None):
|
| 145 |
+
if isinstance(result, indra_ocr.InputError):
|
| 146 |
+
continue # unreadable or locked file
|
| 147 |
+
for i, page in result:
|
| 148 |
+
print(path, i, page.source)
|
| 149 |
+
```
|
| 150 |
+
|
| 151 |
+
Without the `if __name__ == "__main__":` guard (needed on macOS and Windows), `convert_many` warns and reads everything in one process.
|
| 152 |
+
|
| 153 |
+
`indra_ocr.load` takes `path` (model folder), `threads`, `dpi`, `mode`, `pdf_order`, `image_ocr`, `furniture` and `math`. `conv.convert` takes `max_pages` and `password`.
|
| 154 |
+
|
| 155 |
+
## Viewer
|
| 156 |
+
|
| 157 |
+
```shell
|
| 158 |
+
indra-ocr-viewer --port 8765 # open http://localhost:8765
|
| 159 |
+
```
|
| 160 |
+
|
| 161 |
+
A local web viewer: drop in a PDF or an image, and each page shows its lines, layout blocks and reading order over the page, with the rendered markdown, the markdown source, the lines with their confidences, the JSON output and the time of each stage. The pipeline options (text layer or OCR, math, headers and footers, reading order, OCR of text inside images, dpi, page limit) are set in the page, and lines, blocks, block labels and reading order can each be shown or hidden. Files stay on your machine. `--tesseract` lets the page compare with Tesseract (needs `pytesseract` and a Tesseract install). Over SSH, forward the port: `ssh -N -L 8765:localhost:8765 host`.
|
| 162 |
+
|
| 163 |
+
## Output
|
| 164 |
+
|
| 165 |
+
Every page is a `Page` with:
|
| 166 |
+
|
| 167 |
+
- `markdown`: headings (`##`), paragraphs, lists, and tables as HTML. Page headers and footers are left out unless `furniture=True` / `--keep-furniture`
|
| 168 |
+
- `text`: one line per detected line, in reading order
|
| 169 |
+
- `lines`: `Line(box, text, conf)`, box is `x0, y0, x1, y1` in image pixels
|
| 170 |
+
- `blocks`: layout blocks in reading order, `{"label", "bbox", "text"}`
|
| 171 |
+
- `source`: `text` (PDF text layer), `ocr`, `hybrid` (text layer + OCR of text inside images) or `error`
|
| 172 |
+
- `rotation`, `skew`: the quarter or half turn applied (degrees), and the small tilt straightened
|
| 173 |
+
- `timings`: per-step milliseconds, with `total_ms`
|
| 174 |
+
- `note`: why the page took its path
|
| 175 |
+
|
| 176 |
+
Block labels: `Text`, `Section-Header`, `Page-Header`, `Page-Footer`, `List-Group`, `Table`, `Caption`, `Footnote`, `Table-Of-Contents`, `Form`. `--format json` writes one JSON object per page with all of these fields.
|
| 177 |
+
|
| 178 |
+
# Languages
|
| 179 |
+
|
| 180 |
+
**OCR (scans and images):** Latin (about 50 languages), Cyrillic, Greek, Devanagari, Bengali, and Chinese (simplified and traditional). Each line with Chinese characters goes to the Chinese recognizer.
|
| 181 |
+
|
| 182 |
+
**Text layer (digital PDFs): any script.** We checked text mode on 1,125 pages from 75 language families. On pages whose layer passes the checks, Indra OCR keeps at least 98% of the layer's characters in every family, including Arabic, Hebrew, Persian, Urdu, Thai, Khmer, Lao, Myanmar, Georgian, Armenian, Ethiopic, the Indic scripts, Chinese, Japanese and Korean. Right-to-left lines keep their right-to-left order.
|
| 183 |
+
|
| 184 |
+
# Limitations
|
| 185 |
+
|
| 186 |
+
Indra OCR is built for speed on clean print. It is not the right tool for:
|
| 187 |
+
|
| 188 |
+
- **Complex tables.** Tables with merged or spanning cells, cell text that wraps over several lines, nested headers, or no ruling lines can come out with cells in the wrong rows or columns. Simple grid tables read well.
|
| 189 |
+
- **Handwriting.** Handwritten text is not supported (about 8% on the OmniParseBench handwriting tests).
|
| 190 |
+
- **Scripts outside the OCR list.** OCR reads Latin, Cyrillic, Greek, Devanagari, Bengali and Chinese. Other scripts, including Arabic, Hebrew, Thai, Japanese kana and Korean Hangul, work only through a digital PDF's text layer. On scans and images they come out wrong; a Hangul line, for example, comes out as wrong Chinese characters.
|
| 191 |
+
- **Degraded scans.** Old, faded or low-resolution scans read poorly (about 20% on the OmniParseBench old-scan tests).
|
| 192 |
+
|
| 193 |
+
Also:
|
| 194 |
+
|
| 195 |
+
- Math comes out as plain text unless `--math` is on (see [Math](#math)).
|
| 196 |
+
- No vertical text (such as traditional Chinese newspapers).
|
| 197 |
+
- Very wide, banner-shaped pages read poorly.
|
| 198 |
+
- Labels inside engineering drawings often come out as small tables.
|
| 199 |
+
- Pages above 100 megapixels at 200 dpi render at a lower dpi, so small text on them can be lost.
|
| 200 |
+
- The orientation model turns about half of the upright Arabic and Hebrew pages that go to OCR by mistake. Do not rely on it for those scripts.
|
| 201 |
+
|
| 202 |
+
For these documents, [Chandra on Datalab](https://www.datalab.to/?utm_source=gh-indra) is the accurate option.
|
| 203 |
+
|
| 204 |
+
# Math
|
| 205 |
+
|
| 206 |
+
Math is off by default. With `--math` (or `indra_ocr.load(math=True)`), a small classifier finds the lines that hold math, and a second recognizer reads them as text with inline LaTeX (`we have $x _ { i } \leq 1$`).
|
| 207 |
+
|
| 208 |
+
On OmniParseBench (all 2,937 pages, default settings otherwise):
|
| 209 |
+
|
| 210 |
+
| | headline | text | tables | layout | math tests |
|
| 211 |
+
|---|---|---|---|---|---|
|
| 212 |
+
| Indra OCR | 62.1 | 41.4 | 56.4 | 88.5 | 14.3 |
|
| 213 |
+
| Indra OCR `--math` | **66.8** | **57.5** | 54.8 | 88.1 | **56.5** |
|
| 214 |
+
| PaddleOCR PP-StructureV3 | 58.4 | 55.6 | 47.8 | 71.7 | – |
|
| 215 |
+
|
| 216 |
+
- With `--math`, Indra OCR is ahead of PaddleOCR's PP-StructureV3 on the headline, text, tables and layout. Without it, PP-StructureV3 leads on text.
|
| 217 |
+
- Tables lose 1.5 points: numbers in some table cells are read as math.
|
| 218 |
+
- `--math` is slower. On x86 (AMD EPYC Genoa), one page at a time on 4 cores in auto mode, the median page takes ~360 ms with `--math` and ~210 ms without. With one single-threaded worker per core, throughput is about 30% lower on 4 cores and 35% lower on 32 (4.8 vs 6.8 and 29.2 vs 44.9 pages/s, measured in one container on 300 OmniParseBench pages).
|
| 219 |
+
- The cost falls on the pages where the math path runs. In auto mode, pages read through a digital PDF's text layer slow down most: their median time per page goes from 130 ms to 345 ms in the throughput run (one single-threaded worker per core).
|
| 220 |
+
- Old scanned math gains little (22.2 to 23.1).
|
| 221 |
+
|
| 222 |
+
The math models download only when `--math` is on.
|
| 223 |
+
|
| 224 |
+
# Benchmarks
|
| 225 |
+
|
| 226 |
+
## OmniParseBench
|
| 227 |
+
|
| 228 |
+
[OmniParseBench](https://github.com/datalab-to/omni_parse_bench) has 16,288 pass/fail tests on 2,937 pages. The headline is the mean of the text, tables and layout families. All runs: x86, 4 vCPUs per engine, one page at a time, every engine on the same pages with the same scorer and its default settings.
|
| 229 |
+
|
| 230 |
+
<img src="assets/benchmarks/speed_score.png" width="700px"/>
|
| 231 |
+
|
| 232 |
+
| engine | headline | text | tables | layout | median ms / page |
|
| 233 |
+
|---|---|---|---|---|---|
|
| 234 |
+
| Indra OCR (auto: text layer + OCR) | 62.1 | 41.3 | **56.4** | **88.6** | **228** |
|
| 235 |
+
| Indra OCR (OCR only) | 57.5 | 33.5 | 50.9 | 88.2 | 303 |
|
| 236 |
+
| Indra OCR `--math` (auto) | **66.8** | **57.5** | 54.8 | 88.1 | ~360† |
|
| 237 |
+
| PaddleOCR PP-OCRv6 small, text models only (RapidOCR) | – | 35.8 | 0.0 | – | 2,685 |
|
| 238 |
+
| PaddleOCR PP-StructureV3 (2,918 pages\*) | 58.4 | 55.6 | 47.8 | 71.7 | 40,958 |
|
| 239 |
+
| Tesseract 5 (our run, Latin model) | – | 19.3 | 0.0 | – | 5,496 |
|
| 240 |
+
|
| 241 |
+
- The PP-OCR text models and Tesseract output lines, not tables or layout blocks, so their table scores are 0 and their layout family is not scored.
|
| 242 |
+
- PP-StructureV3 is PaddleOCR's full document pipeline (layout, tables, formulas). Without `--math`, it reads text better than Indra OCR (55.6 vs 41.3). With `--math`, Indra OCR is ahead on text too (57.5). It takes about 180 times as long per page as Indra OCR's default mode. \*One of its 150 shards stalled for 6 hours on its first page, so its row leaves out 19 pages.
|
| 243 |
+
- Faster settings exist, but we did not score their accuracy. Tesseract with the `eng` model and 1 thread takes 3.4 s a page (6.1 s as run). PP-StructureV3 without formula, seal and chart recognition, unwarping and orientation, and with its mobile OCR models, takes 10.5 s a page. Indra OCR is still 15-45x faster per page.
|
| 244 |
+
- The OmniParseBench leaderboard's own Tesseract run scores 17.6 on text.
|
| 245 |
+
- Median latency varies by about 15% between runs of the same code. †The `--math` latency is from a separate run on x86 (AMD EPYC Genoa, 4 cores, one page at a time), where the default mode gave 208 ms.
|
| 246 |
+
|
| 247 |
+
**Chinese** (136 OmniParseBench Chinese tests): Indra OCR 64.7, PaddleOCR PP-OCRv5 mobile Chinese text models 65.4, PP-StructureV3 68.4.
|
| 248 |
+
|
| 249 |
+
## olmOCR-bench
|
| 250 |
+
|
| 251 |
+
Born-digital splits and old scans, our harness, 200 dpi:
|
| 252 |
+
|
| 253 |
+
| split | Indra OCR | Tesseract 4 |
|
| 254 |
+
|---|---|---|
|
| 255 |
+
| headers_footers | 91.2 | 35.5 |
|
| 256 |
+
| multi_column | 70.0 | 70.4 |
|
| 257 |
+
| long_tiny_text | 67.9 | 64.0 |
|
| 258 |
+
| table_tests | 60.5 | 1.0 |
|
| 259 |
+
| old_scans | 20.0 | 18.4 |
|
| 260 |
+
|
| 261 |
+
## Throughput
|
| 262 |
+
|
| 263 |
+
The CLI and `convert_many` run one single-threaded worker process per usable core, one page per task, so a single long PDF also uses every core. `--cores` limits it.
|
| 264 |
+
|
| 265 |
+
Pages per second. Every engine ran on the same x86 machine (AMD EPYC, no GPU) on the same 300 OmniParseBench pages, with one single-threaded worker per core, OCR mode and default settings. Indra OCR is the mean of 3 runs. The other engines ran once.
|
| 266 |
+
|
| 267 |
+
| engine | 4 cores | 8 cores | 16 cores | 32 cores |
|
| 268 |
+
|---|---|---|---|---|
|
| 269 |
+
| Indra OCR | **7.3** | **14.2** | **27.2** | **49.4** |
|
| 270 |
+
| Indra OCR `--math`† | ~5 | – | – | ~30 |
|
| 271 |
+
| PaddleOCR PP-OCRv6 small, text models only (RapidOCR) | 0.65 | 1.24 | 2.42 | 4.28 |
|
| 272 |
+
| Tesseract 5 | 0.78 | 1.37 | 2.27 | 2.92 |
|
| 273 |
+
| PaddleOCR PP-StructureV3\* | ≤ 0.022 | ≤ 0.044 | ≤ 0.089 | ≤ 0.18 |
|
| 274 |
+
|
| 275 |
+
At 32 cores, Indra OCR is 11.5x PaddleOCR's PP-OCR text models, 16.9x Tesseract and about 280x PP-StructureV3 (against its upper bound). \*PP-StructureV3 ran a time-capped subset of the pages, so its numbers are upper bounds. †`--math` ran in a separate container with the same CPU type and the same pages, where it gave 4.8 pages/s on 4 cores and 29.2 on 32, against 6.8 and 44.9 without it (about 30% and 35% lower).
|
| 276 |
+
|
| 277 |
+
Model loading and one warm-up page per worker are not timed. A short 300-page job through `convert_many`, model loading included, runs at 7.2 pages/s on 4 cores and 37.7 on 32.
|
| 278 |
+
|
| 279 |
+
## Reproducing
|
| 280 |
+
|
| 281 |
+
```shell
|
| 282 |
+
git clone https://github.com/datalab-to/indra.git && cd indra
|
| 283 |
+
pip install indra-ocr 'omni-parse-bench[benchmark]'
|
| 284 |
+
python bench/opb_indra.py --out runs/opb_auto --mode auto --procs 8
|
| 285 |
+
```
|
| 286 |
+
|
| 287 |
+
Each page runs in a single-threaded worker process, page headers and footers kept. `--mode ocr` gives the OCR-only row, `--math` the `--math` row, `--suites` scores only the listed suites, `--limit N` scores the first N pages, and `--models` points at a local model folder. The run writes `summary.json` (the benchmark's own summary, plus median and p90 ms per page) and `pages.jsonl` (per page: time, source, tests passed).
|
| 288 |
+
|
| 289 |
+
# Credits
|
| 290 |
+
|
| 291 |
+
Thank you to the following open source projects:
|
| 292 |
+
|
| 293 |
+
- [ONNX Runtime](https://github.com/microsoft/onnxruntime)
|
| 294 |
+
- [pypdfium2](https://github.com/pypdfium2-team/pypdfium2)
|
| 295 |
+
- [OpenCV](https://github.com/opencv/opencv)
|
| 296 |
+
- [Hugging Face Hub](https://github.com/huggingface/huggingface_hub)
|
OCR_INDRA/indra/assets/benchmarks/speed_score.png
ADDED
|
Git LFS Details
|
OCR_INDRA/indra/assets/benchmarks/throughput.png
ADDED
|
Git LFS Details
|
OCR_INDRA/indra/assets/datalab-logo.png
ADDED
|
OCR_INDRA/indra/assets/examples/chinese_essay.png
ADDED
|
Git LFS Details
|
OCR_INDRA/indra/assets/examples/multi_column.png
ADDED
|
Git LFS Details
|
OCR_INDRA/indra/assets/examples/rotated_letter.png
ADDED
|
Git LFS Details
|
OCR_INDRA/indra/assets/examples/table_pmc.png
ADDED
|
Git LFS Details
|
OCR_INDRA/indra/bench/opb_indra.py
ADDED
|
@@ -0,0 +1,180 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""Score Indra OCR on Omni Parse Bench (github.com/datalab-to/omni_parse_bench).
|
| 2 |
+
|
| 3 |
+
pip install 'omni-parse-bench[benchmark]'
|
| 4 |
+
python bench/opb_indra.py --out runs/opb_auto --mode auto --procs 8
|
| 5 |
+
|
| 6 |
+
Each page goes through the Indra OCR Converter (one thread per process, page
|
| 7 |
+
headers and footers kept: the benchmark asks for everything on the page). The
|
| 8 |
+
markdown becomes HTML through the benchmark's own converter, as every vendor's
|
| 9 |
+
markdown does, and the layout blocks map onto the benchmark's labels. Latency
|
| 10 |
+
is the converter's wall time per page, page loading included.
|
| 11 |
+
|
| 12 |
+
Writes summary.json (the benchmark's summarize) and pages.jsonl (per page:
|
| 13 |
+
latency, source, verdict counts).
|
| 14 |
+
"""
|
| 15 |
+
|
| 16 |
+
from __future__ import annotations
|
| 17 |
+
|
| 18 |
+
import argparse
|
| 19 |
+
import json
|
| 20 |
+
import time
|
| 21 |
+
from pathlib import Path
|
| 22 |
+
|
| 23 |
+
# Indra OCR block class -> Omni Parse Bench label (views.LABELS)
|
| 24 |
+
LABELS = {
|
| 25 |
+
"Text": "text",
|
| 26 |
+
"Section-Header": "heading",
|
| 27 |
+
"Page-Header": "page_furniture",
|
| 28 |
+
"Page-Footer": "page_furniture",
|
| 29 |
+
"List-Group": "list",
|
| 30 |
+
"Table": "table",
|
| 31 |
+
"Caption": "caption",
|
| 32 |
+
"Footnote": "footnote",
|
| 33 |
+
"Table-Of-Contents": "text",
|
| 34 |
+
"Form": "form",
|
| 35 |
+
}
|
| 36 |
+
|
| 37 |
+
_CONV = None
|
| 38 |
+
_ARGS: dict = {}
|
| 39 |
+
|
| 40 |
+
|
| 41 |
+
def init(args: dict) -> None:
|
| 42 |
+
global _CONV, _ARGS
|
| 43 |
+
import indra_ocr
|
| 44 |
+
|
| 45 |
+
_ARGS = args
|
| 46 |
+
_CONV = indra_ocr.load(
|
| 47 |
+
args["models"],
|
| 48 |
+
threads=1,
|
| 49 |
+
mode=args["mode"],
|
| 50 |
+
furniture=True,
|
| 51 |
+
math=args.get("math", False),
|
| 52 |
+
)
|
| 53 |
+
|
| 54 |
+
|
| 55 |
+
def outputs_of(page, w: float, h: float) -> dict:
|
| 56 |
+
from omni_parse_bench import markdown
|
| 57 |
+
|
| 58 |
+
md = page.markdown if page.markdown is not None else page.text
|
| 59 |
+
clip = lambda v: min(1.0, max(0.0, v)) # noqa: E731
|
| 60 |
+
blocks = [
|
| 61 |
+
{
|
| 62 |
+
"label": LABELS.get(b["label"], "text"),
|
| 63 |
+
"bbox": [
|
| 64 |
+
clip(b["bbox"][0] / w),
|
| 65 |
+
clip(b["bbox"][1] / h),
|
| 66 |
+
clip(b["bbox"][2] / w),
|
| 67 |
+
clip(b["bbox"][3] / h),
|
| 68 |
+
],
|
| 69 |
+
"text": b["text"],
|
| 70 |
+
}
|
| 71 |
+
for b in page.blocks
|
| 72 |
+
]
|
| 73 |
+
return {"html": markdown.to_html(md), "blocks": blocks}
|
| 74 |
+
|
| 75 |
+
|
| 76 |
+
def run_page(item: tuple[str, str]) -> dict:
|
| 77 |
+
import cv2
|
| 78 |
+
import pypdfium2 as pdfium
|
| 79 |
+
from omni_parse_bench import benchmark, score
|
| 80 |
+
|
| 81 |
+
page_id, doc_path = item
|
| 82 |
+
root, _, by_page = benchmark.load(_ARGS["dataset"])
|
| 83 |
+
path = benchmark.page_file(_ARGS["dataset"], root, doc_path)
|
| 84 |
+
tests = by_page[page_id]
|
| 85 |
+
t0 = time.perf_counter()
|
| 86 |
+
try:
|
| 87 |
+
if str(path).lower().endswith(".pdf"):
|
| 88 |
+
doc = pdfium.PdfDocument(str(path))
|
| 89 |
+
page = _CONV.pdf_page(str(path), doc, 0)
|
| 90 |
+
pw, ph = doc[0].get_size()
|
| 91 |
+
k = _CONV.dpi / 72.0
|
| 92 |
+
w, h = pw * k, ph * k
|
| 93 |
+
doc.close()
|
| 94 |
+
else:
|
| 95 |
+
gray = cv2.imread(str(path), cv2.IMREAD_GRAYSCALE)
|
| 96 |
+
page = _CONV.ocr(gray)
|
| 97 |
+
h, w = gray.shape
|
| 98 |
+
ms = 1000 * (time.perf_counter() - t0)
|
| 99 |
+
verdicts = score(outputs_of(page, w, h), tests)
|
| 100 |
+
source = page.source
|
| 101 |
+
except Exception as e: # noqa: BLE001 (a failed page fails its tests, it does not stop the run)
|
| 102 |
+
ms, source = (
|
| 103 |
+
1000 * (time.perf_counter() - t0),
|
| 104 |
+
f"error: {type(e).__name__}: {e}",
|
| 105 |
+
)
|
| 106 |
+
verdicts = score({}, tests)
|
| 107 |
+
return {"page_id": page_id, "ms": ms, "source": source, "verdicts": verdicts}
|
| 108 |
+
|
| 109 |
+
|
| 110 |
+
def main():
|
| 111 |
+
import multiprocessing as mp
|
| 112 |
+
|
| 113 |
+
from omni_parse_bench import benchmark, summarize
|
| 114 |
+
|
| 115 |
+
ap = argparse.ArgumentParser(
|
| 116 |
+
description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter
|
| 117 |
+
)
|
| 118 |
+
ap.add_argument("--out", type=Path, required=True)
|
| 119 |
+
ap.add_argument("--dataset", default=benchmark.DEFAULT_DATASET)
|
| 120 |
+
ap.add_argument("--models", default=None, help="model folder (default: download)")
|
| 121 |
+
ap.add_argument("--mode", choices=["auto", "ocr", "text"], default="auto")
|
| 122 |
+
ap.add_argument("--procs", type=int, default=8)
|
| 123 |
+
ap.add_argument("--limit", type=int, default=0)
|
| 124 |
+
ap.add_argument(
|
| 125 |
+
"--suites", default="", help="comma-separated suites (default: all)"
|
| 126 |
+
)
|
| 127 |
+
ap.add_argument("--math", action="store_true", help="write math as LaTeX")
|
| 128 |
+
a = ap.parse_args()
|
| 129 |
+
a.out.mkdir(parents=True, exist_ok=True)
|
| 130 |
+
root, pages, by_page = benchmark.load(a.dataset)
|
| 131 |
+
keep = set(a.suites.split(",")) if a.suites else None
|
| 132 |
+
pages = [p for p in pages if keep is None or p.suite in keep][: a.limit or None]
|
| 133 |
+
items = [(p.page_id, p.doc_path) for p in pages]
|
| 134 |
+
args = {"models": a.models, "mode": a.mode, "dataset": a.dataset, "math": a.math}
|
| 135 |
+
t0 = time.time()
|
| 136 |
+
rows, verdicts = [], []
|
| 137 |
+
with mp.get_context("spawn").Pool(
|
| 138 |
+
a.procs, initializer=init, initargs=(args,)
|
| 139 |
+
) as pool:
|
| 140 |
+
for k, r in enumerate(pool.imap_unordered(run_page, items, chunksize=4), 1):
|
| 141 |
+
verdicts += r["verdicts"]
|
| 142 |
+
rows.append(
|
| 143 |
+
{
|
| 144 |
+
"page_id": r["page_id"],
|
| 145 |
+
"ms": round(r["ms"], 1),
|
| 146 |
+
"source": r["source"],
|
| 147 |
+
"pass": sum(v.result == "pass" for v in r["verdicts"]),
|
| 148 |
+
"tests": len(r["verdicts"]),
|
| 149 |
+
}
|
| 150 |
+
)
|
| 151 |
+
if k % 200 == 0:
|
| 152 |
+
print(f"{k}/{len(items)} pages, {time.time() - t0:.0f} s", flush=True)
|
| 153 |
+
tests = [t for p in pages for t in by_page[p.page_id]]
|
| 154 |
+
summary = summarize(verdicts, tests, pages)
|
| 155 |
+
ms = sorted(r["ms"] for r in rows)
|
| 156 |
+
summary["indra_ocr"] = {
|
| 157 |
+
"mode": a.mode,
|
| 158 |
+
"pages": len(rows),
|
| 159 |
+
"ms_median": ms[len(ms) // 2],
|
| 160 |
+
"ms_p90": ms[int(0.9 * (len(ms) - 1))],
|
| 161 |
+
"sources": {
|
| 162 |
+
s: sum(r["source"] == s for r in rows)
|
| 163 |
+
for s in {r["source"][:20] for r in rows}
|
| 164 |
+
},
|
| 165 |
+
}
|
| 166 |
+
(a.out / "summary.json").write_text(json.dumps(summary, indent=1, default=str))
|
| 167 |
+
with open(a.out / "pages.jsonl", "w") as f:
|
| 168 |
+
for r in rows:
|
| 169 |
+
f.write(json.dumps(r) + "\n")
|
| 170 |
+
print(
|
| 171 |
+
json.dumps(
|
| 172 |
+
{"headline": summary["headline"], **summary["indra_ocr"]},
|
| 173 |
+
indent=1,
|
| 174 |
+
default=str,
|
| 175 |
+
)
|
| 176 |
+
)
|
| 177 |
+
|
| 178 |
+
|
| 179 |
+
if __name__ == "__main__":
|
| 180 |
+
main()
|
OCR_INDRA/indra/indra_ocr/__init__.py
ADDED
|
@@ -0,0 +1,51 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""Indra OCR: fast CPU OCR (line detection, CTC recognition, reading order, markdown)."""
|
| 2 |
+
|
| 3 |
+
from .document import Converter
|
| 4 |
+
from .models import model_dir
|
| 5 |
+
from .parallel import convert_many, usable_cores
|
| 6 |
+
from .models import ModelError
|
| 7 |
+
from .pipeline import InputError, Line, OrtOCR, Page, load_pages
|
| 8 |
+
|
| 9 |
+
__all__ = [
|
| 10 |
+
"Converter",
|
| 11 |
+
"InputError",
|
| 12 |
+
"ModelError",
|
| 13 |
+
"Line",
|
| 14 |
+
"OrtOCR",
|
| 15 |
+
"Page",
|
| 16 |
+
"convert_many",
|
| 17 |
+
"usable_cores",
|
| 18 |
+
"load_pages",
|
| 19 |
+
"load",
|
| 20 |
+
"model_dir",
|
| 21 |
+
]
|
| 22 |
+
__version__ = "0.3.2"
|
| 23 |
+
|
| 24 |
+
|
| 25 |
+
def load(
|
| 26 |
+
path=None,
|
| 27 |
+
threads: int = 4,
|
| 28 |
+
dpi: float = 200.0,
|
| 29 |
+
mode: str = "auto",
|
| 30 |
+
pdf_order: bool = True,
|
| 31 |
+
image_ocr: bool = True,
|
| 32 |
+
providers: list | None = None,
|
| 33 |
+
furniture: bool = False,
|
| 34 |
+
math: bool = False,
|
| 35 |
+
) -> Converter:
|
| 36 |
+
"""A ready Converter (see ``Converter`` for the options). mode="ocr" never
|
| 37 |
+
reads the PDF text layer. furniture: keep page headers and footers in the
|
| 38 |
+
markdown. math: write math as LaTeX (slower on math pages; off by default)."""
|
| 39 |
+
return Converter(
|
| 40 |
+
OrtOCR(
|
| 41 |
+
model_dir(path, math=math),
|
| 42 |
+
threads=threads,
|
| 43 |
+
providers=providers,
|
| 44 |
+
furniture=furniture,
|
| 45 |
+
math=math,
|
| 46 |
+
),
|
| 47 |
+
dpi=dpi,
|
| 48 |
+
mode=mode,
|
| 49 |
+
pdf_order=pdf_order,
|
| 50 |
+
image_ocr=image_ocr,
|
| 51 |
+
)
|
OCR_INDRA/indra/indra_ocr/__main__.py
ADDED
|
@@ -0,0 +1,4 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
from indra_ocr.cli import main
|
| 2 |
+
|
| 3 |
+
if __name__ == "__main__":
|
| 4 |
+
main()
|
OCR_INDRA/indra/indra_ocr/cli.py
ADDED
|
@@ -0,0 +1,175 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""Command line: PDFs or images in, markdown (or text, or JSON lines) out.
|
| 2 |
+
|
| 3 |
+
indra-ocr paper.pdf # markdown to stdout
|
| 4 |
+
indra-ocr *.pdf --out results/ # one .md per input
|
| 5 |
+
indra-ocr scan.png --format json # lines with boxes, confidences, timings
|
| 6 |
+
|
| 7 |
+
Pages run in parallel on all the cores the process may use (--cores to limit).
|
| 8 |
+
"""
|
| 9 |
+
|
| 10 |
+
from __future__ import annotations
|
| 11 |
+
|
| 12 |
+
import argparse
|
| 13 |
+
import json
|
| 14 |
+
import sys
|
| 15 |
+
from dataclasses import asdict
|
| 16 |
+
from pathlib import Path
|
| 17 |
+
|
| 18 |
+
from . import InputError
|
| 19 |
+
from .parallel import convert_many, usable_cores
|
| 20 |
+
|
| 21 |
+
|
| 22 |
+
def add_source_args(ap: argparse.ArgumentParser) -> None:
|
| 23 |
+
"""Options that choose between the PDF text layer and OCR."""
|
| 24 |
+
g = ap.add_argument_group("text layer or OCR (PDFs)")
|
| 25 |
+
g.add_argument(
|
| 26 |
+
"--mode",
|
| 27 |
+
choices=["auto", "ocr", "text"],
|
| 28 |
+
default="auto",
|
| 29 |
+
help="auto: text layer where it passes the checks, OCR elsewhere (default); "
|
| 30 |
+
"ocr: never read the text layer; text: use the text layer even when the "
|
| 31 |
+
"checks fail",
|
| 32 |
+
)
|
| 33 |
+
g.add_argument(
|
| 34 |
+
"--model-order",
|
| 35 |
+
action="store_true",
|
| 36 |
+
help="order text-layer lines with the order model, not the PDF content order",
|
| 37 |
+
)
|
| 38 |
+
g.add_argument(
|
| 39 |
+
"--no-image-ocr",
|
| 40 |
+
action="store_true",
|
| 41 |
+
help="on text-layer pages, do not OCR text inside images",
|
| 42 |
+
)
|
| 43 |
+
|
| 44 |
+
|
| 45 |
+
def source_options(a) -> dict:
|
| 46 |
+
return {
|
| 47 |
+
"mode": a.mode,
|
| 48 |
+
"pdf_order": not a.model_order,
|
| 49 |
+
"image_ocr": not a.no_image_ocr,
|
| 50 |
+
}
|
| 51 |
+
|
| 52 |
+
|
| 53 |
+
def main(argv=None):
|
| 54 |
+
"""Exit code 0, or 1 when an input failed or the models are not available."""
|
| 55 |
+
from .models import ModelError
|
| 56 |
+
|
| 57 |
+
try:
|
| 58 |
+
return _main(argv)
|
| 59 |
+
except ModelError as e:
|
| 60 |
+
print(f"indra-ocr: {e}", file=sys.stderr)
|
| 61 |
+
return 1
|
| 62 |
+
|
| 63 |
+
|
| 64 |
+
def _main(argv=None):
|
| 65 |
+
from . import __version__
|
| 66 |
+
|
| 67 |
+
ap = argparse.ArgumentParser(
|
| 68 |
+
prog="indra-ocr",
|
| 69 |
+
description=__doc__,
|
| 70 |
+
formatter_class=argparse.RawDescriptionHelpFormatter,
|
| 71 |
+
)
|
| 72 |
+
ap.add_argument("--version", action="version", version=f"indra-ocr {__version__}")
|
| 73 |
+
ap.add_argument("inputs", nargs="+", type=Path)
|
| 74 |
+
ap.add_argument(
|
| 75 |
+
"--out", type=Path, default=None, help="write one file per input here"
|
| 76 |
+
)
|
| 77 |
+
ap.add_argument("--format", choices=["md", "text", "json"], default="md")
|
| 78 |
+
add_source_args(ap)
|
| 79 |
+
ap.add_argument("--model-dir", type=Path, default=None)
|
| 80 |
+
ap.add_argument(
|
| 81 |
+
"--cores",
|
| 82 |
+
"--threads",
|
| 83 |
+
dest="cores",
|
| 84 |
+
type=int,
|
| 85 |
+
default=0,
|
| 86 |
+
help=f"cores to use (default: all this process may run on, here {usable_cores()})",
|
| 87 |
+
)
|
| 88 |
+
ap.add_argument("--dpi", type=float, default=200.0)
|
| 89 |
+
ap.add_argument("--max-pages", type=int, default=0)
|
| 90 |
+
ap.add_argument("--password", default=None, help="password of encrypted PDFs")
|
| 91 |
+
ap.add_argument(
|
| 92 |
+
"--math",
|
| 93 |
+
action="store_true",
|
| 94 |
+
help="write math as LaTeX ($...$); slower on pages with math (off by default)",
|
| 95 |
+
)
|
| 96 |
+
ap.add_argument(
|
| 97 |
+
"--keep-furniture",
|
| 98 |
+
action="store_true",
|
| 99 |
+
help="keep page headers and footers (page numbers, running titles) in the markdown",
|
| 100 |
+
)
|
| 101 |
+
ap.add_argument(
|
| 102 |
+
"--timings", action="store_true", help="print per-page timings to stderr"
|
| 103 |
+
)
|
| 104 |
+
a = ap.parse_args(argv)
|
| 105 |
+
if a.out:
|
| 106 |
+
a.out.mkdir(parents=True, exist_ok=True)
|
| 107 |
+
ext = {"md": ".md", "text": ".txt", "json": ".jsonl"}[a.format]
|
| 108 |
+
failed, names = 0, set()
|
| 109 |
+
results = convert_many(
|
| 110 |
+
a.inputs,
|
| 111 |
+
cores=a.cores,
|
| 112 |
+
max_pages=a.max_pages,
|
| 113 |
+
password=a.password,
|
| 114 |
+
path=a.model_dir,
|
| 115 |
+
dpi=a.dpi,
|
| 116 |
+
furniture=a.keep_furniture,
|
| 117 |
+
math=a.math,
|
| 118 |
+
**source_options(a),
|
| 119 |
+
)
|
| 120 |
+
for path, result in results:
|
| 121 |
+
path = Path(path)
|
| 122 |
+
chunks = []
|
| 123 |
+
if isinstance(result, InputError):
|
| 124 |
+
print(f"indra-ocr: {result}", file=sys.stderr)
|
| 125 |
+
failed += 1
|
| 126 |
+
continue
|
| 127 |
+
for i, page in result:
|
| 128 |
+
if page.source == "error":
|
| 129 |
+
print(f"indra-ocr: {path} page {i + 1}: {page.note}", file=sys.stderr)
|
| 130 |
+
if a.format == "json":
|
| 131 |
+
chunks.append(
|
| 132 |
+
json.dumps(
|
| 133 |
+
{
|
| 134 |
+
"file": str(path),
|
| 135 |
+
"page": i,
|
| 136 |
+
"source": page.source,
|
| 137 |
+
"rotation": page.rotation,
|
| 138 |
+
"skew": page.skew,
|
| 139 |
+
"note": page.note,
|
| 140 |
+
"lines": [asdict(ln) for ln in page.lines],
|
| 141 |
+
"markdown": page.markdown,
|
| 142 |
+
"blocks": page.blocks,
|
| 143 |
+
"timings": page.timings,
|
| 144 |
+
},
|
| 145 |
+
ensure_ascii=False,
|
| 146 |
+
)
|
| 147 |
+
)
|
| 148 |
+
elif a.format == "md":
|
| 149 |
+
chunks.append(page.markdown if page.markdown is not None else page.text)
|
| 150 |
+
else:
|
| 151 |
+
chunks.append(page.text)
|
| 152 |
+
if a.timings:
|
| 153 |
+
t = page.timings
|
| 154 |
+
print(
|
| 155 |
+
f"{path} page {i + 1}: {page.source}, {t.get('total_ms', 0):.0f} ms {page.note}",
|
| 156 |
+
file=sys.stderr,
|
| 157 |
+
)
|
| 158 |
+
sep = "\n" if a.format == "json" else "\n\n"
|
| 159 |
+
body = sep.join(chunks) + "\n"
|
| 160 |
+
if a.out:
|
| 161 |
+
name, k = path.stem, 1
|
| 162 |
+
while name in names: # two inputs with one name: keep both outputs
|
| 163 |
+
k += 1
|
| 164 |
+
name = f"{path.stem}_{k}"
|
| 165 |
+
names.add(name)
|
| 166 |
+
(a.out / (name + ext)).write_text(body, encoding="utf-8")
|
| 167 |
+
else:
|
| 168 |
+
sys.stdout.write(body)
|
| 169 |
+
if failed:
|
| 170 |
+
print(f"indra-ocr: {failed} of {len(a.inputs)} inputs failed", file=sys.stderr)
|
| 171 |
+
return 1 if failed else 0
|
| 172 |
+
|
| 173 |
+
|
| 174 |
+
if __name__ == "__main__":
|
| 175 |
+
sys.exit(main())
|
OCR_INDRA/indra/indra_ocr/det_post.py
ADDED
|
@@ -0,0 +1,155 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""Torch-free pre- and post-processing for LineNet (shared by training and inference).
|
| 2 |
+
|
| 3 |
+
Inference with ONNX Runtime needs only numpy and cv2, so these functions live
|
| 4 |
+
apart from the PyTorch model in det_model.py, which re-exports them.
|
| 5 |
+
"""
|
| 6 |
+
|
| 7 |
+
from __future__ import annotations
|
| 8 |
+
|
| 9 |
+
import math
|
| 10 |
+
|
| 11 |
+
import numpy as np
|
| 12 |
+
|
| 13 |
+
# Target geometry, shared by training (encode) and inference (decode).
|
| 14 |
+
CORE_FRAC = 0.4 # core band height as a fraction of the line ink height
|
| 15 |
+
CORE_X_SHRINK = 0.15 # the core is shorter than the line by this fraction of its height, at each end
|
| 16 |
+
OUT_STRIDE = 2
|
| 17 |
+
INPUT_MULTIPLE = 32 # H and W must divide by this
|
| 18 |
+
|
| 19 |
+
|
| 20 |
+
# ---------------------------------------------------------------------------
|
| 21 |
+
# Input preprocessing (shared by training and inference)
|
| 22 |
+
# ---------------------------------------------------------------------------
|
| 23 |
+
|
| 24 |
+
|
| 25 |
+
def page_to_input(gray: np.ndarray, long_side: int) -> tuple[np.ndarray, float]:
|
| 26 |
+
"""Resize a uint8 grayscale page so its long side is ``long_side`` and pad
|
| 27 |
+
to a multiple of 32 with white. Returns (float32 [H, W] ink map, scale)."""
|
| 28 |
+
import cv2
|
| 29 |
+
|
| 30 |
+
h, w = gray.shape
|
| 31 |
+
scale = long_side / max(h, w)
|
| 32 |
+
nh, nw = max(1, round(h * scale)), max(1, round(w * scale))
|
| 33 |
+
if (nh, nw) != (h, w):
|
| 34 |
+
interp = cv2.INTER_AREA if scale < 1 else cv2.INTER_LINEAR
|
| 35 |
+
gray = cv2.resize(gray, (nw, nh), interpolation=interp)
|
| 36 |
+
ph = math.ceil(nh / INPUT_MULTIPLE) * INPUT_MULTIPLE
|
| 37 |
+
pw = math.ceil(nw / INPUT_MULTIPLE) * INPUT_MULTIPLE
|
| 38 |
+
out = np.zeros((ph, pw), np.float32) # ink map: 0 = white
|
| 39 |
+
out[:nh, :nw] = 1.0 - gray.astype(np.float32) / 255.0
|
| 40 |
+
return out, scale
|
| 41 |
+
|
| 42 |
+
|
| 43 |
+
# ---------------------------------------------------------------------------
|
| 44 |
+
# Targets
|
| 45 |
+
# ---------------------------------------------------------------------------
|
| 46 |
+
|
| 47 |
+
|
| 48 |
+
def encode_targets(
|
| 49 |
+
boxes: np.ndarray,
|
| 50 |
+
in_hw: tuple[int, int],
|
| 51 |
+
valid_hw: tuple[int, int],
|
| 52 |
+
ignore: np.ndarray | None = None,
|
| 53 |
+
):
|
| 54 |
+
"""Build training targets at 1/2 resolution.
|
| 55 |
+
|
| 56 |
+
boxes: [N, 4] line ink boxes in input pixels (x0, y0, x1, y1)
|
| 57 |
+
in_hw: padded input size; valid_hw: size of the real page inside it
|
| 58 |
+
ignore: [M, 4] regions with no loss, in input pixels
|
| 59 |
+
|
| 60 |
+
Returns core [h, w] float32, dist [2, h, w] float32, weight [h, w] float32
|
| 61 |
+
(0 on padding and ignore regions), and reg_mask [h, w] bool (core pixels).
|
| 62 |
+
"""
|
| 63 |
+
s = OUT_STRIDE
|
| 64 |
+
h, w = in_hw[0] // s, in_hw[1] // s
|
| 65 |
+
core = np.zeros((h, w), np.float32)
|
| 66 |
+
dist = np.zeros((2, h, w), np.float32)
|
| 67 |
+
weight = np.zeros((h, w), np.float32)
|
| 68 |
+
weight[: math.ceil(valid_hw[0] / s), : math.ceil(valid_hw[1] / s)] = 1.0
|
| 69 |
+
if ignore is not None:
|
| 70 |
+
for x0, y0, x1, y1 in ignore:
|
| 71 |
+
weight[
|
| 72 |
+
max(0, int(y0 // s)) : max(0, math.ceil(y1 / s)),
|
| 73 |
+
max(0, int(x0 // s)) : max(0, math.ceil(x1 / s)),
|
| 74 |
+
] = 0.0
|
| 75 |
+
# Larger lines first, so a small line drawn later keeps its own band.
|
| 76 |
+
order = np.argsort(-(boxes[:, 3] - boxes[:, 1])) if len(boxes) else []
|
| 77 |
+
ys = (np.arange(h, dtype=np.float32) + 0.5) * s # pixel centres in input px
|
| 78 |
+
for i in order:
|
| 79 |
+
x0, y0, x1, y1 = boxes[i]
|
| 80 |
+
lh = y1 - y0
|
| 81 |
+
if lh < 2 or x1 - x0 < 2:
|
| 82 |
+
continue
|
| 83 |
+
cy = 0.5 * (y0 + y1)
|
| 84 |
+
half = 0.5 * CORE_FRAC * lh
|
| 85 |
+
r0 = max(0, int(math.floor((cy - half) / s)))
|
| 86 |
+
r1 = min(h, max(r0 + 1, int(math.ceil((cy + half) / s))))
|
| 87 |
+
sx = min(CORE_X_SHRINK * lh, 0.25 * (x1 - x0))
|
| 88 |
+
c0 = max(0, int(math.floor((x0 + sx) / s)))
|
| 89 |
+
c1 = min(w, max(c0 + 1, int(math.ceil((x1 - sx) / s))))
|
| 90 |
+
if r0 >= h or c0 >= w:
|
| 91 |
+
continue
|
| 92 |
+
core[r0:r1, c0:c1] = 1.0
|
| 93 |
+
dist[0, r0:r1, c0:c1] = (ys[r0:r1] - y0)[:, None]
|
| 94 |
+
dist[1, r0:r1, c0:c1] = (y1 - ys[r0:r1])[:, None]
|
| 95 |
+
# A line inside an ignore region is still a positive.
|
| 96 |
+
weight[r0:r1, c0:c1] = np.maximum(weight[r0:r1, c0:c1], 1.0)
|
| 97 |
+
return core, dist, weight, core > 0.5
|
| 98 |
+
|
| 99 |
+
|
| 100 |
+
# ---------------------------------------------------------------------------
|
| 101 |
+
# Decoding
|
| 102 |
+
# ---------------------------------------------------------------------------
|
| 103 |
+
|
| 104 |
+
|
| 105 |
+
def decode(
|
| 106 |
+
out: np.ndarray,
|
| 107 |
+
thresh: float = 0.5,
|
| 108 |
+
min_area: int = 2,
|
| 109 |
+
grow_y: float = 0.0,
|
| 110 |
+
grow_x: float = 0.0,
|
| 111 |
+
) -> np.ndarray:
|
| 112 |
+
"""Decode one page. out: [3, h, w] (core logit, top dist, bottom dist).
|
| 113 |
+
|
| 114 |
+
``grow_y`` / ``grow_x`` widen every box by that fraction of its height
|
| 115 |
+
(each side), to absorb the +-1 px error of the 1/2-resolution output.
|
| 116 |
+
|
| 117 |
+
Returns [N, 5] boxes in input pixels: x0, y0, x1, y1, score."""
|
| 118 |
+
import cv2
|
| 119 |
+
|
| 120 |
+
prob = 1.0 / (1.0 + np.exp(-out[0]))
|
| 121 |
+
mask = (prob > thresh).astype(np.uint8)
|
| 122 |
+
n, lab, stats, _ = cv2.connectedComponentsWithStats(mask, connectivity=4)
|
| 123 |
+
if n <= 1:
|
| 124 |
+
return np.zeros((0, 5), np.float32)
|
| 125 |
+
s = OUT_STRIDE
|
| 126 |
+
flat = lab.ravel()
|
| 127 |
+
keep = flat > 0
|
| 128 |
+
idx = flat[keep]
|
| 129 |
+
ys = (np.indices(lab.shape)[0].ravel()[keep].astype(np.float32) + 0.5) * s
|
| 130 |
+
top = ys - out[1].ravel()[keep]
|
| 131 |
+
bot = ys + out[2].ravel()[keep]
|
| 132 |
+
sc = prob.ravel()[keep]
|
| 133 |
+
cnt = np.bincount(idx, minlength=n).astype(np.float32)
|
| 134 |
+
# Mean is a robust enough summary here and is far cheaper than a median.
|
| 135 |
+
y0 = np.bincount(idx, weights=top, minlength=n) / np.maximum(cnt, 1)
|
| 136 |
+
y1 = np.bincount(idx, weights=bot, minlength=n) / np.maximum(cnt, 1)
|
| 137 |
+
score = np.bincount(idx, weights=sc, minlength=n) / np.maximum(cnt, 1)
|
| 138 |
+
boxes = []
|
| 139 |
+
for i in range(1, n):
|
| 140 |
+
if stats[i, cv2.CC_STAT_AREA] < min_area:
|
| 141 |
+
continue
|
| 142 |
+
lh = max(1.0, y1[i] - y0[i])
|
| 143 |
+
ex = min(CORE_X_SHRINK * lh, 0.5 * lh)
|
| 144 |
+
x0 = stats[i, cv2.CC_STAT_LEFT] * s - ex
|
| 145 |
+
x1 = (stats[i, cv2.CC_STAT_LEFT] + stats[i, cv2.CC_STAT_WIDTH]) * s + ex
|
| 146 |
+
boxes.append(
|
| 147 |
+
(
|
| 148 |
+
x0 - grow_x * lh,
|
| 149 |
+
y0[i] - grow_y * lh,
|
| 150 |
+
x1 + grow_x * lh,
|
| 151 |
+
y1[i] + grow_y * lh,
|
| 152 |
+
score[i],
|
| 153 |
+
)
|
| 154 |
+
)
|
| 155 |
+
return np.asarray(boxes, np.float32).reshape(-1, 5)
|
OCR_INDRA/indra/indra_ocr/document.py
ADDED
|
@@ -0,0 +1,291 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""Whole documents: the PDF text layer where it can be trusted, OCR elsewhere.
|
| 2 |
+
|
| 3 |
+
For each PDF page (``mode="auto"``):
|
| 4 |
+
|
| 5 |
+
1. Read the text layer with pdfium and check it (``textlayer.read_text_layer``). A page
|
| 6 |
+
without a sound layer (a scan, an OCR layer, a broken font map) goes to OCR.
|
| 7 |
+
2. The detector finds the visual lines, as in OCR. Each text-layer character
|
| 8 |
+
goes to the detected line that contains its center. Inside a line, the
|
| 9 |
+
characters of each text-layer run (pdfium's segment between line breaks) keep the PDF order
|
| 10 |
+
and spaces, and runs are ordered by x, right to left in a right-to-left line. So inline math, sub- and
|
| 11 |
+
superscripts and footnote marks land in their line, and a run across two
|
| 12 |
+
table cells splits between them. A detected line with no text (text inside
|
| 13 |
+
an image) is OCR'd when the recognizer reads the page's script.
|
| 14 |
+
3. The rest is the OCR path: lines split at wide gaps, reading order, layout
|
| 15 |
+
classes and markdown. The PDF content order is kept when it is coherent
|
| 16 |
+
(few upward jumps that are not column changes, and the order model agrees
|
| 17 |
+
with at least 85% of consecutive pairs).
|
| 18 |
+
|
| 19 |
+
``mode="ocr"`` never reads the text layer; ``mode="text"`` uses it even when
|
| 20 |
+
the checks fail (and OCR only on pages with no text at all).
|
| 21 |
+
"""
|
| 22 |
+
|
| 23 |
+
from __future__ import annotations
|
| 24 |
+
|
| 25 |
+
import time
|
| 26 |
+
import unicodedata
|
| 27 |
+
from pathlib import Path
|
| 28 |
+
|
| 29 |
+
import numpy as np
|
| 30 |
+
|
| 31 |
+
from .pipeline import InputError, OrtOCR, Page, is_pdf, read_images
|
| 32 |
+
from .textlayer import BROKEN_FONT_MAP, read_text_layer
|
| 33 |
+
|
| 34 |
+
CHECK_DPI = 72.0 # resolution of the render for the text-layer ink checks
|
| 35 |
+
MAX_PAGE_PIXELS = 100e6 # a page larger than this at the set dpi renders at a lower dpi (large scans reach ~65 MP)
|
| 36 |
+
|
| 37 |
+
READABLE_MIN = 0.9 # share of the layer's letters the recognizer must know
|
| 38 |
+
|
| 39 |
+
|
| 40 |
+
def readable(ocr: OrtOCR, runs: list) -> bool:
|
| 41 |
+
"""Whether the recognizer can read the script of these text-layer runs."""
|
| 42 |
+
letters = [c for r in runs for c in r.text if c.isalpha()]
|
| 43 |
+
if not letters:
|
| 44 |
+
return True
|
| 45 |
+
cjk = ocr.vocab_cjk.index if ocr.vocab_cjk is not None else {}
|
| 46 |
+
known = sum(c in ocr.vocab.index or c in cjk for c in letters)
|
| 47 |
+
return known >= READABLE_MIN * len(letters)
|
| 48 |
+
|
| 49 |
+
|
| 50 |
+
def right_to_left(tokens: list) -> bool:
|
| 51 |
+
"""Unicode bidi base direction: most strong characters are right-to-left."""
|
| 52 |
+
rtl = ltr = 0
|
| 53 |
+
for t in tokens:
|
| 54 |
+
b = unicodedata.bidirectional(t[0]) if len(t[0]) == 1 else ""
|
| 55 |
+
rtl += b in ("R", "AL")
|
| 56 |
+
ltr += b == "L"
|
| 57 |
+
return rtl > ltr
|
| 58 |
+
|
| 59 |
+
|
| 60 |
+
def _render(page, scale: float) -> np.ndarray:
|
| 61 |
+
arr = page.render(scale=scale, grayscale=True).to_numpy()
|
| 62 |
+
return np.ascontiguousarray(arr[..., 0] if arr.ndim == 3 else arr)
|
| 63 |
+
|
| 64 |
+
|
| 65 |
+
def assign_runs(runs: list, boxes: np.ndarray, k: float) -> tuple[list[list], int]:
|
| 66 |
+
"""Tokens (char, x, content index) per detected line.
|
| 67 |
+
|
| 68 |
+
A run is one text-layer line piece (``textlayer.TextLine``), in content
|
| 69 |
+
order. Each character goes to the smallest detected box that contains its
|
| 70 |
+
center (a space follows the character before it), so a run that spans two
|
| 71 |
+
table cells splits between them. Inside a detected line, each run's part
|
| 72 |
+
keeps the PDF character order, and the parts are ordered by x. ``boxes``
|
| 73 |
+
are pixels, ``k`` pixels per point. Returns (tokens per detected line,
|
| 74 |
+
characters outside every box)."""
|
| 75 |
+
parts: list[dict] = [{} for _ in range(len(boxes))] # line -> run idx -> tokens
|
| 76 |
+
lost = 0
|
| 77 |
+
if not len(boxes):
|
| 78 |
+
return [], sum(not c.isspace() for r in runs for c, _ in r.chars)
|
| 79 |
+
area = (boxes[:, 2] - boxes[:, 0]) * (boxes[:, 3] - boxes[:, 1])
|
| 80 |
+
for idx, run in enumerate(runs):
|
| 81 |
+
j = None
|
| 82 |
+
for c, b in run.chars:
|
| 83 |
+
if not c.isspace():
|
| 84 |
+
cx, cy = (b[0] + b[2]) / 2 * k, (b[1] + b[3]) / 2 * k
|
| 85 |
+
inside = (
|
| 86 |
+
(boxes[:, 0] <= cx)
|
| 87 |
+
& (cx <= boxes[:, 2])
|
| 88 |
+
& (boxes[:, 1] <= cy)
|
| 89 |
+
& (cy <= boxes[:, 3])
|
| 90 |
+
)
|
| 91 |
+
if not inside.any():
|
| 92 |
+
lost += 1
|
| 93 |
+
j = None
|
| 94 |
+
continue
|
| 95 |
+
j = int(np.flatnonzero(inside)[np.argmin(area[inside])])
|
| 96 |
+
parts[j].setdefault(idx, []).append((c, cx, idx))
|
| 97 |
+
elif j is not None:
|
| 98 |
+
prev = parts[j][idx][-1]
|
| 99 |
+
parts[j][idx].append((c, prev[1], idx))
|
| 100 |
+
toks: list[list] = []
|
| 101 |
+
for line in parts:
|
| 102 |
+
tk: list = []
|
| 103 |
+
rtl = right_to_left([t for part in line.values() for t in part])
|
| 104 |
+
order = (
|
| 105 |
+
sorted(line, key=lambda i: -max(t[1] for t in line[i]))
|
| 106 |
+
if rtl
|
| 107 |
+
else sorted(line, key=lambda i: min(t[1] for t in line[i]))
|
| 108 |
+
)
|
| 109 |
+
for idx in order:
|
| 110 |
+
if tk:
|
| 111 |
+
tk.append((" ", tk[-1][1], idx)) # parts of different runs
|
| 112 |
+
tk += line[idx]
|
| 113 |
+
toks.append(tk)
|
| 114 |
+
return toks, lost
|
| 115 |
+
|
| 116 |
+
|
| 117 |
+
class Converter:
|
| 118 |
+
def __init__(
|
| 119 |
+
self,
|
| 120 |
+
ocr: OrtOCR,
|
| 121 |
+
dpi: float = 200.0,
|
| 122 |
+
mode: str = "auto",
|
| 123 |
+
pdf_order: bool = True,
|
| 124 |
+
image_ocr: bool = True,
|
| 125 |
+
):
|
| 126 |
+
"""mode: auto (text layer where sound, OCR elsewhere), ocr (always OCR,
|
| 127 |
+
never read the text layer), text (text layer even when the checks fail).
|
| 128 |
+
pdf_order: keep the PDF content order when it is coherent (False: the
|
| 129 |
+
order model always orders the lines). image_ocr: OCR detected lines that
|
| 130 |
+
hold no text-layer characters (False: drop them)."""
|
| 131 |
+
if mode not in ("auto", "ocr", "text"):
|
| 132 |
+
raise ValueError(f"mode must be auto, ocr or text, not {mode!r}")
|
| 133 |
+
self.ocr, self.dpi, self.mode = ocr, dpi, mode
|
| 134 |
+
self.pdf_order, self.image_ocr = pdf_order, image_ocr
|
| 135 |
+
|
| 136 |
+
def page_scale(self, page) -> float:
|
| 137 |
+
"""Pixels per point for this page: the set dpi, lowered for pages so
|
| 138 |
+
large that the render would pass MAX_PAGE_PIXELS (posters, plans)."""
|
| 139 |
+
w, h = page.get_size()
|
| 140 |
+
k = self.dpi / 72.0
|
| 141 |
+
return min(k, (MAX_PAGE_PIXELS / max(w * h, 1.0)) ** 0.5)
|
| 142 |
+
|
| 143 |
+
def pdf_page(self, pdf_path: str, doc, index: int) -> Page:
|
| 144 |
+
page = doc[index]
|
| 145 |
+
t: dict = {}
|
| 146 |
+
note = "mode=ocr"
|
| 147 |
+
if self.mode != "ocr":
|
| 148 |
+
t0 = time.perf_counter()
|
| 149 |
+
s = CHECK_DPI / 72.0
|
| 150 |
+
tl = read_text_layer(page, _render(page, s), s)
|
| 151 |
+
t["text_ms"] = 1000 * (time.perf_counter() - t0)
|
| 152 |
+
# A layer the recognizer cannot read beats OCR, unless its codes are junk.
|
| 153 |
+
unreadable = tl.reason != BROKEN_FONT_MAP and not readable(
|
| 154 |
+
self.ocr, tl.lines
|
| 155 |
+
)
|
| 156 |
+
if tl.lines and (tl.trusted or self.mode == "text" or unreadable):
|
| 157 |
+
return self._text_page(page, tl, t)
|
| 158 |
+
note = tl.reason
|
| 159 |
+
# A layer with a broken font map still places its glyphs: when most
|
| 160 |
+
# run upright, the page is upright (orient.onnx is not asked).
|
| 161 |
+
st = tl.stats
|
| 162 |
+
turn = not (
|
| 163 |
+
tl.reason == BROKEN_FONT_MAP and st["rotated"] < 0.5 * st["chars"]
|
| 164 |
+
)
|
| 165 |
+
else:
|
| 166 |
+
turn = True
|
| 167 |
+
t0 = time.perf_counter()
|
| 168 |
+
gray = _render(page, self.page_scale(page))
|
| 169 |
+
t["render_ms"] = 1000 * (time.perf_counter() - t0)
|
| 170 |
+
out = self.ocr(gray, turn)
|
| 171 |
+
out.timings = {**t, **out.timings}
|
| 172 |
+
out.timings["total_ms"] = sum(
|
| 173 |
+
v for k, v in out.timings.items() if k != "total_ms"
|
| 174 |
+
)
|
| 175 |
+
out.note = f"{note}; {out.note}"
|
| 176 |
+
return out
|
| 177 |
+
|
| 178 |
+
def _text_page(self, page, tl, t: dict) -> Page:
|
| 179 |
+
k = self.page_scale(page)
|
| 180 |
+
t0 = time.perf_counter()
|
| 181 |
+
gray = _render(page, k)
|
| 182 |
+
t["render_ms"] = 1000 * (time.perf_counter() - t0)
|
| 183 |
+
t0 = time.perf_counter()
|
| 184 |
+
boxes = self.ocr.detect(gray)
|
| 185 |
+
t["det_ms"] = 1000 * (time.perf_counter() - t0)
|
| 186 |
+
toks, lost = assign_runs(tl.lines, boxes[:, :4], k)
|
| 187 |
+
rec: list = [None] * len(boxes)
|
| 188 |
+
empty = []
|
| 189 |
+
for i, tk in enumerate(toks):
|
| 190 |
+
text = self.ocr._decode(tk) if tk else ""
|
| 191 |
+
if text:
|
| 192 |
+
rec[i] = (text, 1.0, tk)
|
| 193 |
+
else:
|
| 194 |
+
empty.append(i)
|
| 195 |
+
source = "text"
|
| 196 |
+
if empty and self.image_ocr and readable(self.ocr, tl.lines):
|
| 197 |
+
t0 = time.perf_counter()
|
| 198 |
+
for i, r in zip(empty, self.ocr.recognize(gray, boxes[empty])):
|
| 199 |
+
if r[0].strip():
|
| 200 |
+
rec[i] = r
|
| 201 |
+
source = "hybrid"
|
| 202 |
+
t["rec_ms"] = 1000 * (time.perf_counter() - t0)
|
| 203 |
+
if getattr(self.ocr, "math", None) is not None:
|
| 204 |
+
t0 = time.perf_counter()
|
| 205 |
+
self._read_math(gray, boxes, rec)
|
| 206 |
+
t["math_ms"] = 1000 * (time.perf_counter() - t0)
|
| 207 |
+
keep = [i for i, r in enumerate(rec) if r is not None]
|
| 208 |
+
out = self.ocr.assemble(
|
| 209 |
+
(gray.shape[1], gray.shape[0]),
|
| 210 |
+
boxes[keep].reshape(-1, boxes.shape[1]),
|
| 211 |
+
[rec[i] for i in keep],
|
| 212 |
+
t,
|
| 213 |
+
content_order=self.pdf_order,
|
| 214 |
+
)
|
| 215 |
+
out.source = source
|
| 216 |
+
out.note += f"; {lost} chars outside detected lines" if lost else ""
|
| 217 |
+
return out
|
| 218 |
+
|
| 219 |
+
def _read_math(self, gray, boxes, rec: list) -> None:
|
| 220 |
+
"""Text-layer lines that hold math (the classifier, on their crops) are read
|
| 221 |
+
by MathRec instead: the layer has the symbols but not the LaTeX. The line
|
| 222 |
+
keeps its place in the PDF content order (its first character's index)."""
|
| 223 |
+
from .line_mask import crop_lines
|
| 224 |
+
|
| 225 |
+
idx = [
|
| 226 |
+
i
|
| 227 |
+
for i, r in enumerate(rec)
|
| 228 |
+
if r is not None
|
| 229 |
+
and r[2]
|
| 230 |
+
and isinstance(r[2][0][0], str)
|
| 231 |
+
and len(r[2][0]) > 2
|
| 232 |
+
]
|
| 233 |
+
if not idx:
|
| 234 |
+
return
|
| 235 |
+
crops = crop_lines(gray, boxes[idx, :4], self.ocr.height, None)
|
| 236 |
+
math = [i for i, f in zip(idx, self.ocr.math.flags(crops)) if f]
|
| 237 |
+
if not math:
|
| 238 |
+
return
|
| 239 |
+
from .math import MATH_MIN_CONF
|
| 240 |
+
|
| 241 |
+
for i, (text, conf) in zip(math, self.ocr.math.read(gray, boxes[math])):
|
| 242 |
+
# A reading without math, or not confident, leaves the layer's text.
|
| 243 |
+
if "$" in text and conf >= MATH_MIN_CONF:
|
| 244 |
+
first = min(tk[2] for tk in rec[i][2])
|
| 245 |
+
xc = float(boxes[i, 0] + boxes[i, 2]) / 2
|
| 246 |
+
rec[i] = (text, 1.0, [(text, xc, first)])
|
| 247 |
+
|
| 248 |
+
def convert(
|
| 249 |
+
self, path: str | Path, max_pages: int = 0, password: str | None = None
|
| 250 |
+
):
|
| 251 |
+
"""Yield (page index, Page) for a PDF or an image file (the file's bytes
|
| 252 |
+
decide which). A page that fails yields an empty Page whose ``source``
|
| 253 |
+
is "error" and whose ``note`` says why; a file that cannot be opened
|
| 254 |
+
raises InputError."""
|
| 255 |
+
path = Path(path)
|
| 256 |
+
if not path.is_file():
|
| 257 |
+
raise InputError(f"{path}: no such file")
|
| 258 |
+
if not is_pdf(path):
|
| 259 |
+
for i, gray in enumerate(read_images(path)[: max_pages or None]):
|
| 260 |
+
yield i, self._safe(lambda: self.ocr(gray))
|
| 261 |
+
return
|
| 262 |
+
import pypdfium2 as pdfium
|
| 263 |
+
|
| 264 |
+
try:
|
| 265 |
+
doc = pdfium.PdfDocument(str(path), password=password)
|
| 266 |
+
except pdfium.PdfiumError as e:
|
| 267 |
+
locked = "password" in str(e).lower()
|
| 268 |
+
raise InputError(
|
| 269 |
+
f"{path}: password-protected PDF (pass password=, or --password)"
|
| 270 |
+
if locked
|
| 271 |
+
else f"{path}: cannot open the PDF ({e})"
|
| 272 |
+
) from e
|
| 273 |
+
try:
|
| 274 |
+
n = len(doc) if not max_pages else min(len(doc), max_pages)
|
| 275 |
+
for i in range(n):
|
| 276 |
+
yield i, self._safe(lambda: self.pdf_page(str(path), doc, i))
|
| 277 |
+
finally:
|
| 278 |
+
doc.close()
|
| 279 |
+
|
| 280 |
+
@staticmethod
|
| 281 |
+
def _safe(read) -> Page:
|
| 282 |
+
try:
|
| 283 |
+
return read()
|
| 284 |
+
except Exception as e: # noqa: BLE001 (one bad page must not end the document)
|
| 285 |
+
return Page(
|
| 286 |
+
[],
|
| 287 |
+
{"total_ms": 0.0},
|
| 288 |
+
markdown="",
|
| 289 |
+
source="error",
|
| 290 |
+
note=f"{type(e).__name__}: {e}",
|
| 291 |
+
)
|
OCR_INDRA/indra/indra_ocr/layout_post.py
ADDED
|
@@ -0,0 +1,385 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""Page text from the ordered lines and the OrderNet layout heads (torch-free).
|
| 2 |
+
|
| 3 |
+
OrderNet with ``layout`` gives every line a block class and every pair of lines
|
| 4 |
+
a "same block" logit. Walking the reading order, a new block starts wherever
|
| 5 |
+
the same-block logit between consecutive lines is negative. Each block is then
|
| 6 |
+
rendered by its class:
|
| 7 |
+
|
| 8 |
+
Section-Header "## " + joined text
|
| 9 |
+
Text, Caption, ... one paragraph; a word split by a hyphen at a line end is joined
|
| 10 |
+
List-Group one line per item (a new item starts at a bullet or number)
|
| 11 |
+
Table an HTML table assembled from the cell lines (rows by vertical
|
| 12 |
+
centre, columns anchored on the fullest rows)
|
| 13 |
+
Page-Header / Footer left out unless ``keep_furniture``
|
| 14 |
+
"""
|
| 15 |
+
|
| 16 |
+
from __future__ import annotations
|
| 17 |
+
|
| 18 |
+
import html
|
| 19 |
+
import re
|
| 20 |
+
|
| 21 |
+
import numpy as np
|
| 22 |
+
|
| 23 |
+
from .order_post import BLOCK_CLASSES, _cluster
|
| 24 |
+
|
| 25 |
+
HEADER, FOOTER = BLOCK_CLASSES.index("Page-Header"), BLOCK_CLASSES.index("Page-Footer")
|
| 26 |
+
SECTION = BLOCK_CLASSES.index("Section-Header")
|
| 27 |
+
LIST = BLOCK_CLASSES.index("List-Group")
|
| 28 |
+
TABLE = BLOCK_CLASSES.index("Table")
|
| 29 |
+
|
| 30 |
+
_WHOLE_MATH = re.compile(r"^\$[^$]+\$$") # a line that is one formula
|
| 31 |
+
# A list marker, followed by text or alone on its line (the detector often cuts
|
| 32 |
+
# a bullet off as a line of its own; the next line then joins its item).
|
| 33 |
+
_ITEM = re.compile(r"^\s*([•\-–▪◦*·→]|\(?\d{1,3}[.)]|\(?[a-zA-Z][.)])(\s|$)")
|
| 34 |
+
|
| 35 |
+
|
| 36 |
+
_PLAIN_WORD = re.compile(r" [A-Za-z]{3,}\b(?![_^{(])")
|
| 37 |
+
|
| 38 |
+
|
| 39 |
+
def balance_math(text: str) -> str:
|
| 40 |
+
"""Close $ math spans the recognizer left open.
|
| 41 |
+
|
| 42 |
+
Left to right: a $ opens math only when LaTeX (\\ _ ^ {) or a closing $
|
| 43 |
+
follows within 15 characters, so prices ("$5 per item") stay text. Inside
|
| 44 |
+
math, a following plain word of 3+ letters closes the span; an open span
|
| 45 |
+
closes at the end of the line."""
|
| 46 |
+
if "$" not in text:
|
| 47 |
+
return text
|
| 48 |
+
out, in_math, i = [], False, 0
|
| 49 |
+
while i < len(text):
|
| 50 |
+
c = text[i]
|
| 51 |
+
if c == "$":
|
| 52 |
+
if in_math:
|
| 53 |
+
in_math = False
|
| 54 |
+
else:
|
| 55 |
+
ahead = text[i + 1 : i + 16]
|
| 56 |
+
in_math = bool(re.search(r"[\\_^{]", ahead)) or "$" in ahead
|
| 57 |
+
out.append(c)
|
| 58 |
+
i += 1
|
| 59 |
+
continue
|
| 60 |
+
if in_math and c == " " and _PLAIN_WORD.match(text, i):
|
| 61 |
+
out.append("$")
|
| 62 |
+
in_math = False
|
| 63 |
+
out.append(c)
|
| 64 |
+
i += 1
|
| 65 |
+
if in_math:
|
| 66 |
+
out.append("$")
|
| 67 |
+
return "".join(out)
|
| 68 |
+
|
| 69 |
+
|
| 70 |
+
def _cjk(c: str) -> bool:
|
| 71 |
+
"""Han, kana, CJK punctuation and fullwidth forms: written without spaces
|
| 72 |
+
(Korean Hangul is not: Korean separates words with spaces)."""
|
| 73 |
+
o = ord(c)
|
| 74 |
+
return (
|
| 75 |
+
0x3000 <= o <= 0x30FF
|
| 76 |
+
or 0x3400 <= o <= 0x9FFF
|
| 77 |
+
or 0xF900 <= o <= 0xFAFF
|
| 78 |
+
or 0xFF00 <= o <= 0xFFEF
|
| 79 |
+
or 0x20000 <= o <= 0x2FA1F
|
| 80 |
+
)
|
| 81 |
+
|
| 82 |
+
|
| 83 |
+
def join_lines(texts: list[str]) -> str:
|
| 84 |
+
"""Join the lines of one paragraph. 'tumori-' + 'genesis' -> 'tumorigenesis';
|
| 85 |
+
a line break between two CJK characters takes no space."""
|
| 86 |
+
out = ""
|
| 87 |
+
for t in texts:
|
| 88 |
+
t = t.strip()
|
| 89 |
+
if not t:
|
| 90 |
+
continue
|
| 91 |
+
if (
|
| 92 |
+
len(out) > 1
|
| 93 |
+
and out.endswith("-")
|
| 94 |
+
and out[-2].isalpha()
|
| 95 |
+
and t[0].isalpha()
|
| 96 |
+
and t[0].islower()
|
| 97 |
+
):
|
| 98 |
+
out = out[:-1] + t
|
| 99 |
+
elif out and _cjk(out[-1]) and _cjk(t[0]):
|
| 100 |
+
out += t
|
| 101 |
+
else:
|
| 102 |
+
out = f"{out} {t}" if out else t
|
| 103 |
+
return out
|
| 104 |
+
|
| 105 |
+
|
| 106 |
+
def split_blocks(order: list[int], same: np.ndarray) -> list[list[int]]:
|
| 107 |
+
"""Runs of the reading order with a positive same-block logit between neighbours."""
|
| 108 |
+
blocks: list[list[int]] = []
|
| 109 |
+
for i in order:
|
| 110 |
+
if blocks and same[blocks[-1][-1], i] > 0:
|
| 111 |
+
blocks[-1].append(i)
|
| 112 |
+
else:
|
| 113 |
+
blocks.append([i])
|
| 114 |
+
return blocks
|
| 115 |
+
|
| 116 |
+
|
| 117 |
+
def _rows(boxes: np.ndarray) -> list[list[int]]:
|
| 118 |
+
"""Group cell lines into rows by vertical centre, top to bottom, left to right."""
|
| 119 |
+
cy = (boxes[:, 1] + boxes[:, 3]) / 2
|
| 120 |
+
h = float(np.median(boxes[:, 3] - boxes[:, 1]))
|
| 121 |
+
rows: list[list[int]] = []
|
| 122 |
+
for i in np.argsort(cy):
|
| 123 |
+
if rows and abs(cy[i] - np.mean(cy[rows[-1]])) < 0.5 * h:
|
| 124 |
+
rows[-1].append(int(i))
|
| 125 |
+
else:
|
| 126 |
+
rows.append([int(i)])
|
| 127 |
+
return [sorted(r, key=lambda k: boxes[k, 0]) for r in rows]
|
| 128 |
+
|
| 129 |
+
|
| 130 |
+
def _merge_wrapped(rows: list[list[int]], boxes: np.ndarray) -> list[list[int]]:
|
| 131 |
+
"""Join the line rows of wrapped cells into one table row.
|
| 132 |
+
|
| 133 |
+
Inside a wrapped cell the gap to the next line is the font's line gap; a new
|
| 134 |
+
table row adds cell padding. A row joins the row above when its gap is under
|
| 135 |
+
half the table's median row gap and under 0.6 line heights, and it has fewer
|
| 136 |
+
lines than the row above (a continuation fills only the wrapped columns)."""
|
| 137 |
+
if len(rows) < 3:
|
| 138 |
+
return rows
|
| 139 |
+
h = float(np.median(boxes[:, 3] - boxes[:, 1]))
|
| 140 |
+
gaps = [
|
| 141 |
+
float(boxes[nxt, 1].min() - boxes[cur, 3].max())
|
| 142 |
+
for cur, nxt in zip(rows[:-1], rows[1:])
|
| 143 |
+
]
|
| 144 |
+
med = float(np.median(gaps))
|
| 145 |
+
out = [list(rows[0])]
|
| 146 |
+
prev = rows[0]
|
| 147 |
+
for g, r in zip(gaps, rows[1:]):
|
| 148 |
+
# A continuation fills only the wrapped columns: fewer lines than the row above.
|
| 149 |
+
if g < 0.5 * med and g < 0.6 * h and len(r) < len(prev):
|
| 150 |
+
out[-1] += r
|
| 151 |
+
else:
|
| 152 |
+
out.append(list(r))
|
| 153 |
+
prev = r
|
| 154 |
+
return out
|
| 155 |
+
|
| 156 |
+
|
| 157 |
+
def table_html(boxes: np.ndarray, texts: list[str]) -> str:
|
| 158 |
+
"""An HTML table from cell lines (one text line per cell, as born-digital PDFs give).
|
| 159 |
+
|
| 160 |
+
A first row that is one line across most of the table is a title: it goes
|
| 161 |
+
before the table as text. The first remaining row is the header (<th>)."""
|
| 162 |
+
rows = _rows(boxes)
|
| 163 |
+
span_x = boxes[:, 2].max() - boxes[:, 0].min()
|
| 164 |
+
if (
|
| 165 |
+
len(rows) > 2
|
| 166 |
+
and len(rows[0]) == 1
|
| 167 |
+
and max(len(r) for r in rows) > 1
|
| 168 |
+
and boxes[rows[0][0], 2] - boxes[rows[0][0], 0] > 0.6 * span_x
|
| 169 |
+
):
|
| 170 |
+
keep = [k for k in range(len(texts)) if k != rows[0][0]]
|
| 171 |
+
return (
|
| 172 |
+
texts[rows[0][0]]
|
| 173 |
+
+ "\n\n"
|
| 174 |
+
+ table_html(boxes[keep], [texts[k] for k in keep])
|
| 175 |
+
)
|
| 176 |
+
width = max(len(r) for r in rows)
|
| 177 |
+
full = [r for r in rows if len(r) == width]
|
| 178 |
+
rows = _merge_wrapped(rows, boxes)
|
| 179 |
+
# Column bands from the rows with the most cells: those rows rarely span.
|
| 180 |
+
bands = np.array(
|
| 181 |
+
[
|
| 182 |
+
[
|
| 183 |
+
np.median([boxes[r[c], 0] for r in full]),
|
| 184 |
+
np.median([boxes[r[c], 2] for r in full]),
|
| 185 |
+
]
|
| 186 |
+
for c in range(width)
|
| 187 |
+
]
|
| 188 |
+
)
|
| 189 |
+
# Widen each band to the midpoints between neighbours, so every x falls in one band.
|
| 190 |
+
edges = (
|
| 191 |
+
[-np.inf]
|
| 192 |
+
+ [(bands[c, 1] + bands[c + 1, 0]) / 2 for c in range(width - 1)]
|
| 193 |
+
+ [np.inf]
|
| 194 |
+
)
|
| 195 |
+
out = ["<table>"]
|
| 196 |
+
for ri, r in enumerate(rows):
|
| 197 |
+
tag = "th" if ri == 0 and len(rows) > 1 else "td"
|
| 198 |
+
cells: dict[int, tuple[int, str]] = {}
|
| 199 |
+
for k in sorted(r, key=lambda k: (boxes[k, 1], boxes[k, 0])):
|
| 200 |
+
x0, x1 = boxes[k, 0], boxes[k, 2]
|
| 201 |
+
first = int(np.searchsorted(edges, x0, side="right") - 1)
|
| 202 |
+
last = int(np.searchsorted(edges, x1, side="right") - 1)
|
| 203 |
+
# A cell spans every band it covers by more than half of the band.
|
| 204 |
+
span = [
|
| 205 |
+
c
|
| 206 |
+
for c in range(max(first, 0), min(last, width - 1) + 1)
|
| 207 |
+
if min(x1, bands[c, 1]) - max(x0, bands[c, 0])
|
| 208 |
+
> 0.5 * (bands[c, 1] - bands[c, 0])
|
| 209 |
+
] or [min(max(first, 0), width - 1)]
|
| 210 |
+
c0 = span[0]
|
| 211 |
+
prev = cells.get(c0)
|
| 212 |
+
text = texts[k] if prev is None else prev[1] + " " + texts[k]
|
| 213 |
+
cells[c0] = (len(span) if prev is None else max(prev[0], len(span)), text)
|
| 214 |
+
tds, c = [], 0
|
| 215 |
+
while c < width:
|
| 216 |
+
if c in cells:
|
| 217 |
+
n, text = cells[c]
|
| 218 |
+
attr = f' colspan="{n}"' if n > 1 else ""
|
| 219 |
+
tds.append(f"<{tag}{attr}>{html.escape(text, quote=False)}</{tag}>")
|
| 220 |
+
c += n
|
| 221 |
+
else:
|
| 222 |
+
tds.append(f"<{tag}></{tag}>")
|
| 223 |
+
c += 1
|
| 224 |
+
out.append("<tr>" + "".join(tds) + "</tr>")
|
| 225 |
+
out.append("</table>")
|
| 226 |
+
return "\n".join(out)
|
| 227 |
+
|
| 228 |
+
|
| 229 |
+
def heads_plausible(boxes: np.ndarray, tab: np.ndarray) -> bool:
|
| 230 |
+
"""The lines of one cell are stacked (they overlap horizontally). When the
|
| 231 |
+
cell head joins side-by-side lines in more than 10% of its cells, it has
|
| 232 |
+
failed on this table and the geometric assembly is used instead."""
|
| 233 |
+
cells = _cluster(tab[2])
|
| 234 |
+
bad = 0
|
| 235 |
+
for c in cells:
|
| 236 |
+
if len(c) < 2:
|
| 237 |
+
continue
|
| 238 |
+
x0, x1 = boxes[c, 0], boxes[c, 2]
|
| 239 |
+
if (np.minimum.outer(x1, x1) - np.maximum.outer(x0, x0)).min() < 0:
|
| 240 |
+
bad += 1
|
| 241 |
+
return bad <= 0.1 * len(cells)
|
| 242 |
+
|
| 243 |
+
|
| 244 |
+
def table_html_heads(boxes: np.ndarray, texts: list[str], tab: np.ndarray) -> str:
|
| 245 |
+
"""An HTML table from the OrderNet table heads. tab [3, N, N] = logits
|
| 246 |
+
same row, same column, same cell, for the N lines of one table block."""
|
| 247 |
+
cells = _cluster(tab[2])
|
| 248 |
+
cb = np.array(
|
| 249 |
+
[
|
| 250 |
+
[boxes[c, 0].min(), boxes[c, 1].min(), boxes[c, 2].max(), boxes[c, 3].max()]
|
| 251 |
+
for c in cells
|
| 252 |
+
]
|
| 253 |
+
)
|
| 254 |
+
ctext = [
|
| 255 |
+
" ".join(texts[k] for k in sorted(c, key=lambda k: (boxes[k, 1], boxes[k, 0])))
|
| 256 |
+
for c in cells
|
| 257 |
+
]
|
| 258 |
+
m = len(cells)
|
| 259 |
+
|
| 260 |
+
def cell_logits(k: int) -> np.ndarray:
|
| 261 |
+
out = np.zeros((m, m), np.float32)
|
| 262 |
+
for a in range(m):
|
| 263 |
+
for b in range(a + 1, m):
|
| 264 |
+
out[a, b] = out[b, a] = tab[k][np.ix_(cells[a], cells[b])].mean()
|
| 265 |
+
return out
|
| 266 |
+
|
| 267 |
+
rows = sorted(_cluster(cell_logits(0)), key=lambda g: cb[g, 1].min())
|
| 268 |
+
cols = sorted(_cluster(cell_logits(1)), key=lambda g: cb[g, 0].min())
|
| 269 |
+
row_of = {c: r for r, g in enumerate(rows) for c in g}
|
| 270 |
+
col_of = {c: k for k, g in enumerate(cols) for c in g}
|
| 271 |
+
centers = [float(np.median((cb[g, 0] + cb[g, 2]) / 2)) for g in cols]
|
| 272 |
+
h = float(np.median(boxes[:, 3] - boxes[:, 1]))
|
| 273 |
+
grid: dict[tuple[int, int], tuple[int, str]] = {}
|
| 274 |
+
for c in range(m):
|
| 275 |
+
c0 = col_of[c]
|
| 276 |
+
span = 1
|
| 277 |
+
while (
|
| 278 |
+
c0 + span < len(cols) and cb[c, 0] - h <= centers[c0 + span] <= cb[c, 2] + h
|
| 279 |
+
):
|
| 280 |
+
span += 1
|
| 281 |
+
key = (row_of[c], c0)
|
| 282 |
+
if key in grid: # two cells on one slot: keep both texts
|
| 283 |
+
grid[key] = (max(grid[key][0], span), grid[key][1] + " " + ctext[c])
|
| 284 |
+
else:
|
| 285 |
+
grid[key] = (span, ctext[c])
|
| 286 |
+
out = ["<table>"]
|
| 287 |
+
for r in range(len(rows)):
|
| 288 |
+
tag = "th" if r == 0 and len(rows) > 1 else "td"
|
| 289 |
+
tds, c = [], 0
|
| 290 |
+
while c < len(cols):
|
| 291 |
+
if (r, c) in grid:
|
| 292 |
+
span, text = grid[(r, c)]
|
| 293 |
+
attr = f' colspan="{span}"' if span > 1 else ""
|
| 294 |
+
tds.append(f"<{tag}{attr}>{html.escape(text, quote=False)}</{tag}>")
|
| 295 |
+
c += span
|
| 296 |
+
else:
|
| 297 |
+
tds.append(f"<{tag}></{tag}>")
|
| 298 |
+
c += 1
|
| 299 |
+
out.append("<tr>" + "".join(tds) + "</tr>")
|
| 300 |
+
out.append("</table>")
|
| 301 |
+
return "\n".join(out)
|
| 302 |
+
|
| 303 |
+
|
| 304 |
+
def page_blocks(
|
| 305 |
+
boxes: np.ndarray,
|
| 306 |
+
texts: list[str],
|
| 307 |
+
order: list[int],
|
| 308 |
+
cls_logits: np.ndarray,
|
| 309 |
+
same: np.ndarray,
|
| 310 |
+
) -> list[dict]:
|
| 311 |
+
"""The layout blocks behind ``page_markdown``, in reading order:
|
| 312 |
+
{"label" (a BLOCK_CLASSES name), "bbox" (pixels, the union of its lines),
|
| 313 |
+
"text" (its lines, newline-joined)}."""
|
| 314 |
+
out = []
|
| 315 |
+
for blk in split_blocks(order, same):
|
| 316 |
+
b = boxes[blk, :4]
|
| 317 |
+
out.append(
|
| 318 |
+
{
|
| 319 |
+
"label": BLOCK_CLASSES[int(cls_logits[blk].sum(0).argmax())],
|
| 320 |
+
"bbox": [
|
| 321 |
+
round(float(b[:, 0].min()), 1),
|
| 322 |
+
round(float(b[:, 1].min()), 1),
|
| 323 |
+
round(float(b[:, 2].max()), 1),
|
| 324 |
+
round(float(b[:, 3].max()), 1),
|
| 325 |
+
],
|
| 326 |
+
"text": "\n".join(texts[i] for i in blk),
|
| 327 |
+
}
|
| 328 |
+
)
|
| 329 |
+
return out
|
| 330 |
+
|
| 331 |
+
|
| 332 |
+
def page_markdown(
|
| 333 |
+
boxes: np.ndarray,
|
| 334 |
+
texts: list[str],
|
| 335 |
+
order: list[int],
|
| 336 |
+
cls_logits: np.ndarray,
|
| 337 |
+
same: np.ndarray,
|
| 338 |
+
keep_furniture: bool = False,
|
| 339 |
+
cells: dict | None = None,
|
| 340 |
+
tab: np.ndarray | None = None,
|
| 341 |
+
) -> str:
|
| 342 |
+
"""Markdown for one page. boxes [N, 4], texts [N], order = reading order,
|
| 343 |
+
cls_logits [N, C], same [N, N] (logit "same block"). ``cells`` maps a line
|
| 344 |
+
to its table cells [(box, text)] when the line holds several. ``tab``
|
| 345 |
+
[3, N, N] (same row / column / cell logits) builds tables from the heads."""
|
| 346 |
+
parts = []
|
| 347 |
+
for blk in split_blocks(order, same):
|
| 348 |
+
cls = int(cls_logits[blk].sum(0).argmax())
|
| 349 |
+
texts_b = [
|
| 350 |
+
texts[i] for i in blk
|
| 351 |
+
] # balance_math measured worse on olmOCR arxiv_math
|
| 352 |
+
if cls in (HEADER, FOOTER) and not keep_furniture:
|
| 353 |
+
continue
|
| 354 |
+
if all(_WHOLE_MATH.match(t.strip()) for t in texts_b): # display equations
|
| 355 |
+
parts.append("\n".join("$$" + t.strip()[1:-1] + "$$" for t in texts_b))
|
| 356 |
+
continue
|
| 357 |
+
if cls == SECTION:
|
| 358 |
+
parts.append("## " + join_lines(texts_b))
|
| 359 |
+
elif (
|
| 360 |
+
cls == TABLE
|
| 361 |
+
and len(blk) > 1
|
| 362 |
+
and tab is not None
|
| 363 |
+
and heads_plausible(boxes[blk, :4], tab[:, blk][:, :, blk])
|
| 364 |
+
):
|
| 365 |
+
parts.append(
|
| 366 |
+
table_html_heads(boxes[blk, :4], texts_b, tab[:, blk][:, :, blk])
|
| 367 |
+
)
|
| 368 |
+
elif cls == TABLE and len(blk) > 1:
|
| 369 |
+
cb, ct = [], []
|
| 370 |
+
for i in blk:
|
| 371 |
+
for bx, tx in (cells or {}).get(i) or [(boxes[i, :4], texts[i])]:
|
| 372 |
+
cb.append(bx)
|
| 373 |
+
ct.append(tx)
|
| 374 |
+
parts.append(table_html(np.array(cb, np.float32).reshape(-1, 4), ct))
|
| 375 |
+
elif cls == LIST:
|
| 376 |
+
items: list[list[str]] = []
|
| 377 |
+
for t in texts_b:
|
| 378 |
+
if not items or _ITEM.match(t):
|
| 379 |
+
items.append([t])
|
| 380 |
+
else:
|
| 381 |
+
items[-1].append(t)
|
| 382 |
+
parts.append("\n".join(join_lines(it) for it in items))
|
| 383 |
+
else:
|
| 384 |
+
parts.append(join_lines(texts_b))
|
| 385 |
+
return "\n\n".join(p for p in parts if p)
|
OCR_INDRA/indra/indra_ocr/line_mask.py
ADDED
|
@@ -0,0 +1,129 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""Mask a line window with CenterMask before the recognizer crops the line.
|
| 2 |
+
|
| 3 |
+
One function for training and inference: the recognizer must only ever see
|
| 4 |
+
crops that went through the same exported mask model (mask.onnx).
|
| 5 |
+
|
| 6 |
+
sess = load_mask("mask.onnx")
|
| 7 |
+
win, box_in_win = masked_window(gray, box, sess)
|
| 8 |
+
img = crop_line(win, box_in_win, height)
|
| 9 |
+
"""
|
| 10 |
+
|
| 11 |
+
from __future__ import annotations
|
| 12 |
+
|
| 13 |
+
import numpy as np
|
| 14 |
+
|
| 15 |
+
WINDOW_H = 96 # window height in pixels; the centre line is about a third of it
|
| 16 |
+
WINDOW_ABOVE = 1.0 # box heights above and below the line box
|
| 17 |
+
WINDOW_SIDE = 0.5 # box heights left and right
|
| 18 |
+
|
| 19 |
+
|
| 20 |
+
def window_box(box, w_page: float, h_page: float) -> tuple[float, float, float, float]:
|
| 21 |
+
"""The window around a line box (page pixels), clipped to the page."""
|
| 22 |
+
x0, y0, x1, y1 = (float(v) for v in box[:4])
|
| 23 |
+
h = max(1.0, y1 - y0)
|
| 24 |
+
return (
|
| 25 |
+
max(0.0, x0 - WINDOW_SIDE * h),
|
| 26 |
+
max(0.0, y0 - WINDOW_ABOVE * h),
|
| 27 |
+
min(w_page, x1 + WINDOW_SIDE * h),
|
| 28 |
+
min(h_page, y1 + WINDOW_ABOVE * h),
|
| 29 |
+
)
|
| 30 |
+
|
| 31 |
+
|
| 32 |
+
KEEP = 0.5 # pixels with a lower centre-line probability become paper
|
| 33 |
+
|
| 34 |
+
|
| 35 |
+
def resize_window(a: np.ndarray, size: tuple[int, int]) -> np.ndarray:
|
| 36 |
+
"""Scale a window to ``size`` (w, h): linear down to half scale, area below
|
| 37 |
+
(anti-aliasing). INTER_AREA at a non-integer scale costs 20x more."""
|
| 38 |
+
import cv2
|
| 39 |
+
|
| 40 |
+
shrink = size[1] / max(1, a.shape[0])
|
| 41 |
+
interp = cv2.INTER_LINEAR if shrink >= 0.5 else cv2.INTER_AREA
|
| 42 |
+
return cv2.resize(a, size, interpolation=interp)
|
| 43 |
+
|
| 44 |
+
|
| 45 |
+
def load_mask(path: str, threads: int = 1):
|
| 46 |
+
import onnxruntime as ort
|
| 47 |
+
|
| 48 |
+
so = ort.SessionOptions()
|
| 49 |
+
so.intra_op_num_threads = threads
|
| 50 |
+
so.inter_op_num_threads = 1
|
| 51 |
+
return ort.InferenceSession(str(path), so, providers=["CPUExecutionProvider"])
|
| 52 |
+
|
| 53 |
+
|
| 54 |
+
def _window(gray: np.ndarray, box) -> tuple[np.ndarray, tuple[float, float]]:
|
| 55 |
+
H, W = gray.shape
|
| 56 |
+
x0, y0, x1, y1 = (int(round(v)) for v in window_box(box, W, H))
|
| 57 |
+
return gray[y0:y1, x0:x1], (x0, y0)
|
| 58 |
+
|
| 59 |
+
|
| 60 |
+
def mask_windows(sess, wins: list[np.ndarray]) -> list[np.ndarray]:
|
| 61 |
+
"""Centre-line keep masks (bool, window resolution) for grayscale windows.
|
| 62 |
+
|
| 63 |
+
The probabilities are scaled back bilinearly before the threshold: a
|
| 64 |
+
threshold at mask resolution with a nearest-neighbour upscale breaks the
|
| 65 |
+
stroke edges (surya OCR 2 CER on M804 lines 0.30 -> 0.36)."""
|
| 66 |
+
import cv2
|
| 67 |
+
|
| 68 |
+
if not wins:
|
| 69 |
+
return []
|
| 70 |
+
xs, sizes = [], []
|
| 71 |
+
for w in wins:
|
| 72 |
+
h, wd = w.shape
|
| 73 |
+
W = max(32, int(round(wd * WINDOW_H / max(h, 1) / 8)) * 8)
|
| 74 |
+
ink = 1.0 - resize_window(w, (W, WINDOW_H)).astype(np.float32) / 255.0
|
| 75 |
+
xs.append(ink)
|
| 76 |
+
sizes.append((wd, h))
|
| 77 |
+
out: list = [None] * len(wins)
|
| 78 |
+
order = np.argsort([x.shape[1] for x in xs])
|
| 79 |
+
for k in range(0, len(order), 16): # similar widths together: less padding
|
| 80 |
+
idx = order[k : k + 16]
|
| 81 |
+
Wm = max(xs[i].shape[1] for i in idx)
|
| 82 |
+
batch = np.zeros((len(idx), 1, WINDOW_H, Wm), np.float32)
|
| 83 |
+
for j, i in enumerate(idx):
|
| 84 |
+
batch[j, 0, :, : xs[i].shape[1]] = xs[i]
|
| 85 |
+
p = sess.run(None, {"x": batch})[0]
|
| 86 |
+
for j, i in enumerate(idx):
|
| 87 |
+
pm = p[j, 0, :, : xs[i].shape[1]]
|
| 88 |
+
out[i] = cv2.resize(pm, sizes[i], interpolation=cv2.INTER_LINEAR) >= KEEP
|
| 89 |
+
return out
|
| 90 |
+
|
| 91 |
+
|
| 92 |
+
def masked_windows(
|
| 93 |
+
gray: np.ndarray, boxes, sess
|
| 94 |
+
) -> list[tuple[np.ndarray, list[float]]]:
|
| 95 |
+
"""For each line box: (window with foreign ink whitened, the box inside it)."""
|
| 96 |
+
wins, offs = zip(*[_window(gray, b) for b in boxes]) if len(boxes) else ((), ())
|
| 97 |
+
out = []
|
| 98 |
+
for w, (ox, oy), keep, b in zip(wins, offs, mask_windows(sess, list(wins)), boxes):
|
| 99 |
+
m = w.copy()
|
| 100 |
+
# paper level: 90th percentile grey on a 1/16 subsample of the window
|
| 101 |
+
m[~keep] = np.uint8(np.percentile(w[::4, ::4], 90)) if w.size else 255
|
| 102 |
+
out.append((m, [b[0] - ox, b[1] - oy, b[2] - ox, b[3] - oy]))
|
| 103 |
+
return out
|
| 104 |
+
|
| 105 |
+
|
| 106 |
+
def masked_window(gray: np.ndarray, box, sess) -> tuple[np.ndarray, list[float]]:
|
| 107 |
+
return masked_windows(gray, [box], sess)[0]
|
| 108 |
+
|
| 109 |
+
|
| 110 |
+
def crop_lines(gray: np.ndarray, boxes, height: int, sess=None) -> list[np.ndarray]:
|
| 111 |
+
"""Recognizer inputs for the line boxes of one page: ``crop_line`` of each
|
| 112 |
+
line, after CenterMask when ``sess`` is given (one batched call per page)."""
|
| 113 |
+
from .rec_preproc import crop_line
|
| 114 |
+
|
| 115 |
+
if sess is None:
|
| 116 |
+
return [crop_line(gray, b, height) for b in boxes]
|
| 117 |
+
return [crop_line(w, b, height) for w, b in masked_windows(gray, list(boxes), sess)]
|
| 118 |
+
|
| 119 |
+
|
| 120 |
+
_SESSIONS: dict = {}
|
| 121 |
+
|
| 122 |
+
|
| 123 |
+
def session_for(path: str | None):
|
| 124 |
+
"""One CPU session per process (dataloader workers call this lazily)."""
|
| 125 |
+
if not path:
|
| 126 |
+
return None
|
| 127 |
+
if path not in _SESSIONS:
|
| 128 |
+
_SESSIONS[path] = load_mask(path, threads=1)
|
| 129 |
+
return _SESSIONS[path]
|
OCR_INDRA/indra/indra_ocr/math.py
ADDED
|
@@ -0,0 +1,145 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""Math lines: a classifier finds lines that hold math, and MathRec reads them as
|
| 2 |
+
text with inline LaTeX ("we have $x _ { i } \\leq 1$").
|
| 3 |
+
|
| 4 |
+
The CTC recognizer reads one symbol per column, so it cannot write a fraction or
|
| 5 |
+
a sum with limits. MathRec reads a taller crop of the line with an attention
|
| 6 |
+
decoder, one token per step, with the decoder's keys and values cached.
|
| 7 |
+
|
| 8 |
+
Models (all optional; the pipeline skips math without them): mathcls.onnx,
|
| 9 |
+
math_enc.onnx, math_dec_init.onnx, math_dec_step.onnx and math_meta.json.
|
| 10 |
+
"""
|
| 11 |
+
|
| 12 |
+
from __future__ import annotations
|
| 13 |
+
|
| 14 |
+
import json
|
| 15 |
+
from pathlib import Path
|
| 16 |
+
|
| 17 |
+
import numpy as np
|
| 18 |
+
|
| 19 |
+
from .line_mask import crop_lines
|
| 20 |
+
from .rec_preproc import pad_batch, width_batches
|
| 21 |
+
|
| 22 |
+
MATH_MIN = 0.5 # classifier probability above which a line is read as math
|
| 23 |
+
# A math reading replaces a line only at this mean token probability or more: on
|
| 24 |
+
# print the model has not seen (old scans) it writes confident-looking LaTeX
|
| 25 |
+
# that is wrong, at low probability.
|
| 26 |
+
MATH_MIN_CONF = 0.9
|
| 27 |
+
# A line whose mean token probability is this far below MATH_MIN_CONF after 8
|
| 28 |
+
# tokens stops decoding: it would be rejected anyway (garbage on unseen print).
|
| 29 |
+
EARLY_STOP_MARGIN = 0.1
|
| 30 |
+
BOS, EOS, N_SPECIAL = 1, 2, 4
|
| 31 |
+
FILES = ("mathcls.onnx", "math_enc.onnx", "math_dec_init.onnx", "math_dec_step.onnx")
|
| 32 |
+
|
| 33 |
+
|
| 34 |
+
def detokenize(tokens: list[str]) -> str:
|
| 35 |
+
"""Tokens -> text; inside math (between "$" tokens) they are joined by spaces."""
|
| 36 |
+
out, math, in_math = [], [], False
|
| 37 |
+
for t in tokens:
|
| 38 |
+
if t == "$":
|
| 39 |
+
if in_math:
|
| 40 |
+
out.append("$" + " ".join(math) + "$")
|
| 41 |
+
math = []
|
| 42 |
+
in_math = not in_math
|
| 43 |
+
elif in_math:
|
| 44 |
+
math.append(t)
|
| 45 |
+
else:
|
| 46 |
+
out.append(t)
|
| 47 |
+
if in_math and math:
|
| 48 |
+
out.append("$" + " ".join(math) + "$")
|
| 49 |
+
return " ".join("".join(out).split())
|
| 50 |
+
|
| 51 |
+
|
| 52 |
+
class MathReader:
|
| 53 |
+
def __init__(self, onnx_dir: Path, so, providers):
|
| 54 |
+
import onnxruntime as ort
|
| 55 |
+
|
| 56 |
+
s = {
|
| 57 |
+
f: ort.InferenceSession(str(onnx_dir / f), so, providers=providers)
|
| 58 |
+
for f in FILES
|
| 59 |
+
}
|
| 60 |
+
self.cls, self.enc = s["mathcls.onnx"], s["math_enc.onnx"]
|
| 61 |
+
self.init, self.step = s["math_dec_init.onnx"], s["math_dec_step.onnx"]
|
| 62 |
+
meta = json.loads((onnx_dir / "math_meta.json").read_text())
|
| 63 |
+
cfg = meta["config"]
|
| 64 |
+
self.height = cfg["height"]
|
| 65 |
+
self.max_tokens = cfg["max_tokens"]
|
| 66 |
+
self.layers, self.heads = cfg["dec_layers"], cfg["dec_heads"]
|
| 67 |
+
self.head_dim = cfg["channels"] // cfg["dec_heads"]
|
| 68 |
+
self.itos = ["", "", "", ""] + meta["tokens"]
|
| 69 |
+
# Exports drop unused inputs (the init graph does not read the mask).
|
| 70 |
+
self.inputs = {f: {i.name for i in v.get_inputs()} for f, v in s.items()}
|
| 71 |
+
|
| 72 |
+
@classmethod
|
| 73 |
+
def load(cls, onnx_dir: Path, so, providers) -> "MathReader | None":
|
| 74 |
+
names = (*FILES, "math_meta.json")
|
| 75 |
+
if all((onnx_dir / f).exists() for f in names):
|
| 76 |
+
return cls(onnx_dir, so, providers)
|
| 77 |
+
return None
|
| 78 |
+
|
| 79 |
+
def flags(self, crops: list, batch: int = 64) -> list[bool]:
|
| 80 |
+
"""Per line crop (the recognizer's crop): does it hold math?"""
|
| 81 |
+
out = [False] * len(crops)
|
| 82 |
+
for idx in width_batches([c.shape[1] for c in crops], batch):
|
| 83 |
+
arr, widths = pad_batch([crops[j] for j in idx])
|
| 84 |
+
p = self.cls.run(None, {"x": arr, "widths": widths})[0]
|
| 85 |
+
for k, j in enumerate(idx):
|
| 86 |
+
out[j] = bool(p[k] > MATH_MIN)
|
| 87 |
+
return out
|
| 88 |
+
|
| 89 |
+
def read(
|
| 90 |
+
self, gray: np.ndarray, boxes: np.ndarray, batch: int = 64
|
| 91 |
+
) -> list[tuple[str, float]]:
|
| 92 |
+
"""(text with inline LaTeX, mean token probability) for each box."""
|
| 93 |
+
crops = crop_lines(gray, boxes[:, :4], self.height, None)
|
| 94 |
+
out = [("", 0.0)] * len(crops)
|
| 95 |
+
# Few, wide batches: each batch is one decoding loop of up to max_tokens
|
| 96 |
+
# steps, and a step costs about the same for 8 lines as for 40.
|
| 97 |
+
widths = [c.shape[1] for c in crops]
|
| 98 |
+
for idx in width_batches(widths, batch, max_ratio=4.0, max_area=16 * 64 * 640):
|
| 99 |
+
arr, widths = pad_batch([crops[j] for j in idx])
|
| 100 |
+
mem, mask = self.enc.run(None, {"x": arr, "widths": widths})
|
| 101 |
+
feed = {"memory": mem, "mask": mask}
|
| 102 |
+
names = self.inputs["math_dec_init.onnx"]
|
| 103 |
+
ck, cv = self.init.run(None, {k: v for k, v in feed.items() if k in names})
|
| 104 |
+
B = len(idx)
|
| 105 |
+
sk = np.zeros((self.layers, B, self.heads, 0, self.head_dim), np.float32)
|
| 106 |
+
sv = sk
|
| 107 |
+
tok = np.full(B, BOS, np.int64)
|
| 108 |
+
step_in = self.inputs["math_dec_step.onnx"]
|
| 109 |
+
seqs: list[list[int]] = [[] for _ in range(B)]
|
| 110 |
+
probs: list[list[float]] = [[] for _ in range(B)]
|
| 111 |
+
done = np.zeros(B, bool)
|
| 112 |
+
for t in range(self.max_tokens - 1):
|
| 113 |
+
feed = {
|
| 114 |
+
"token": tok,
|
| 115 |
+
"pos": np.array(t, np.int64),
|
| 116 |
+
"self_k": sk,
|
| 117 |
+
"self_v": sv,
|
| 118 |
+
"cross_k": ck,
|
| 119 |
+
"cross_v": cv,
|
| 120 |
+
"mask": mask,
|
| 121 |
+
}
|
| 122 |
+
logits, sk, sv = self.step.run(
|
| 123 |
+
None, {k: v for k, v in feed.items() if k in step_in}
|
| 124 |
+
)
|
| 125 |
+
tok = logits.argmax(-1).astype(np.int64)
|
| 126 |
+
e = np.exp(logits - logits.max(-1, keepdims=True))
|
| 127 |
+
top = 1.0 / e.sum(-1) # softmax probability of the argmax token
|
| 128 |
+
for b in range(B):
|
| 129 |
+
if not done[b]:
|
| 130 |
+
probs[b].append(float(top[b]))
|
| 131 |
+
if tok[b] == EOS:
|
| 132 |
+
done[b] = True
|
| 133 |
+
elif (
|
| 134 |
+
len(probs[b]) >= 8
|
| 135 |
+
and np.mean(probs[b]) < MATH_MIN_CONF - EARLY_STOP_MARGIN
|
| 136 |
+
):
|
| 137 |
+
done[b] = True # rejected: keeps a low mean probability
|
| 138 |
+
elif tok[b] >= N_SPECIAL:
|
| 139 |
+
seqs[b].append(int(tok[b]))
|
| 140 |
+
if done.all():
|
| 141 |
+
break
|
| 142 |
+
for k, j in enumerate(idx):
|
| 143 |
+
text = detokenize([self.itos[i] for i in seqs[k]])
|
| 144 |
+
out[j] = (text, float(np.mean(probs[k])) if probs[k] else 0.0)
|
| 145 |
+
return out
|
OCR_INDRA/indra/indra_ocr/models.py
ADDED
|
@@ -0,0 +1,106 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""Model files: a local folder, or a download from the Hugging Face repo."""
|
| 2 |
+
|
| 3 |
+
from __future__ import annotations
|
| 4 |
+
|
| 5 |
+
import os
|
| 6 |
+
import time
|
| 7 |
+
from pathlib import Path
|
| 8 |
+
|
| 9 |
+
HF_REPO = "datalab-to/indra-ocr"
|
| 10 |
+
# Model set this package version was built and tested against (a tag in HF_REPO).
|
| 11 |
+
HF_REVISION = "v3.1.1"
|
| 12 |
+
FILES = ("det.onnx", "rec.onnx", "order.onnx", "meta.json")
|
| 13 |
+
# A cached model folder is checked against the Hub at most this often (a moved
|
| 14 |
+
# tag or a fixed file arrives within a day; no network on the other runs).
|
| 15 |
+
CHECK_EVERY_S = 24 * 3600
|
| 16 |
+
|
| 17 |
+
|
| 18 |
+
class ModelError(OSError):
|
| 19 |
+
"""The models are not available: a download failed (no network, no access)
|
| 20 |
+
or a local model folder lacks files."""
|
| 21 |
+
|
| 22 |
+
|
| 23 |
+
MATH_FILES = (
|
| 24 |
+
"mathcls.onnx",
|
| 25 |
+
"math_enc.onnx",
|
| 26 |
+
"math_dec_init.onnx",
|
| 27 |
+
"math_dec_step.onnx",
|
| 28 |
+
"math_meta.json",
|
| 29 |
+
)
|
| 30 |
+
|
| 31 |
+
|
| 32 |
+
def model_dir(
|
| 33 |
+
path: str | Path | None = None, revision: str | None = None, math: bool = False
|
| 34 |
+
) -> Path:
|
| 35 |
+
"""Folder with det.onnx, rec.onnx, order.onnx and meta.json (and, with
|
| 36 |
+
``math``, the math models, which are downloaded only then).
|
| 37 |
+
|
| 38 |
+
Order of lookup: ``path``, the INDRA_OCR_MODEL_DIR environment variable, then
|
| 39 |
+
a download of ``HF_REPO``. Downloads are cached by huggingface_hub."""
|
| 40 |
+
path = path or os.environ.get("INDRA_OCR_MODEL_DIR")
|
| 41 |
+
if path:
|
| 42 |
+
p = Path(path).expanduser()
|
| 43 |
+
missing = [f for f in FILES[:2] + FILES[3:] if not (p / f).exists()]
|
| 44 |
+
if math:
|
| 45 |
+
missing += [f for f in MATH_FILES if not (p / f).exists()]
|
| 46 |
+
if missing:
|
| 47 |
+
raise ModelError(f"the model folder {p} lacks {', '.join(missing)}")
|
| 48 |
+
return p
|
| 49 |
+
from huggingface_hub import snapshot_download
|
| 50 |
+
|
| 51 |
+
rev = revision or HF_REVISION
|
| 52 |
+
# config.json is the file the Hub counts downloads by: it is requested with
|
| 53 |
+
# the models on the first download and on each daily check.
|
| 54 |
+
kw = dict(
|
| 55 |
+
revision=rev,
|
| 56 |
+
allow_patterns=[
|
| 57 |
+
"*.onnx",
|
| 58 |
+
"meta.json",
|
| 59 |
+
"config.json",
|
| 60 |
+
*(MATH_FILES if math else ()),
|
| 61 |
+
],
|
| 62 |
+
ignore_patterns=[] if math else list(MATH_FILES),
|
| 63 |
+
)
|
| 64 |
+
try:
|
| 65 |
+
cached = Path(snapshot_download(HF_REPO, local_files_only=True, **kw))
|
| 66 |
+
except Exception: # noqa: BLE001 (not cached yet)
|
| 67 |
+
cached = None
|
| 68 |
+
stamp = _check_stamp(rev)
|
| 69 |
+
if cached is not None and _age(stamp) < CHECK_EVERY_S:
|
| 70 |
+
return cached
|
| 71 |
+
try:
|
| 72 |
+
p = Path(snapshot_download(HF_REPO, **kw))
|
| 73 |
+
_touch(stamp)
|
| 74 |
+
return p
|
| 75 |
+
except Exception as e: # noqa: BLE001 (network, access, disk: one clear message)
|
| 76 |
+
if cached is not None: # offline or the Hub is down: keep the cached models
|
| 77 |
+
return cached
|
| 78 |
+
reason = (str(e).strip().splitlines() or [type(e).__name__])[0]
|
| 79 |
+
raise ModelError(
|
| 80 |
+
f"could not download the models from Hugging Face ({HF_REPO}, {rev}): "
|
| 81 |
+
f"{type(e).__name__}: {reason[:200]}. Check the network connection, or "
|
| 82 |
+
"point at a local model folder with --model-dir or INDRA_OCR_MODEL_DIR."
|
| 83 |
+
) from e
|
| 84 |
+
|
| 85 |
+
|
| 86 |
+
def _check_stamp(rev: str) -> Path:
|
| 87 |
+
"""The file whose modification time records the last check of ``rev``."""
|
| 88 |
+
from huggingface_hub import constants
|
| 89 |
+
|
| 90 |
+
name = f".indra_ocr_checked--{HF_REPO.replace('/', '--')}--{rev}"
|
| 91 |
+
return Path(constants.HF_HUB_CACHE) / name
|
| 92 |
+
|
| 93 |
+
|
| 94 |
+
def _age(stamp: Path) -> float:
|
| 95 |
+
try:
|
| 96 |
+
return time.time() - stamp.stat().st_mtime
|
| 97 |
+
except OSError:
|
| 98 |
+
return float("inf")
|
| 99 |
+
|
| 100 |
+
|
| 101 |
+
def _touch(stamp: Path) -> None:
|
| 102 |
+
try:
|
| 103 |
+
stamp.parent.mkdir(parents=True, exist_ok=True)
|
| 104 |
+
stamp.touch()
|
| 105 |
+
except OSError: # a read-only cache: check again next time
|
| 106 |
+
pass
|
OCR_INDRA/indra/indra_ocr/order_post.py
ADDED
|
@@ -0,0 +1,307 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""Torch-free parts of the reading-order stage: line features, decoding,
|
| 2 |
+
baselines, metrics, and the ground-truth order of a labelled page.
|
| 3 |
+
|
| 4 |
+
order_model.py (the PyTorch OrderNet) re-exports all of these.
|
| 5 |
+
"""
|
| 6 |
+
|
| 7 |
+
from __future__ import annotations
|
| 8 |
+
|
| 9 |
+
import math
|
| 10 |
+
|
| 11 |
+
import numpy as np
|
| 12 |
+
|
| 13 |
+
N_GEOM = 10
|
| 14 |
+
N_TEXT = 6
|
| 15 |
+
N_FEATS = N_GEOM + N_TEXT
|
| 16 |
+
|
| 17 |
+
|
| 18 |
+
def line_features(
|
| 19 |
+
boxes: np.ndarray, page_wh: tuple[float, float], texts: list[str] | None = None
|
| 20 |
+
) -> np.ndarray:
|
| 21 |
+
"""[N, 4] boxes (x0, y0, x1, y1) -> [N, N_FEATS] float32 features."""
|
| 22 |
+
W, H = page_wh
|
| 23 |
+
b = boxes.astype(np.float32)
|
| 24 |
+
n = len(b)
|
| 25 |
+
if n == 0:
|
| 26 |
+
return np.zeros((0, N_FEATS), np.float32)
|
| 27 |
+
w = np.maximum(b[:, 2] - b[:, 0], 1e-3)
|
| 28 |
+
h = np.maximum(b[:, 3] - b[:, 1], 1e-3)
|
| 29 |
+
med_h = float(np.median(h))
|
| 30 |
+
geom = np.stack(
|
| 31 |
+
[
|
| 32 |
+
b[:, 0] / W,
|
| 33 |
+
b[:, 1] / H,
|
| 34 |
+
b[:, 2] / W,
|
| 35 |
+
b[:, 3] / H,
|
| 36 |
+
(b[:, 0] + b[:, 2]) / (2 * W),
|
| 37 |
+
(b[:, 1] + b[:, 3]) / (2 * H),
|
| 38 |
+
w / W,
|
| 39 |
+
np.log(h / med_h),
|
| 40 |
+
np.log(w / h) / 4.0,
|
| 41 |
+
np.full(n, med_h / H * 20.0),
|
| 42 |
+
],
|
| 43 |
+
1,
|
| 44 |
+
)
|
| 45 |
+
txt = np.zeros((n, N_TEXT), np.float32)
|
| 46 |
+
if texts is not None:
|
| 47 |
+
for i, t in enumerate(texts):
|
| 48 |
+
t = t.strip()
|
| 49 |
+
if not t:
|
| 50 |
+
continue
|
| 51 |
+
txt[i, 0] = math.log1p(len(t)) / 5.0
|
| 52 |
+
txt[i, 1] = sum(c.isdigit() for c in t) / len(t)
|
| 53 |
+
txt[i, 2] = float(
|
| 54 |
+
t[0] in "•-–▪◦*·→" or (t[:3].rstrip(".)").isdigit() and len(t) > 3)
|
| 55 |
+
)
|
| 56 |
+
txt[i, 3] = float(t.endswith("-"))
|
| 57 |
+
txt[i, 4] = float(t[-1] in ".!?:;")
|
| 58 |
+
txt[i, 5] = float(t[0].isupper())
|
| 59 |
+
return np.concatenate([geom, txt], 1).astype(np.float32)
|
| 60 |
+
|
| 61 |
+
|
| 62 |
+
def decode_order(a: np.ndarray, sweeps: int = 4) -> list[int]:
|
| 63 |
+
"""Order from a pairwise logit matrix [N, N] (a[i, j] > 0: i before j).
|
| 64 |
+
|
| 65 |
+
Start from the row-sum ranking, then local search with insertion moves:
|
| 66 |
+
take each line out and put it back at the position that most increases
|
| 67 |
+
the summed log-probability of the pairwise relations. Unlike adjacent
|
| 68 |
+
swaps, an insertion can move a line far (e.g. a short last line of a
|
| 69 |
+
column that the ranking put after the next column). Vectorized: each
|
| 70 |
+
move costs a few numpy calls, O(N) work."""
|
| 71 |
+
n = a.shape[0]
|
| 72 |
+
if n <= 1:
|
| 73 |
+
return list(range(n))
|
| 74 |
+
lp = -np.logaddexp(0.0, -a) # log P(i before j)
|
| 75 |
+
order = np.argsort(-lp.sum(1) + lp.sum(0))
|
| 76 |
+
for _ in range(sweeps):
|
| 77 |
+
improved = False
|
| 78 |
+
for x in order.copy():
|
| 79 |
+
rest = order[order != x]
|
| 80 |
+
before = lp[rest, x] # x placed after these
|
| 81 |
+
after = lp[x, rest] # x placed before these
|
| 82 |
+
gains = np.concatenate(([0.0], np.cumsum(before))) + np.concatenate(
|
| 83 |
+
(np.cumsum(after[::-1])[::-1], [0.0])
|
| 84 |
+
)
|
| 85 |
+
k = int(np.argmax(gains))
|
| 86 |
+
cur = int(np.nonzero(order == x)[0][0])
|
| 87 |
+
if gains[k] > gains[cur] + 1e-6:
|
| 88 |
+
order = np.insert(rest, k, x)
|
| 89 |
+
improved = True
|
| 90 |
+
if not improved:
|
| 91 |
+
break
|
| 92 |
+
return [int(v) for v in order]
|
| 93 |
+
|
| 94 |
+
|
| 95 |
+
# ---------------------------------------------------------------------------
|
| 96 |
+
# Baselines and metrics
|
| 97 |
+
# ---------------------------------------------------------------------------
|
| 98 |
+
|
| 99 |
+
|
| 100 |
+
def order_top_down(boxes: np.ndarray) -> list[int]:
|
| 101 |
+
"""Rows top to bottom (centres within half a line height share a row), left to right."""
|
| 102 |
+
if len(boxes) == 0:
|
| 103 |
+
return []
|
| 104 |
+
cy = (boxes[:, 1] + boxes[:, 3]) / 2
|
| 105 |
+
h = np.median(boxes[:, 3] - boxes[:, 1])
|
| 106 |
+
idx = sorted(
|
| 107 |
+
range(len(boxes)),
|
| 108 |
+
key=lambda i: (round(cy[i] / max(h * 0.5, 1e-3)), boxes[i, 0]),
|
| 109 |
+
)
|
| 110 |
+
return idx
|
| 111 |
+
|
| 112 |
+
|
| 113 |
+
def order_xycut(boxes: np.ndarray) -> list[int]:
|
| 114 |
+
"""Recursive XY-cut: split at the widest whitespace gap on either axis.
|
| 115 |
+
|
| 116 |
+
A horizontal cut (between rows) puts the upper part first; a vertical cut
|
| 117 |
+
(between columns) puts the left part first. Without a gap, fall back to
|
| 118 |
+
top-down order."""
|
| 119 |
+
|
| 120 |
+
def best_cut(b, axis):
|
| 121 |
+
lo, hi = (b[:, 1], b[:, 3]) if axis == 1 else (b[:, 0], b[:, 2])
|
| 122 |
+
o = np.argsort(lo)
|
| 123 |
+
end = hi[o[0]]
|
| 124 |
+
best_gap, best_k = 0.0, -1
|
| 125 |
+
for k in range(1, len(o)):
|
| 126 |
+
gap = lo[o[k]] - end
|
| 127 |
+
if gap > best_gap:
|
| 128 |
+
best_gap, best_k = gap, k
|
| 129 |
+
end = max(end, hi[o[k]])
|
| 130 |
+
return best_gap, best_k, o
|
| 131 |
+
|
| 132 |
+
def rec(ids):
|
| 133 |
+
if len(ids) <= 1:
|
| 134 |
+
return list(ids)
|
| 135 |
+
b = boxes[ids]
|
| 136 |
+
gy, ky, oy = best_cut(b, 1)
|
| 137 |
+
gx, kx, ox = best_cut(b, 0)
|
| 138 |
+
if ky < 0 and kx < 0:
|
| 139 |
+
return [ids[i] for i in order_top_down(b)]
|
| 140 |
+
k, o = (ky, oy) if gy >= gx else (kx, ox)
|
| 141 |
+
return rec([ids[i] for i in o[:k]]) + rec([ids[i] for i in o[k:]])
|
| 142 |
+
|
| 143 |
+
return rec(list(range(len(boxes))))
|
| 144 |
+
|
| 145 |
+
|
| 146 |
+
def kendall_tau(pred: list[int], n: int) -> float:
|
| 147 |
+
"""Kendall tau between a predicted order and the identity order 0..n-1."""
|
| 148 |
+
if n < 2:
|
| 149 |
+
return 1.0
|
| 150 |
+
pos = np.empty(n, int)
|
| 151 |
+
pos[np.asarray(pred)] = np.arange(n)
|
| 152 |
+
conc = 0
|
| 153 |
+
for i in range(n):
|
| 154 |
+
conc += int((pos[i] < pos[i + 1 :]).sum())
|
| 155 |
+
total = n * (n - 1) // 2
|
| 156 |
+
return (2 * conc - total) / total
|
| 157 |
+
|
| 158 |
+
|
| 159 |
+
def successor_acc(pred: list[int], n: int) -> float:
|
| 160 |
+
"""Fraction of true consecutive pairs (i, i+1) that are also consecutive in the prediction."""
|
| 161 |
+
if n < 2:
|
| 162 |
+
return 1.0
|
| 163 |
+
nxt = {pred[k]: pred[k + 1] for k in range(len(pred) - 1)}
|
| 164 |
+
return sum(nxt.get(i) == i + 1 for i in range(n - 1)) / (n - 1)
|
| 165 |
+
|
| 166 |
+
|
| 167 |
+
# ---------------------------------------------------------------------------
|
| 168 |
+
# Ground-truth order from page labels
|
| 169 |
+
# ---------------------------------------------------------------------------
|
| 170 |
+
|
| 171 |
+
GRID_LABELS = {"Table", "Form", "Table-Of-Contents"}
|
| 172 |
+
|
| 173 |
+
# Block classes the layout heads predict for each line (chandra block labels).
|
| 174 |
+
BLOCK_CLASSES = [
|
| 175 |
+
"Text",
|
| 176 |
+
"Section-Header",
|
| 177 |
+
"Page-Header",
|
| 178 |
+
"Page-Footer",
|
| 179 |
+
"List-Group",
|
| 180 |
+
"Table",
|
| 181 |
+
"Caption",
|
| 182 |
+
"Footnote",
|
| 183 |
+
"Table-Of-Contents",
|
| 184 |
+
"Form",
|
| 185 |
+
]
|
| 186 |
+
_CLASS_ID = {c: i for i, c in enumerate(BLOCK_CLASSES)}
|
| 187 |
+
|
| 188 |
+
|
| 189 |
+
def block_class(label: str) -> int:
|
| 190 |
+
"""Class id of a chandra block label (unknown labels count as Text)."""
|
| 191 |
+
return _CLASS_ID.get(label, 0)
|
| 192 |
+
|
| 193 |
+
|
| 194 |
+
def label_order(lab: dict) -> list[int] | None:
|
| 195 |
+
"""True reading order of a labelled page (indices into lab["lines"]).
|
| 196 |
+
|
| 197 |
+
Blocks come in chandra's order. Inside a table / form / table-of-contents
|
| 198 |
+
block, lines go row by row. Inside any other block, lines go by XY-cut,
|
| 199 |
+
so a list or text block that spans two columns reads column by column
|
| 200 |
+
(row order interleaved the columns). Returns None when a line has no block."""
|
| 201 |
+
lines = lab["lines"]
|
| 202 |
+
if any(ln.get("block") is None for ln in lines):
|
| 203 |
+
return None
|
| 204 |
+
blocks = lab["blocks"]
|
| 205 |
+
by_block: dict[int, list[int]] = {}
|
| 206 |
+
for i, ln in enumerate(lines):
|
| 207 |
+
by_block.setdefault(ln["block"], []).append(i)
|
| 208 |
+
out: list[int] = []
|
| 209 |
+
for b in sorted(by_block):
|
| 210 |
+
idx = by_block[b]
|
| 211 |
+
boxes = np.array(
|
| 212 |
+
[(lines[i]["ink"] or lines[i]["bbox"]) for i in idx], np.float32
|
| 213 |
+
).reshape(-1, 4)
|
| 214 |
+
label = blocks[b][0] if b < len(blocks) else "Text"
|
| 215 |
+
local = order_top_down(boxes) if label in GRID_LABELS else order_xycut(boxes)
|
| 216 |
+
out += [idx[k] for k in local]
|
| 217 |
+
return out
|
| 218 |
+
|
| 219 |
+
|
| 220 |
+
def _cluster(logits: np.ndarray) -> list[list[int]]:
|
| 221 |
+
"""Average-linkage clustering of pairwise "same group" logits: merge the two
|
| 222 |
+
groups with the highest mean logit between them while that mean is positive
|
| 223 |
+
(probability above one half). Unlike connected components, one wrong pair
|
| 224 |
+
cannot join two groups that the other pairs keep apart.
|
| 225 |
+
|
| 226 |
+
A merge changes only the merged group's row and column of the mean matrix,
|
| 227 |
+
and each row keeps its best partner, so a page of n lines costs about n^2
|
| 228 |
+
instead of n^3 (an 800-line page: 1.5 s -> tens of ms)."""
|
| 229 |
+
n = logits.shape[0]
|
| 230 |
+
if n == 0:
|
| 231 |
+
return []
|
| 232 |
+
S = logits.astype(np.float64).copy()
|
| 233 |
+
np.fill_diagonal(S, 0.0)
|
| 234 |
+
size = np.ones(n)
|
| 235 |
+
alive = np.ones(n, bool)
|
| 236 |
+
members = [[i] for i in range(n)]
|
| 237 |
+
M = S.copy() # mean logit between groups; -inf on the diagonal and dead groups
|
| 238 |
+
np.fill_diagonal(M, -np.inf)
|
| 239 |
+
best = M.argmax(1)
|
| 240 |
+
best_v = M[np.arange(n), best]
|
| 241 |
+
for _ in range(n - 1):
|
| 242 |
+
a = int(np.argmax(best_v))
|
| 243 |
+
b = int(best[a])
|
| 244 |
+
if best_v[a] <= 0:
|
| 245 |
+
break
|
| 246 |
+
a, b = min(a, b), max(a, b)
|
| 247 |
+
S[a] += S[b]
|
| 248 |
+
S[:, a] += S[:, b]
|
| 249 |
+
S[a, a] = 0.0
|
| 250 |
+
size[a] += size[b]
|
| 251 |
+
alive[b] = False
|
| 252 |
+
members[a] += members[b]
|
| 253 |
+
row = S[a] / (size[a] * size)
|
| 254 |
+
row[~alive] = -np.inf
|
| 255 |
+
row[a] = -np.inf
|
| 256 |
+
M[a], M[:, a] = row, row
|
| 257 |
+
M[b], M[:, b] = -np.inf, -np.inf
|
| 258 |
+
best_v[b] = -np.inf
|
| 259 |
+
best[a] = int(row.argmax())
|
| 260 |
+
best_v[a] = row[best[a]]
|
| 261 |
+
# other rows: (j, a) changed and (j, b) is gone
|
| 262 |
+
stale = alive & ((best == a) | (best == b))
|
| 263 |
+
stale[a] = False
|
| 264 |
+
for j in np.flatnonzero(stale):
|
| 265 |
+
best[j] = int(M[j].argmax())
|
| 266 |
+
best_v[j] = M[j, best[j]]
|
| 267 |
+
up = alive & (row > best_v)
|
| 268 |
+
up[a] = False
|
| 269 |
+
best[up], best_v[up] = a, row[up]
|
| 270 |
+
return [sorted(members[i]) for i in range(n) if alive[i]]
|
| 271 |
+
|
| 272 |
+
|
| 273 |
+
def decode_blocks_order(
|
| 274 |
+
a: np.ndarray,
|
| 275 |
+
same: np.ndarray,
|
| 276 |
+
boxes: np.ndarray,
|
| 277 |
+
cls: np.ndarray | None = None,
|
| 278 |
+
) -> list[int]:
|
| 279 |
+
"""Reading order with the layout heads: blocks first, then lines inside blocks.
|
| 280 |
+
|
| 281 |
+
Blocks come from average-linkage clustering of the same-block logits
|
| 282 |
+
(independent of the order). Inside a block, lines go by XY-cut, or
|
| 283 |
+
row by row in a table / form / table-of-contents block (``cls`` ids of
|
| 284 |
+
BLOCK_CLASSES). Blocks go by the mean pairwise order logit between them."""
|
| 285 |
+
n = a.shape[0]
|
| 286 |
+
if n <= 1:
|
| 287 |
+
return list(range(n))
|
| 288 |
+
blocks = _cluster(same)
|
| 289 |
+
grid = {_CLASS_ID[c] for c in GRID_LABELS if c in _CLASS_ID}
|
| 290 |
+
inner = []
|
| 291 |
+
for b in blocks:
|
| 292 |
+
bx = boxes[b, :4]
|
| 293 |
+
if cls is not None and int(cls[b].sum(0).argmax()) in grid:
|
| 294 |
+
local = order_top_down(bx)
|
| 295 |
+
else:
|
| 296 |
+
local = order_xycut(bx)
|
| 297 |
+
inner.append([b[k] for k in local])
|
| 298 |
+
# Mean order logit between blocks p < q, antisymmetric: G a G^T with G the
|
| 299 |
+
# block-membership matrix scaled by 1 / block size (one matrix product
|
| 300 |
+
# instead of a loop over block pairs).
|
| 301 |
+
m = len(blocks)
|
| 302 |
+
G = np.zeros((m, n), np.float64)
|
| 303 |
+
for k, b in enumerate(blocks):
|
| 304 |
+
G[k, b] = 1.0 / len(b)
|
| 305 |
+
up = np.triu(G @ a.astype(np.float64) @ G.T, 1)
|
| 306 |
+
ab = (up - up.T).astype(np.float32)
|
| 307 |
+
return [i for k in decode_order(ab) for i in inner[k]]
|
OCR_INDRA/indra/indra_ocr/parallel.py
ADDED
|
@@ -0,0 +1,210 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""Many pages on many cores: a pool of worker processes, one page per task.
|
| 2 |
+
|
| 3 |
+
ONNX Runtime's own threads speed up one page only a little: rendering, the text
|
| 4 |
+
layer and the layout steps run on one core. Pages are independent, so one
|
| 5 |
+
single-threaded worker process per core uses the cores far better. pdfium
|
| 6 |
+
is not thread-safe, so the workers are processes, not threads.
|
| 7 |
+
"""
|
| 8 |
+
|
| 9 |
+
from __future__ import annotations
|
| 10 |
+
|
| 11 |
+
import os
|
| 12 |
+
from pathlib import Path
|
| 13 |
+
|
| 14 |
+
from .pipeline import InputError, Page, is_pdf
|
| 15 |
+
|
| 16 |
+
THREADS_PER_WORKER = 1 # ORT threads in each worker process
|
| 17 |
+
MIN_POOL_PAGES = 4 # fewer pages: one process, all cores on each page
|
| 18 |
+
WORKER_MEMORY = 1.5e9 # bytes a worker can reach (giant scans); caps the pool
|
| 19 |
+
|
| 20 |
+
_conv = None
|
| 21 |
+
_doc: tuple = (None, None)
|
| 22 |
+
|
| 23 |
+
|
| 24 |
+
def usable_cores() -> int:
|
| 25 |
+
"""Cores this process may run on (taskset and cgroup CPU sets count)."""
|
| 26 |
+
try:
|
| 27 |
+
return len(os.sched_getaffinity(0))
|
| 28 |
+
except AttributeError: # macOS, Windows
|
| 29 |
+
return os.cpu_count() or 1
|
| 30 |
+
|
| 31 |
+
|
| 32 |
+
def _max_workers_for_memory() -> int:
|
| 33 |
+
try:
|
| 34 |
+
ram = os.sysconf("SC_PAGE_SIZE") * os.sysconf("SC_PHYS_PAGES")
|
| 35 |
+
except (AttributeError, ValueError, OSError): # Windows
|
| 36 |
+
return 1 << 30
|
| 37 |
+
return max(1, int(ram // WORKER_MEMORY))
|
| 38 |
+
|
| 39 |
+
|
| 40 |
+
def _init(load_kwargs: dict) -> None:
|
| 41 |
+
global _conv
|
| 42 |
+
import cv2
|
| 43 |
+
|
| 44 |
+
from . import load
|
| 45 |
+
|
| 46 |
+
cv2.setNumThreads(load_kwargs["threads"])
|
| 47 |
+
_conv = load(**load_kwargs)
|
| 48 |
+
|
| 49 |
+
|
| 50 |
+
def _run(unit: tuple):
|
| 51 |
+
"""One unit: (path, page index, password) of a PDF, or (path, None, max pages)
|
| 52 |
+
of an image file. Returns [(index, Page)], or the InputError."""
|
| 53 |
+
global _doc
|
| 54 |
+
path, index, extra = unit
|
| 55 |
+
try:
|
| 56 |
+
if index is None:
|
| 57 |
+
return list(_conv.convert(path, extra))
|
| 58 |
+
if _doc[0] != path:
|
| 59 |
+
import pypdfium2 as pdfium
|
| 60 |
+
|
| 61 |
+
_doc = (path, pdfium.PdfDocument(path, password=extra))
|
| 62 |
+
doc = _doc[1]
|
| 63 |
+
return [(index, _conv._safe(lambda: _conv.pdf_page(path, doc, index)))]
|
| 64 |
+
except InputError as e:
|
| 65 |
+
return e
|
| 66 |
+
|
| 67 |
+
|
| 68 |
+
def _plan(paths: list, max_pages: int, password: str | None) -> list:
|
| 69 |
+
"""Per input: its work units, or the InputError that stops it."""
|
| 70 |
+
import pypdfium2 as pdfium
|
| 71 |
+
|
| 72 |
+
plan = []
|
| 73 |
+
for p in paths:
|
| 74 |
+
path = str(p)
|
| 75 |
+
if not Path(path).is_file():
|
| 76 |
+
plan.append(InputError(f"{path}: no such file"))
|
| 77 |
+
elif not is_pdf(path):
|
| 78 |
+
plan.append([(path, None, max_pages)])
|
| 79 |
+
else:
|
| 80 |
+
try:
|
| 81 |
+
doc = pdfium.PdfDocument(path, password=password)
|
| 82 |
+
n = len(doc)
|
| 83 |
+
doc.close()
|
| 84 |
+
except pdfium.PdfiumError as e:
|
| 85 |
+
locked = "password" in str(e).lower()
|
| 86 |
+
plan.append(
|
| 87 |
+
InputError(
|
| 88 |
+
f"{path}: password-protected PDF (pass password=, or --password)"
|
| 89 |
+
if locked
|
| 90 |
+
else f"{path}: cannot open the PDF ({e})"
|
| 91 |
+
)
|
| 92 |
+
)
|
| 93 |
+
continue
|
| 94 |
+
n = min(n, max_pages) if max_pages else n
|
| 95 |
+
plan.append([(path, i, password) for i in range(n)])
|
| 96 |
+
return plan
|
| 97 |
+
|
| 98 |
+
|
| 99 |
+
def convert_many(
|
| 100 |
+
paths,
|
| 101 |
+
cores: int | None = None,
|
| 102 |
+
max_pages: int = 0,
|
| 103 |
+
password: str | None = None,
|
| 104 |
+
**load_kwargs,
|
| 105 |
+
):
|
| 106 |
+
"""Yield (path, result) per input, in input order: result is the list of
|
| 107 |
+
(page index, Page) in page order, or the InputError that stopped the file.
|
| 108 |
+
|
| 109 |
+
cores: how many cores to use (default: all this process may run on). The
|
| 110 |
+
pages of all inputs are spread over one worker process per core (fewer when
|
| 111 |
+
physical memory is under WORKER_MEMORY per worker); with fewer than
|
| 112 |
+
MIN_POOL_PAGES pages, one process uses all the cores on each page.
|
| 113 |
+
load_kwargs go to indra_ocr.load (not threads)."""
|
| 114 |
+
from .models import model_dir
|
| 115 |
+
|
| 116 |
+
# Resolve (and on a first run download) the models once, not in every worker.
|
| 117 |
+
load_kwargs["path"] = str(
|
| 118 |
+
model_dir(load_kwargs.get("path"), math=load_kwargs.get("math", False))
|
| 119 |
+
)
|
| 120 |
+
paths = [str(p) for p in paths]
|
| 121 |
+
cores = max(1, cores or usable_cores())
|
| 122 |
+
plan = _plan(paths, max_pages, password)
|
| 123 |
+
n_pages = sum(len(u) for u in plan if isinstance(u, list))
|
| 124 |
+
workers = min(cores // THREADS_PER_WORKER, n_pages, _max_workers_for_memory())
|
| 125 |
+
if workers < 2 or n_pages < MIN_POOL_PAGES:
|
| 126 |
+
yield from _serial(paths, plan, max_pages, password, cores, load_kwargs)
|
| 127 |
+
return
|
| 128 |
+
yield from _pooled(paths, plan, workers, max_pages, password, cores, load_kwargs)
|
| 129 |
+
|
| 130 |
+
|
| 131 |
+
def _serial(paths, plan, max_pages, password, threads: int, load_kwargs: dict):
|
| 132 |
+
"""The inputs in this process, ``threads`` ONNX Runtime threads on each page."""
|
| 133 |
+
from . import load
|
| 134 |
+
|
| 135 |
+
conv = load(threads=threads, **load_kwargs)
|
| 136 |
+
for path, units in zip(paths, plan):
|
| 137 |
+
if isinstance(units, InputError):
|
| 138 |
+
yield path, units
|
| 139 |
+
continue
|
| 140 |
+
try:
|
| 141 |
+
yield path, list(conv.convert(path, max_pages, password=password))
|
| 142 |
+
except InputError as e:
|
| 143 |
+
yield path, e
|
| 144 |
+
|
| 145 |
+
|
| 146 |
+
def _pooled(paths, plan, workers, max_pages, password, cores, load_kwargs: dict):
|
| 147 |
+
"""The pages on a pool of worker processes. If the workers cannot run (with
|
| 148 |
+
the spawn start method a worker imports the caller's main module, so on
|
| 149 |
+
macOS and Windows a script must call convert_many under
|
| 150 |
+
``if __name__ == "__main__":``), warn and read the rest in this process."""
|
| 151 |
+
import warnings
|
| 152 |
+
from concurrent.futures import ProcessPoolExecutor
|
| 153 |
+
from concurrent.futures.process import BrokenProcessPool
|
| 154 |
+
from multiprocessing import get_context
|
| 155 |
+
|
| 156 |
+
threads = THREADS_PER_WORKER
|
| 157 |
+
flat = [u for units in plan if isinstance(units, list) for u in units]
|
| 158 |
+
# Thread pools of the libraries the workers import (OpenMP, BLAS) take
|
| 159 |
+
# their size from the environment at import: keep each to the worker's
|
| 160 |
+
# share while the workers start (map submits every task, starting them all).
|
| 161 |
+
env = {
|
| 162 |
+
k: str(threads)
|
| 163 |
+
for k in ("OMP_NUM_THREADS", "OPENBLAS_NUM_THREADS", "MKL_NUM_THREADS")
|
| 164 |
+
}
|
| 165 |
+
saved = {k: os.environ.get(k) for k in env}
|
| 166 |
+
os.environ.update(env)
|
| 167 |
+
done = 0 # inputs already yielded
|
| 168 |
+
ex = ProcessPoolExecutor(
|
| 169 |
+
workers,
|
| 170 |
+
mp_context=get_context("spawn"),
|
| 171 |
+
initializer=_init,
|
| 172 |
+
initargs=({**load_kwargs, "threads": threads},),
|
| 173 |
+
)
|
| 174 |
+
try:
|
| 175 |
+
try:
|
| 176 |
+
results = ex.map(_run, flat)
|
| 177 |
+
finally:
|
| 178 |
+
for k, v in saved.items():
|
| 179 |
+
if v is None:
|
| 180 |
+
os.environ.pop(k, None)
|
| 181 |
+
else:
|
| 182 |
+
os.environ[k] = v
|
| 183 |
+
for path, units in zip(paths, plan):
|
| 184 |
+
if isinstance(units, InputError):
|
| 185 |
+
yield path, units
|
| 186 |
+
done += 1
|
| 187 |
+
continue
|
| 188 |
+
pages: list[tuple[int, Page]] = []
|
| 189 |
+
error = None
|
| 190 |
+
for _ in units:
|
| 191 |
+
r = next(results)
|
| 192 |
+
if isinstance(r, InputError):
|
| 193 |
+
error = error or r
|
| 194 |
+
else:
|
| 195 |
+
pages += r
|
| 196 |
+
yield path, (error if error is not None else pages)
|
| 197 |
+
done += 1
|
| 198 |
+
except BrokenProcessPool:
|
| 199 |
+
warnings.warn(
|
| 200 |
+
"indra_ocr.convert_many: the worker processes stopped (with the spawn "
|
| 201 |
+
'start method, call convert_many under `if __name__ == "__main__":`); '
|
| 202 |
+
"reading the remaining inputs in this process",
|
| 203 |
+
RuntimeWarning,
|
| 204 |
+
stacklevel=2,
|
| 205 |
+
)
|
| 206 |
+
yield from _serial(
|
| 207 |
+
paths[done:], plan[done:], max_pages, password, cores, load_kwargs
|
| 208 |
+
)
|
| 209 |
+
finally:
|
| 210 |
+
ex.shutdown(wait=False, cancel_futures=True)
|
OCR_INDRA/indra/indra_ocr/pipeline.py
ADDED
|
@@ -0,0 +1,604 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""The OCR pipeline on ONNX Runtime: line detection, CTC line recognition,
|
| 2 |
+
reading order and layout classes. CPU only, no PyTorch.
|
| 3 |
+
|
| 4 |
+
The model folder holds det.onnx, rec.onnx (or rec.int8.onnx), order.onnx and
|
| 5 |
+
meta.json (see ``indra_ocr.models``). With script.onnx and rec_cjk.onnx, each
|
| 6 |
+
line goes to the recognizer for its script: lines with Chinese characters to
|
| 7 |
+
rec_cjk.onnx (which also reads Latin), the rest to rec.onnx. With orient.onnx,
|
| 8 |
+
a page turned by a quarter or half turn is turned upright before detection
|
| 9 |
+
(``Page.rotation``; boxes are in the upright page).
|
| 10 |
+
"""
|
| 11 |
+
|
| 12 |
+
from __future__ import annotations
|
| 13 |
+
|
| 14 |
+
import json
|
| 15 |
+
import time
|
| 16 |
+
from dataclasses import dataclass, field
|
| 17 |
+
from pathlib import Path
|
| 18 |
+
|
| 19 |
+
import cv2
|
| 20 |
+
import numpy as np
|
| 21 |
+
|
| 22 |
+
from .det_post import decode, page_to_input
|
| 23 |
+
from .layout_post import page_blocks, page_markdown
|
| 24 |
+
from .line_mask import crop_lines
|
| 25 |
+
from .math import MATH_MIN_CONF, MathReader
|
| 26 |
+
from .order_post import (
|
| 27 |
+
BLOCK_CLASSES,
|
| 28 |
+
decode_blocks_order,
|
| 29 |
+
decode_order,
|
| 30 |
+
line_features,
|
| 31 |
+
order_xycut,
|
| 32 |
+
)
|
| 33 |
+
from .rec_preproc import PAD_X, pad_batch, width_batches
|
| 34 |
+
from .textlayer import pdf_order_coherent
|
| 35 |
+
from .vocab import Vocab
|
| 36 |
+
|
| 37 |
+
ADAPTIVE_MIN_H = (
|
| 38 |
+
9.0 # median line height (input px) that triggers a second detection pass
|
| 39 |
+
)
|
| 40 |
+
ADAPTIVE_TARGET_H = 12.0
|
| 41 |
+
# Gaps between CTC character positions (centres), in line heights. The label
|
| 42 |
+
# rules (2.5 and 1.0 line heights) measure glyph edges; centres add about half a
|
| 43 |
+
# line height of glyph width.
|
| 44 |
+
SPLIT_GAP = 3.0 # cut a detected line at a wider gap (label rule outside tables)
|
| 45 |
+
CELL_GAP = 1.8 # cut a table line into cells at a wider gap (label rule in tables)
|
| 46 |
+
GUTTER_GAP = 1.6 # cut at a smaller gap when other lines' edges line up with it (a column gutter)
|
| 47 |
+
GUTTER_EDGES = 3 # how many other line edges must line up (within half a line height)
|
| 48 |
+
ADAPTIVE_MAX_SIDE = 2048
|
| 49 |
+
ORIENT_SIDE = 768 # orient.onnx input: page thumbnail, long side, padded square
|
| 50 |
+
# A turn is applied only above this probability: real pages are almost all upright,
|
| 51 |
+
# and a wrong turn loses the page. Measured (orient_v1): 2 of 1,000 upright eval
|
| 52 |
+
# pages turned at 0.95 (13 at 0.5), 101 of the 164 Omni Parse Bench pages with
|
| 53 |
+
# rotated tests turned (128 at 0.5).
|
| 54 |
+
ORIENT_MIN = 0.95
|
| 55 |
+
|
| 56 |
+
|
| 57 |
+
# A recognized line under this confidence (mean CTC peak probability) gets no
|
| 58 |
+
# text: on empty form fields, checkboxes and rules the recognizer reads "-", "I"
|
| 59 |
+
# or "1" at low confidence. The line's box stays (layout and reading order).
|
| 60 |
+
MIN_CONF = 0.5
|
| 61 |
+
|
| 62 |
+
DESKEW_MAX = 5.0 # degrees of tilt searched
|
| 63 |
+
DESKEW_MIN = 0.3 # smaller tilts are left alone (the search step is 0.1)
|
| 64 |
+
|
| 65 |
+
|
| 66 |
+
def estimate_skew(gray: np.ndarray, side: int = 800) -> float:
|
| 67 |
+
"""Degrees (counter-clockwise) that straighten the text rows of a page: the
|
| 68 |
+
angle whose horizontal ink profile changes most sharply row to row, on a
|
| 69 |
+
page thumbnail, searched in 0.5 degree steps and then 0.1 degree steps."""
|
| 70 |
+
h, w = gray.shape
|
| 71 |
+
s = side / max(h, w)
|
| 72 |
+
small = cv2.resize(
|
| 73 |
+
gray, (max(1, round(w * s)), max(1, round(h * s))), interpolation=cv2.INTER_AREA
|
| 74 |
+
)
|
| 75 |
+
ink = (255.0 - small.astype(np.float32)) / 255.0
|
| 76 |
+
if ink.mean() < 0.003: # a blank page has no rows to straighten
|
| 77 |
+
return 0.0
|
| 78 |
+
size = (small.shape[1], small.shape[0])
|
| 79 |
+
c = (size[0] / 2, size[1] / 2)
|
| 80 |
+
|
| 81 |
+
def sharpness(a: float) -> float:
|
| 82 |
+
r = cv2.warpAffine(ink, cv2.getRotationMatrix2D(c, a, 1.0), size)
|
| 83 |
+
p = r.sum(1)
|
| 84 |
+
return float(((p[1:] - p[:-1]) ** 2).sum())
|
| 85 |
+
|
| 86 |
+
best = max(np.arange(-DESKEW_MAX, DESKEW_MAX + 1e-6, 0.5), key=sharpness)
|
| 87 |
+
return float(max(np.arange(best - 0.5, best + 0.5 + 1e-6, 0.1), key=sharpness))
|
| 88 |
+
|
| 89 |
+
|
| 90 |
+
def rotate_page(gray: np.ndarray, degrees: float) -> np.ndarray:
|
| 91 |
+
"""The page rotated by ``degrees`` counter-clockwise about its centre (same size)."""
|
| 92 |
+
h, w = gray.shape
|
| 93 |
+
M = cv2.getRotationMatrix2D((w / 2, h / 2), degrees, 1.0)
|
| 94 |
+
return cv2.warpAffine(
|
| 95 |
+
gray, M, (w, h), flags=cv2.INTER_LINEAR, borderMode=cv2.BORDER_REPLICATE
|
| 96 |
+
)
|
| 97 |
+
|
| 98 |
+
|
| 99 |
+
def unrotate_box(box, degrees: float, wh: tuple[int, int]) -> list[float]:
|
| 100 |
+
"""A box of the page that rotate_page(.., degrees) made, mapped back to the
|
| 101 |
+
page before it: the bounding box of the four mapped corners, clipped to it."""
|
| 102 |
+
w, h = wh
|
| 103 |
+
M = cv2.getRotationMatrix2D((w / 2, h / 2), -degrees, 1.0)
|
| 104 |
+
x0, y0, x1, y1 = box[:4]
|
| 105 |
+
c = M @ np.array([[x0, x1, x0, x1], [y0, y0, y1, y1], [1, 1, 1, 1]], float)
|
| 106 |
+
return [
|
| 107 |
+
round(float(np.clip(c[0].min(), 0, w)), 1),
|
| 108 |
+
round(float(np.clip(c[1].min(), 0, h)), 1),
|
| 109 |
+
round(float(np.clip(c[0].max(), 0, w)), 1),
|
| 110 |
+
round(float(np.clip(c[1].max(), 0, h)), 1),
|
| 111 |
+
]
|
| 112 |
+
|
| 113 |
+
|
| 114 |
+
def orient_input(gray: np.ndarray, side: int = ORIENT_SIDE) -> np.ndarray:
|
| 115 |
+
"""[side, side] float32 ink map (1 = ink): long side scaled to ``side``, padded."""
|
| 116 |
+
h, w = gray.shape
|
| 117 |
+
s = side / max(h, w)
|
| 118 |
+
small = cv2.resize(
|
| 119 |
+
gray, (max(1, round(w * s)), max(1, round(h * s))), interpolation=cv2.INTER_AREA
|
| 120 |
+
)
|
| 121 |
+
out = np.zeros((side, side), np.float32)
|
| 122 |
+
out[: small.shape[0], : small.shape[1]] = 1.0 - small.astype(np.float32) / 255.0
|
| 123 |
+
return out
|
| 124 |
+
|
| 125 |
+
|
| 126 |
+
@dataclass
|
| 127 |
+
class Line:
|
| 128 |
+
box: list[float] # x0, y0, x1, y1 in image pixels
|
| 129 |
+
text: str
|
| 130 |
+
conf: float
|
| 131 |
+
|
| 132 |
+
|
| 133 |
+
@dataclass
|
| 134 |
+
class Page:
|
| 135 |
+
lines: list[Line] # reading order
|
| 136 |
+
timings: dict = field(default_factory=dict)
|
| 137 |
+
markdown: str | None = None # with an order model that has layout heads
|
| 138 |
+
source: str = (
|
| 139 |
+
"ocr" # ocr | text (PDF text layer) | hybrid (text layer + OCR) | error
|
| 140 |
+
)
|
| 141 |
+
note: str = "" # why the page took this path
|
| 142 |
+
# layout blocks in reading order: {"label", "bbox" (pixels), "text"}, with an
|
| 143 |
+
# order model that has layout heads
|
| 144 |
+
blocks: list = field(default_factory=list)
|
| 145 |
+
# degrees counter-clockwise the input was turned; boxes are upright
|
| 146 |
+
rotation: int = 0
|
| 147 |
+
# degrees of small tilt undone after the turn (OCR reads the straightened
|
| 148 |
+
# page; boxes are mapped back to the turned page)
|
| 149 |
+
skew: float = 0.0
|
| 150 |
+
|
| 151 |
+
@property
|
| 152 |
+
def text(self) -> str:
|
| 153 |
+
return "\n".join(ln.text for ln in self.lines)
|
| 154 |
+
|
| 155 |
+
|
| 156 |
+
class OrtOCR:
|
| 157 |
+
def __init__(
|
| 158 |
+
self,
|
| 159 |
+
onnx_dir: str | Path,
|
| 160 |
+
threads: int = 4,
|
| 161 |
+
rec_int8: bool | None = None,
|
| 162 |
+
adaptive: bool = True,
|
| 163 |
+
table_heads: bool = False,
|
| 164 |
+
providers: list | None = None,
|
| 165 |
+
furniture: bool = False,
|
| 166 |
+
math: bool = False,
|
| 167 |
+
):
|
| 168 |
+
"""table_heads: build tables from the order model's row / column / cell
|
| 169 |
+
heads (when it has them) instead of from geometry. furniture: keep page
|
| 170 |
+
headers and footers in the markdown (dropped by default). math: read lines
|
| 171 |
+
with math as LaTeX (math.py; needs the math models; off by default). providers: ONNX
|
| 172 |
+
Runtime execution providers (default CPU; e.g. ["CUDAExecutionProvider",
|
| 173 |
+
"CPUExecutionProvider"] with onnxruntime-gpu)."""
|
| 174 |
+
self.table_heads = table_heads
|
| 175 |
+
self.furniture = furniture
|
| 176 |
+
import onnxruntime as ort
|
| 177 |
+
|
| 178 |
+
onnx_dir = Path(onnx_dir)
|
| 179 |
+
so = ort.SessionOptions()
|
| 180 |
+
so.intra_op_num_threads = threads
|
| 181 |
+
so.inter_op_num_threads = 1
|
| 182 |
+
so.graph_optimization_level = ort.GraphOptimizationLevel.ORT_ENABLE_ALL
|
| 183 |
+
# No CPU memory arena: with line batches of changing shape it kept growing
|
| 184 |
+
# (onnxruntime 1.31: rec.onnx +1.4 GB over 10 identical runs; a 40-page
|
| 185 |
+
# scan peaked at 4.9 GB), and it bought no measurable speed.
|
| 186 |
+
so.enable_cpu_mem_arena = False
|
| 187 |
+
prov = providers or ["CPUExecutionProvider"]
|
| 188 |
+
|
| 189 |
+
def need(name: str) -> str:
|
| 190 |
+
p = onnx_dir / name
|
| 191 |
+
if not p.exists():
|
| 192 |
+
raise FileNotFoundError(p)
|
| 193 |
+
return str(p)
|
| 194 |
+
|
| 195 |
+
if rec_int8 is None:
|
| 196 |
+
rec_int8 = (onnx_dir / "rec.int8.onnx").exists()
|
| 197 |
+
self.det = ort.InferenceSession(need("det.onnx"), so, providers=prov)
|
| 198 |
+
self.rec = ort.InferenceSession(
|
| 199 |
+
need("rec.int8.onnx" if rec_int8 else "rec.onnx"), so, providers=prov
|
| 200 |
+
)
|
| 201 |
+
self.mask = (
|
| 202 |
+
ort.InferenceSession(str(onnx_dir / "mask.onnx"), so, providers=prov)
|
| 203 |
+
if (onnx_dir / "mask.onnx").exists()
|
| 204 |
+
else None
|
| 205 |
+
)
|
| 206 |
+
self.order = (
|
| 207 |
+
ort.InferenceSession(str(onnx_dir / "order.onnx"), so, providers=prov)
|
| 208 |
+
if (onnx_dir / "order.onnx").exists()
|
| 209 |
+
else None
|
| 210 |
+
)
|
| 211 |
+
self.orient = (
|
| 212 |
+
ort.InferenceSession(str(onnx_dir / "orient.onnx"), so, providers=prov)
|
| 213 |
+
if (onnx_dir / "orient.onnx").exists()
|
| 214 |
+
else None
|
| 215 |
+
)
|
| 216 |
+
meta = json.loads(Path(need("meta.json")).read_text())
|
| 217 |
+
self.vocab = Vocab.from_json(meta["vocab"])
|
| 218 |
+
# Chinese: a line classifier routes lines with hanzi to their own recognizer.
|
| 219 |
+
self.script = self.rec_cjk = self.vocab_cjk = None
|
| 220 |
+
if (onnx_dir / "script.onnx").exists() and (onnx_dir / "rec_cjk.onnx").exists():
|
| 221 |
+
self.script = ort.InferenceSession(
|
| 222 |
+
str(onnx_dir / "script.onnx"), so, providers=prov
|
| 223 |
+
)
|
| 224 |
+
self.rec_cjk = ort.InferenceSession(
|
| 225 |
+
str(onnx_dir / "rec_cjk.onnx"), so, providers=prov
|
| 226 |
+
)
|
| 227 |
+
self.vocab_cjk = Vocab.from_json(meta["vocab_cjk"])
|
| 228 |
+
# Math: a line classifier routes lines with math to MathRec (math.py).
|
| 229 |
+
self.math = MathReader.load(onnx_dir, so, prov) if math else None
|
| 230 |
+
if math and self.math is None:
|
| 231 |
+
raise FileNotFoundError(f"math=True needs the math models in {onnx_dir}")
|
| 232 |
+
self.long_side = meta["det_long_side"]
|
| 233 |
+
self.height = meta["rec_height"]
|
| 234 |
+
self.adaptive = adaptive
|
| 235 |
+
|
| 236 |
+
def _detect_at(self, gray: np.ndarray, long_side: int):
|
| 237 |
+
x, s = page_to_input(gray, long_side)
|
| 238 |
+
return decode(self.det.run(None, {"x": x[None, None]})[0][0]), s
|
| 239 |
+
|
| 240 |
+
def detect(self, gray: np.ndarray) -> np.ndarray:
|
| 241 |
+
boxes, s = self._detect_at(gray, self.long_side)
|
| 242 |
+
if self.adaptive and len(boxes) >= 8:
|
| 243 |
+
med_h = float(np.median(boxes[:, 3] - boxes[:, 1]))
|
| 244 |
+
if med_h < ADAPTIVE_MIN_H:
|
| 245 |
+
ls = (
|
| 246 |
+
int(
|
| 247 |
+
min(
|
| 248 |
+
ADAPTIVE_MAX_SIDE,
|
| 249 |
+
self.long_side * ADAPTIVE_TARGET_H / max(med_h, 1.0),
|
| 250 |
+
)
|
| 251 |
+
)
|
| 252 |
+
// 32
|
| 253 |
+
* 32
|
| 254 |
+
)
|
| 255 |
+
if ls > self.long_side:
|
| 256 |
+
boxes, s = self._detect_at(gray, ls)
|
| 257 |
+
if len(boxes):
|
| 258 |
+
boxes[:, :4] /= s
|
| 259 |
+
h, w = gray.shape
|
| 260 |
+
boxes[:, [0, 2]] = boxes[:, [0, 2]].clip(0, w)
|
| 261 |
+
boxes[:, [1, 3]] = boxes[:, [1, 3]].clip(0, h)
|
| 262 |
+
return boxes
|
| 263 |
+
|
| 264 |
+
def recognize(
|
| 265 |
+
self, gray: np.ndarray, boxes: np.ndarray, batch: int = 64
|
| 266 |
+
) -> list[tuple[str, float, list]]:
|
| 267 |
+
"""(text, confidence, tokens) per line; tokens = [(token id, page x)],
|
| 268 |
+
x from the CTC peak's timestep mapped back through the crop. A line
|
| 269 |
+
under MIN_CONF comes back empty."""
|
| 270 |
+
# CenterMask first when the model folder has mask.onnx (same step as training)
|
| 271 |
+
crops = crop_lines(gray, boxes[:, :4], self.height, self.mask)
|
| 272 |
+
out: list = [("", 0.0, [])] * len(crops)
|
| 273 |
+
groups = [(self.rec, self.vocab, list(range(len(crops))), False)]
|
| 274 |
+
if self.script is not None and crops:
|
| 275 |
+
hanzi = self.hanzi_lines(crops, batch)
|
| 276 |
+
groups = [
|
| 277 |
+
(
|
| 278 |
+
self.rec,
|
| 279 |
+
self.vocab,
|
| 280 |
+
[j for j, h in enumerate(hanzi) if not h],
|
| 281 |
+
False,
|
| 282 |
+
),
|
| 283 |
+
(
|
| 284 |
+
self.rec_cjk,
|
| 285 |
+
self.vocab_cjk,
|
| 286 |
+
[j for j, h in enumerate(hanzi) if h],
|
| 287 |
+
True,
|
| 288 |
+
),
|
| 289 |
+
]
|
| 290 |
+
for sess, vocab, sel, as_chars in groups:
|
| 291 |
+
for part in width_batches([crops[j].shape[1] for j in sel], batch):
|
| 292 |
+
idx = [sel[i] for i in part]
|
| 293 |
+
arr, widths = pad_batch([crops[j] for j in idx])
|
| 294 |
+
best, pmax, lengths = sess.run(None, {"x": arr, "widths": widths})
|
| 295 |
+
stride = arr.shape[-1] / max(best.shape[1], 1)
|
| 296 |
+
for k, j in enumerate(idx):
|
| 297 |
+
n = int(lengths[k])
|
| 298 |
+
seq, pm = best[k, :n], pmax[k, :n]
|
| 299 |
+
keep = (seq != 0) & np.concatenate([[True], seq[1:] != seq[:-1]])
|
| 300 |
+
x0, y0, x1, y1 = (float(v) for v in boxes[j, :4])
|
| 301 |
+
h = max(1.0, y1 - y0)
|
| 302 |
+
ix0 = np.floor(x0 - PAD_X * h)
|
| 303 |
+
cw = np.ceil(x1 + PAD_X * h) - ix0
|
| 304 |
+
f = cw / crops[j].shape[1]
|
| 305 |
+
t = np.nonzero(keep)[0]
|
| 306 |
+
xs = (ix0 + (t + 0.5) * stride * f).tolist()
|
| 307 |
+
ids = seq[t].tolist()
|
| 308 |
+
# The CJK recognizer's ids mean nothing in self.vocab: its tokens
|
| 309 |
+
# are characters, like the text-layer tokens.
|
| 310 |
+
toks = (
|
| 311 |
+
list(zip([vocab.chars[i - 1] for i in ids], xs))
|
| 312 |
+
if as_chars
|
| 313 |
+
else list(zip(ids, xs))
|
| 314 |
+
)
|
| 315 |
+
conf = float(pm[keep].mean()) if keep.any() else 0.0
|
| 316 |
+
out[j] = (
|
| 317 |
+
(vocab.decode(seq[keep].tolist()), conf, toks)
|
| 318 |
+
if conf >= MIN_CONF
|
| 319 |
+
else ("", conf, [])
|
| 320 |
+
)
|
| 321 |
+
if getattr(self, "math", None) is not None and crops:
|
| 322 |
+
math = [j for j, f in enumerate(self.math.flags(crops)) if f]
|
| 323 |
+
if math:
|
| 324 |
+
for j, (text, conf) in zip(math, self.math.read(gray, boxes[math])):
|
| 325 |
+
# Only a confident reading with math replaces the line (one
|
| 326 |
+
# token holds the whole line: it is never split).
|
| 327 |
+
if "$" in text and conf >= MATH_MIN_CONF:
|
| 328 |
+
xc = float(boxes[j, 0] + boxes[j, 2]) / 2
|
| 329 |
+
out[j] = (text, 1.0, [(text, xc)])
|
| 330 |
+
return out
|
| 331 |
+
|
| 332 |
+
def hanzi_lines(self, crops: list, batch: int = 64) -> list[bool]:
|
| 333 |
+
"""Per line crop: does it hold Chinese characters (script.onnx)?"""
|
| 334 |
+
out = [False] * len(crops)
|
| 335 |
+
for idx in width_batches([c.shape[1] for c in crops], batch):
|
| 336 |
+
arr, widths = pad_batch([crops[j] for j in idx])
|
| 337 |
+
p = self.script.run(None, {"x": arr, "widths": widths})[0]
|
| 338 |
+
for k, j in enumerate(idx):
|
| 339 |
+
out[j] = bool(p[k] > 0.5)
|
| 340 |
+
return out
|
| 341 |
+
|
| 342 |
+
def gutter_cuts(self, boxes: np.ndarray, rec: list) -> list[list[float]]:
|
| 343 |
+
"""Per line, the x positions of gaps that sit on a column gutter: a gap over
|
| 344 |
+
GUTTER_GAP line heights where at least GUTTER_EDGES other lines end at its
|
| 345 |
+
left side or start at its right side."""
|
| 346 |
+
lefts, rights = boxes[:, 0], boxes[:, 2]
|
| 347 |
+
cuts: list[list[float]] = []
|
| 348 |
+
for i, (b, (_, _, toks)) in enumerate(zip(boxes, rec)):
|
| 349 |
+
h = max(1.0, float(b[3] - b[1]))
|
| 350 |
+
xs = [tk[1] for tk in toks if not self._is_space(tk[0])]
|
| 351 |
+
mine = []
|
| 352 |
+
for a, c in zip(xs[:-1], xs[1:]):
|
| 353 |
+
if c - a <= GUTTER_GAP * h:
|
| 354 |
+
continue
|
| 355 |
+
gl, gr = a + 0.35 * h, c - 0.35 * h # gap edges
|
| 356 |
+
near = (np.abs(rights - gl) < 0.5 * h) | (np.abs(lefts - gr) < 0.5 * h)
|
| 357 |
+
near[i] = False
|
| 358 |
+
if int(near.sum()) >= GUTTER_EDGES:
|
| 359 |
+
mine.append((a + c) / 2)
|
| 360 |
+
cuts.append(mine)
|
| 361 |
+
return cuts
|
| 362 |
+
|
| 363 |
+
def _is_space(self, tok) -> bool:
|
| 364 |
+
return tok == " " or tok == self.vocab.index.get(" ")
|
| 365 |
+
|
| 366 |
+
def _decode(self, toks: list) -> str:
|
| 367 |
+
"""Text of tokens: (vocab id, x) from the recognizer, or (char, x, content
|
| 368 |
+
index) from a PDF text layer."""
|
| 369 |
+
if toks and isinstance(toks[0][0], str):
|
| 370 |
+
return " ".join("".join(tk[0] for tk in toks).split())
|
| 371 |
+
return self.vocab.decode([tk[0] for tk in toks])
|
| 372 |
+
|
| 373 |
+
def split_line(
|
| 374 |
+
self, box, toks: list, gap: float, at: list | None = None
|
| 375 |
+
) -> list[tuple[list, str, list]]:
|
| 376 |
+
"""Cut a line where two neighbouring characters are more than ``gap``
|
| 377 |
+
line heights apart (the detector merged two cells or columns).
|
| 378 |
+
Returns [(box, text, tokens)]; one item when nothing is cut."""
|
| 379 |
+
h = max(1.0, float(box[3] - box[1]))
|
| 380 |
+
segs: list[list] = [[]]
|
| 381 |
+
prev = None
|
| 382 |
+
at = sorted(at or [])
|
| 383 |
+
for tk in toks:
|
| 384 |
+
x = tk[1]
|
| 385 |
+
if not self._is_space(tk[0]):
|
| 386 |
+
cut = prev is not None and (
|
| 387 |
+
x - prev > gap * h or any(prev < c < x for c in at)
|
| 388 |
+
)
|
| 389 |
+
if cut:
|
| 390 |
+
segs.append([])
|
| 391 |
+
prev = x
|
| 392 |
+
segs[-1].append(tk)
|
| 393 |
+
if len(segs) == 1:
|
| 394 |
+
return [(list(box[:4]), self._decode(toks), toks)]
|
| 395 |
+
out = []
|
| 396 |
+
for sg in segs:
|
| 397 |
+
text = self._decode(sg).strip()
|
| 398 |
+
xs = [tk[1] for tk in sg if not self._is_space(tk[0])]
|
| 399 |
+
if text and xs:
|
| 400 |
+
bx = [
|
| 401 |
+
max(box[0], min(xs) - 0.35 * h),
|
| 402 |
+
box[1],
|
| 403 |
+
min(box[2], max(xs) + 0.35 * h),
|
| 404 |
+
box[3],
|
| 405 |
+
]
|
| 406 |
+
out.append((bx, text, sg))
|
| 407 |
+
return out
|
| 408 |
+
|
| 409 |
+
def reading_order(
|
| 410 |
+
self, boxes: np.ndarray, texts: list[str], wh: tuple[int, int]
|
| 411 |
+
) -> tuple[list[int], tuple | None, np.ndarray | None]:
|
| 412 |
+
"""Reading order, (cls logits, same-block logits, table logits) when the
|
| 413 |
+
order model has layout heads, and the pairwise order logits."""
|
| 414 |
+
if len(boxes) < 2 or self.order is None:
|
| 415 |
+
order = order_xycut(boxes[:, :4]) if len(boxes) else []
|
| 416 |
+
return order, None, None
|
| 417 |
+
f = line_features(boxes[:, :4], wh, texts)[None]
|
| 418 |
+
outs = self.order.run(
|
| 419 |
+
None, {"feats": f, "mask": np.ones((1, len(boxes)), bool)}
|
| 420 |
+
)
|
| 421 |
+
if len(outs) >= 3:
|
| 422 |
+
# Layout heads: blocks from the same-block head, lines inside a block by
|
| 423 |
+
# geometry, blocks by the order head (exact page order 0.68 -> 0.81 on eval).
|
| 424 |
+
cls, same = outs[1][0], outs[2][0]
|
| 425 |
+
# same row / column / cell logits, when asked for
|
| 426 |
+
tab = outs[3][0] if len(outs) > 3 and self.table_heads else None
|
| 427 |
+
order = decode_blocks_order(outs[0][0], same, boxes[:, :4], cls)
|
| 428 |
+
return order, (cls, same, tab), outs[0][0]
|
| 429 |
+
return decode_order(outs[0][0]), None, outs[0][0]
|
| 430 |
+
|
| 431 |
+
def orientation(self, gray: np.ndarray) -> int:
|
| 432 |
+
"""Quarter turns (counter-clockwise) the page content is turned by, from
|
| 433 |
+
orient.onnx: 0 without the model or when no turn is more likely than ORIENT_MIN."""
|
| 434 |
+
if self.orient is None:
|
| 435 |
+
return 0
|
| 436 |
+
p = self.orient.run(None, {"x": orient_input(gray)[None, None]})[0][0]
|
| 437 |
+
k = int(p.argmax())
|
| 438 |
+
return k if p[k] > ORIENT_MIN else 0
|
| 439 |
+
|
| 440 |
+
def upright(
|
| 441 |
+
self, gray: np.ndarray, turn: bool = True
|
| 442 |
+
) -> tuple[np.ndarray, int, float]:
|
| 443 |
+
"""The page turned upright (orient.onnx) and straightened (estimate_skew):
|
| 444 |
+
(page, quarter turns undone, degrees of skew undone)."""
|
| 445 |
+
k = self.orientation(gray) if turn else 0
|
| 446 |
+
if k:
|
| 447 |
+
gray = np.ascontiguousarray(np.rot90(gray, -k))
|
| 448 |
+
a = estimate_skew(gray)
|
| 449 |
+
if abs(a) >= DESKEW_MIN:
|
| 450 |
+
gray = rotate_page(gray, a)
|
| 451 |
+
else:
|
| 452 |
+
a = 0.0
|
| 453 |
+
return gray, k, a
|
| 454 |
+
|
| 455 |
+
def __call__(self, gray: np.ndarray, turn: bool = True) -> Page:
|
| 456 |
+
"""OCR of a page image. turn=False: the page is known to be upright (no
|
| 457 |
+
orient.onnx)."""
|
| 458 |
+
t = {}
|
| 459 |
+
t0 = time.perf_counter()
|
| 460 |
+
gray, k, skew = self.upright(gray, turn)
|
| 461 |
+
t["orient_ms"] = 1000 * (time.perf_counter() - t0)
|
| 462 |
+
t0 = time.perf_counter()
|
| 463 |
+
boxes = self.detect(gray)
|
| 464 |
+
t["det_ms"] = 1000 * (time.perf_counter() - t0)
|
| 465 |
+
t0 = time.perf_counter()
|
| 466 |
+
rec = self.recognize(gray, boxes)
|
| 467 |
+
t["rec_ms"] = 1000 * (time.perf_counter() - t0)
|
| 468 |
+
page = self.assemble((gray.shape[1], gray.shape[0]), boxes, rec, t)
|
| 469 |
+
page.rotation, page.skew = 90 * k, round(skew, 2)
|
| 470 |
+
if skew: # boxes back on the page as turned, before the deskew
|
| 471 |
+
wh = (gray.shape[1], gray.shape[0])
|
| 472 |
+
for ln in page.lines:
|
| 473 |
+
ln.box = unrotate_box(ln.box, skew, wh)
|
| 474 |
+
for b in page.blocks:
|
| 475 |
+
b["bbox"] = unrotate_box(b["bbox"], skew, wh)
|
| 476 |
+
return page
|
| 477 |
+
|
| 478 |
+
def assemble(
|
| 479 |
+
self, wh, boxes: np.ndarray, rec: list, t: dict, content_order: bool = False
|
| 480 |
+
) -> Page:
|
| 481 |
+
"""Lines -> split at wide gaps -> reading order -> markdown.
|
| 482 |
+
|
| 483 |
+
``rec`` holds (text, confidence, tokens) per detected line. With
|
| 484 |
+
``content_order`` and text-layer tokens (char, x, content index) on every
|
| 485 |
+
line, the PDF content order is kept when it is coherent."""
|
| 486 |
+
t0 = time.perf_counter()
|
| 487 |
+
# Text-layer labels split a line at wide gaps; do the same (SPLIT_GAP).
|
| 488 |
+
nb, nrec = [], []
|
| 489 |
+
cuts = self.gutter_cuts(boxes, rec) if len(boxes) else []
|
| 490 |
+
for b, (text, conf, toks), at in zip(boxes, rec, cuts):
|
| 491 |
+
for bx, tx, tk in self.split_line(b, toks, SPLIT_GAP, at):
|
| 492 |
+
nb.append(bx)
|
| 493 |
+
nrec.append((tx, conf, tk))
|
| 494 |
+
boxes = np.array(nb, np.float32).reshape(-1, 4)
|
| 495 |
+
rec = nrec
|
| 496 |
+
texts = [r[0] for r in rec]
|
| 497 |
+
order, layout, logits = self.reading_order(boxes, texts, wh)
|
| 498 |
+
if layout is not None and layout[2] is not None:
|
| 499 |
+
# The table heads learned one line per cell, but the detector can join
|
| 500 |
+
# cells: split table lines into cells, then run the order model again.
|
| 501 |
+
table_cls = BLOCK_CLASSES.index("Table")
|
| 502 |
+
is_table = layout[0].argmax(-1) == table_cls
|
| 503 |
+
if is_table.any():
|
| 504 |
+
nb, nrec = [], []
|
| 505 |
+
for i, (b, r) in enumerate(zip(boxes, rec)):
|
| 506 |
+
parts = (
|
| 507 |
+
self.split_line(b, r[2], CELL_GAP)
|
| 508 |
+
if is_table[i]
|
| 509 |
+
else [(b, r[0], r[2])]
|
| 510 |
+
)
|
| 511 |
+
for bx, tx, tk in parts:
|
| 512 |
+
nb.append(bx)
|
| 513 |
+
nrec.append((tx, r[1], tk))
|
| 514 |
+
if len(nb) != len(boxes):
|
| 515 |
+
boxes = np.array(nb, np.float32).reshape(-1, 4)
|
| 516 |
+
rec = nrec
|
| 517 |
+
texts = [r[0] for r in rec]
|
| 518 |
+
order, layout, logits = self.reading_order(boxes, texts, wh)
|
| 519 |
+
note = "order=model"
|
| 520 |
+
if content_order and len(boxes) > 1:
|
| 521 |
+
keys = [
|
| 522 |
+
min((tk[2] for tk in r[2] if len(tk) > 2), default=None) for r in rec
|
| 523 |
+
]
|
| 524 |
+
if all(k is not None for k in keys):
|
| 525 |
+
pdf = sorted(range(len(keys)), key=lambda i: keys[i])
|
| 526 |
+
ok = pdf_order_coherent(
|
| 527 |
+
boxes[pdf, :4],
|
| 528 |
+
None if logits is None else logits[np.ix_(pdf, pdf)],
|
| 529 |
+
)
|
| 530 |
+
if ok:
|
| 531 |
+
order, note = pdf, "order=pdf"
|
| 532 |
+
t["order_ms"] = 1000 * (time.perf_counter() - t0)
|
| 533 |
+
t["total_ms"] = sum(v for k, v in t.items() if k != "total_ms")
|
| 534 |
+
lines = [
|
| 535 |
+
Line(
|
| 536 |
+
[round(float(v), 1) for v in boxes[i, :4]],
|
| 537 |
+
rec[i][0],
|
| 538 |
+
round(rec[i][1], 4),
|
| 539 |
+
)
|
| 540 |
+
for i in order
|
| 541 |
+
]
|
| 542 |
+
md, blocks = None, []
|
| 543 |
+
if layout is not None:
|
| 544 |
+
blocks = page_blocks(boxes[:, :4], texts, order, layout[0], layout[1])
|
| 545 |
+
# Inside tables, labels split cells at smaller gaps (CELL_GAP).
|
| 546 |
+
cells = {
|
| 547 |
+
i: [
|
| 548 |
+
(bx, tx)
|
| 549 |
+
for bx, tx, _ in self.split_line(boxes[i], rec[i][2], CELL_GAP)
|
| 550 |
+
]
|
| 551 |
+
for i in range(len(boxes))
|
| 552 |
+
}
|
| 553 |
+
md = page_markdown(
|
| 554 |
+
boxes[:, :4],
|
| 555 |
+
texts,
|
| 556 |
+
order,
|
| 557 |
+
layout[0],
|
| 558 |
+
layout[1],
|
| 559 |
+
keep_furniture=self.furniture,
|
| 560 |
+
cells=cells,
|
| 561 |
+
tab=layout[2],
|
| 562 |
+
)
|
| 563 |
+
elif len(boxes) < 2:
|
| 564 |
+
md = "\n".join(texts)
|
| 565 |
+
return Page(lines, t, md, note=note, blocks=blocks)
|
| 566 |
+
|
| 567 |
+
|
| 568 |
+
class InputError(Exception):
|
| 569 |
+
"""A file that cannot be read: not a PDF or an image, corrupt, or locked."""
|
| 570 |
+
|
| 571 |
+
|
| 572 |
+
def is_pdf(path: Path) -> bool:
|
| 573 |
+
"""The file's own bytes say PDF (the header may follow a little junk)."""
|
| 574 |
+
with open(path, "rb") as f:
|
| 575 |
+
return b"%PDF-" in f.read(1024)
|
| 576 |
+
|
| 577 |
+
|
| 578 |
+
def read_images(path: Path) -> list[np.ndarray]:
|
| 579 |
+
"""Grayscale pages of an image file; every frame of a multi-page TIFF."""
|
| 580 |
+
ok, frames = cv2.imreadmulti(str(path), flags=cv2.IMREAD_GRAYSCALE)
|
| 581 |
+
if not ok or not frames:
|
| 582 |
+
img = cv2.imread(str(path), cv2.IMREAD_GRAYSCALE)
|
| 583 |
+
frames = [img] if img is not None else []
|
| 584 |
+
if not frames:
|
| 585 |
+
raise InputError(f"{path}: not a PDF or a readable image")
|
| 586 |
+
return [np.ascontiguousarray(f) for f in frames]
|
| 587 |
+
|
| 588 |
+
|
| 589 |
+
def load_pages(path: Path, dpi: float = 200.0):
|
| 590 |
+
"""Yield grayscale pages of a PDF (rendered at ``dpi``) or an image file
|
| 591 |
+
(every frame of a multi-page TIFF); the file's bytes decide which."""
|
| 592 |
+
path = Path(path)
|
| 593 |
+
if is_pdf(path):
|
| 594 |
+
import pypdfium2 as pdfium
|
| 595 |
+
|
| 596 |
+
doc = pdfium.PdfDocument(str(path))
|
| 597 |
+
try:
|
| 598 |
+
for i in range(len(doc)):
|
| 599 |
+
arr = doc[i].render(scale=dpi / 72.0, grayscale=True).to_numpy()
|
| 600 |
+
yield i, np.ascontiguousarray(arr[..., 0] if arr.ndim == 3 else arr)
|
| 601 |
+
finally:
|
| 602 |
+
doc.close()
|
| 603 |
+
else:
|
| 604 |
+
yield from enumerate(read_images(path))
|
OCR_INDRA/indra/indra_ocr/rec_preproc.py
ADDED
|
@@ -0,0 +1,97 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""Line crop rule shared by recognition training and inference.
|
| 2 |
+
|
| 3 |
+
A recognizer input is made from a line box (the ink box from the detector, or
|
| 4 |
+
the ink box from a synthetic or real label) in three steps:
|
| 5 |
+
1. expand the box by PAD_Y x its height on top and bottom and PAD_X x its
|
| 6 |
+
height on left and right,
|
| 7 |
+
2. crop that region from the page (white outside the page),
|
| 8 |
+
3. resize the crop to LINE_HEIGHT pixels high, keeping the aspect ratio.
|
| 9 |
+
|
| 10 |
+
The result is an ink map (0 = paper, 1 = ink) of shape [LINE_HEIGHT, W].
|
| 11 |
+
Training and inference must both call ``crop_line``, so the text scale and the
|
| 12 |
+
margins match.
|
| 13 |
+
"""
|
| 14 |
+
|
| 15 |
+
from __future__ import annotations
|
| 16 |
+
|
| 17 |
+
import math
|
| 18 |
+
|
| 19 |
+
import numpy as np
|
| 20 |
+
|
| 21 |
+
LINE_HEIGHT = 32
|
| 22 |
+
PAD_Y = 0.2
|
| 23 |
+
PAD_X = 0.3
|
| 24 |
+
WIDTH_MULTIPLE = 8 # widths are padded to a multiple of this (timestep stride)
|
| 25 |
+
|
| 26 |
+
|
| 27 |
+
def crop_line(
|
| 28 |
+
gray: np.ndarray,
|
| 29 |
+
box,
|
| 30 |
+
height: int = LINE_HEIGHT,
|
| 31 |
+
pad_y: float = PAD_Y,
|
| 32 |
+
pad_x: float = PAD_X,
|
| 33 |
+
max_width: int = 4096,
|
| 34 |
+
) -> np.ndarray:
|
| 35 |
+
"""Crop one line from a uint8 grayscale page and return a float32 ink map [height, W]."""
|
| 36 |
+
import cv2
|
| 37 |
+
|
| 38 |
+
x0, y0, x1, y1 = (float(v) for v in box[:4])
|
| 39 |
+
h = max(1.0, y1 - y0)
|
| 40 |
+
X0, X1 = x0 - pad_x * h, x1 + pad_x * h
|
| 41 |
+
Y0, Y1 = y0 - pad_y * h, y1 + pad_y * h
|
| 42 |
+
H, W = gray.shape
|
| 43 |
+
ix0, iy0 = int(math.floor(X0)), int(math.floor(Y0))
|
| 44 |
+
ix1, iy1 = int(math.ceil(X1)), int(math.ceil(Y1))
|
| 45 |
+
cw, ch = max(1, ix1 - ix0), max(1, iy1 - iy0)
|
| 46 |
+
crop = np.full((ch, cw), 255, np.uint8)
|
| 47 |
+
sx0, sy0, sx1, sy1 = max(0, ix0), max(0, iy0), min(W, ix1), min(H, iy1)
|
| 48 |
+
if sx1 > sx0 and sy1 > sy0:
|
| 49 |
+
crop[sy0 - iy0 : sy1 - iy0, sx0 - ix0 : sx1 - ix0] = gray[sy0:sy1, sx0:sx1]
|
| 50 |
+
out_w = max(WIDTH_MULTIPLE, min(max_width, round(cw * height / ch)))
|
| 51 |
+
interp = cv2.INTER_AREA if ch > height else cv2.INTER_LINEAR
|
| 52 |
+
resized = cv2.resize(crop, (out_w, height), interpolation=interp)
|
| 53 |
+
return 1.0 - resized.astype(np.float32) / 255.0
|
| 54 |
+
|
| 55 |
+
|
| 56 |
+
def pad_batch(lines: list[np.ndarray], multiple: int = WIDTH_MULTIPLE):
|
| 57 |
+
"""Right-pad ink maps to one width. Returns (array [B, 1, H, W], widths [B])."""
|
| 58 |
+
H = lines[0].shape[0]
|
| 59 |
+
Wm = max(x.shape[1] for x in lines)
|
| 60 |
+
Wm = int(math.ceil(Wm / multiple) * multiple)
|
| 61 |
+
out = np.zeros((len(lines), 1, H, Wm), np.float32)
|
| 62 |
+
widths = np.zeros(len(lines), np.int64)
|
| 63 |
+
for i, x in enumerate(lines):
|
| 64 |
+
out[i, 0, :, : x.shape[1]] = x
|
| 65 |
+
widths[i] = x.shape[1]
|
| 66 |
+
return out, widths
|
| 67 |
+
|
| 68 |
+
|
| 69 |
+
def width_batches(
|
| 70 |
+
widths: list[int],
|
| 71 |
+
max_batch: int = 64,
|
| 72 |
+
max_ratio: float = 1.3,
|
| 73 |
+
min_width: int = 64,
|
| 74 |
+
max_area: int = 64 * 640,
|
| 75 |
+
) -> list[list[int]]:
|
| 76 |
+
"""Group line indices into batches of similar width, so little compute goes to padding.
|
| 77 |
+
|
| 78 |
+
Lines are taken narrowest first. A batch closes at ``max_batch`` lines, when the
|
| 79 |
+
next line is wider than ``max_ratio`` times the batch's first line (widths below
|
| 80 |
+
``min_width`` count as ``min_width``, so short lines do not fragment into tiny batches),
|
| 81 |
+
or when lines x width would pass ``max_area`` pixel columns: ONNX Runtime keeps its
|
| 82 |
+
largest allocation, and 64 lines of 4,096 px held 5 GB on large scanned pages.
|
| 83 |
+
"""
|
| 84 |
+
order = sorted(range(len(widths)), key=lambda i: widths[i])
|
| 85 |
+
batches: list[list[int]] = []
|
| 86 |
+
for i in order:
|
| 87 |
+
if (
|
| 88 |
+
batches
|
| 89 |
+
and len(batches[-1]) < max_batch
|
| 90 |
+
and max(widths[i], min_width)
|
| 91 |
+
<= max_ratio * max(widths[batches[-1][0]], min_width)
|
| 92 |
+
and (len(batches[-1]) + 1) * max(widths[i], min_width) <= max_area
|
| 93 |
+
):
|
| 94 |
+
batches[-1].append(i)
|
| 95 |
+
else:
|
| 96 |
+
batches.append([i])
|
| 97 |
+
return batches
|
OCR_INDRA/indra/indra_ocr/textlayer.py
ADDED
|
@@ -0,0 +1,253 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""Text-layer runs from a PDF page (pdfium), and checks that decide whether to
|
| 2 |
+
trust them.
|
| 3 |
+
|
| 4 |
+
A digital PDF already holds its text. When the text layer is sound, reading it
|
| 5 |
+
is faster than OCR and has no recognition errors. The checks follow the rules
|
| 6 |
+
that select the recognizer's own training labels:
|
| 7 |
+
|
| 8 |
+
- invisible text (render mode 3) marks an OCR layer over a scan;
|
| 9 |
+
- replacement, private-use or control characters mark a broken font map;
|
| 10 |
+
- a text run with no ink under it marks hidden or misplaced text.
|
| 11 |
+
|
| 12 |
+
A run is pdfium's own text segment between the line breaks it generates.
|
| 13 |
+
Coordinates are PDF points in the frame that pypdfium2 renders (crop box and
|
| 14 |
+
page rotation applied), with a top-left origin.
|
| 15 |
+
"""
|
| 16 |
+
|
| 17 |
+
from __future__ import annotations
|
| 18 |
+
|
| 19 |
+
import math
|
| 20 |
+
import unicodedata
|
| 21 |
+
from dataclasses import dataclass, field
|
| 22 |
+
|
| 23 |
+
import numpy as np
|
| 24 |
+
|
| 25 |
+
MAX_INVISIBLE_FRAC = 0.05
|
| 26 |
+
MAX_BAD_CHAR_FRAC = 0.005
|
| 27 |
+
MAX_NO_INK_LINE_FRAC = 0.03
|
| 28 |
+
DARK = 160 # grey level below which a pixel counts as ink
|
| 29 |
+
|
| 30 |
+
|
| 31 |
+
def bad_char_count(text: str) -> int:
|
| 32 |
+
n = 0
|
| 33 |
+
for c in text:
|
| 34 |
+
cat = unicodedata.category(c)
|
| 35 |
+
if c == "�" or cat in ("Co", "Cs", "Cn") or (cat == "Cc" and c != "\t"):
|
| 36 |
+
n += 1
|
| 37 |
+
return n
|
| 38 |
+
|
| 39 |
+
|
| 40 |
+
BROKEN_FONT_MAP = "broken font map" # TextLayer.reason: the layer's codes are not text
|
| 41 |
+
|
| 42 |
+
|
| 43 |
+
@dataclass
|
| 44 |
+
class TextLine:
|
| 45 |
+
"""One text-layer run: its box, text, and characters with boxes."""
|
| 46 |
+
|
| 47 |
+
bbox: tuple[float, float, float, float]
|
| 48 |
+
text: str
|
| 49 |
+
chars: list = field(default_factory=list) # (char, bbox) in PDF order
|
| 50 |
+
|
| 51 |
+
|
| 52 |
+
@dataclass
|
| 53 |
+
class TextLayer:
|
| 54 |
+
lines: list[TextLine] # runs in PDF content order
|
| 55 |
+
size: tuple[float, float] # page width, height in points (as rendered)
|
| 56 |
+
trusted: bool
|
| 57 |
+
reason: str = "" # why the layer is not trusted
|
| 58 |
+
stats: dict = field(default_factory=dict)
|
| 59 |
+
|
| 60 |
+
|
| 61 |
+
def _device_affine(page) -> np.ndarray:
|
| 62 |
+
"""[2, 3] map from PDF page space to the rendered frame (points, top-left
|
| 63 |
+
origin), from pdfium's own FPDF_PageToDevice."""
|
| 64 |
+
import ctypes
|
| 65 |
+
|
| 66 |
+
import pypdfium2.raw as R
|
| 67 |
+
|
| 68 |
+
W, H = page.get_size()
|
| 69 |
+
S = 1000 # device units per point: keeps the integer output precise
|
| 70 |
+
|
| 71 |
+
def dev(x, y):
|
| 72 |
+
dx, dy = ctypes.c_int(), ctypes.c_int()
|
| 73 |
+
R.FPDF_PageToDevice(page.raw, 0, 0, round(W * S), round(H * S), 0, x, y, dx, dy)
|
| 74 |
+
return np.array([dx.value, dy.value], float) / S
|
| 75 |
+
|
| 76 |
+
o, ex, ey = dev(0.0, 0.0), dev(1.0, 0.0), dev(0.0, 1.0)
|
| 77 |
+
return np.stack([ex - o, ey - o, o], 1)
|
| 78 |
+
|
| 79 |
+
|
| 80 |
+
def extract_runs(page) -> tuple[list[TextLine], dict]:
|
| 81 |
+
"""Upright text runs of a pypdfium2 page, in content order."""
|
| 82 |
+
import ctypes
|
| 83 |
+
|
| 84 |
+
import pypdfium2
|
| 85 |
+
import pypdfium2.raw as R
|
| 86 |
+
|
| 87 |
+
tp = page.get_textpage()
|
| 88 |
+
try:
|
| 89 |
+
n = tp.count_chars()
|
| 90 |
+
# A character is upright on the rendered page when its baseline, mapped
|
| 91 |
+
# through the page-to-render transform, runs left to right (any /Rotate,
|
| 92 |
+
# any mirroring).
|
| 93 |
+
L = _device_affine(page)[:, :2]
|
| 94 |
+
# The loop below runs once per character: raw pdfium calls with reused
|
| 95 |
+
# buffers, and the upright and font-map tests cached per angle and per
|
| 96 |
+
# character (a page has few distinct ones).
|
| 97 |
+
get_unicode, get_angle = R.FPDFText_GetUnicode, R.FPDFText_GetCharAngle
|
| 98 |
+
get_box, raw_tp = R.FPDFText_GetCharBox, tp.raw
|
| 99 |
+
bl, br, bb, bt = (ctypes.c_double() for _ in range(4))
|
| 100 |
+
pl, pr, pb, pt = (ctypes.byref(v) for v in (bl, br, bb, bt))
|
| 101 |
+
upright: dict = {}
|
| 102 |
+
bad: dict = {}
|
| 103 |
+
chars, raw, breaks = [], [], [] # breaks: run ends after these positions
|
| 104 |
+
n_chars = n_rot = n_bad = 0
|
| 105 |
+
for i in range(n):
|
| 106 |
+
c = chr(get_unicode(raw_tp, i))
|
| 107 |
+
if c in "\r\n":
|
| 108 |
+
breaks.append(len(chars))
|
| 109 |
+
continue
|
| 110 |
+
if c == "\x02": # pdfium's mark for a hyphen at a line break
|
| 111 |
+
c = "-"
|
| 112 |
+
if not c.isspace():
|
| 113 |
+
n_chars += 1
|
| 114 |
+
k = bad.get(c)
|
| 115 |
+
if k is None:
|
| 116 |
+
k = bad[c] = bad_char_count(c)
|
| 117 |
+
n_bad += k
|
| 118 |
+
a = get_angle(raw_tp, i)
|
| 119 |
+
up = upright.get(a)
|
| 120 |
+
if up is None:
|
| 121 |
+
d = L @ np.array([math.cos(a), math.sin(a)])
|
| 122 |
+
up = upright[a] = not (
|
| 123 |
+
d[0] <= 0 or abs(d[1]) > 1e-3 * max(abs(d[0]), 1e-9)
|
| 124 |
+
)
|
| 125 |
+
if not up:
|
| 126 |
+
n_rot += 1 # not upright on the rendered page
|
| 127 |
+
breaks.append(len(chars))
|
| 128 |
+
continue
|
| 129 |
+
chars.append(c)
|
| 130 |
+
if not get_box(raw_tp, i, pl, pr, pb, pt): # yes, l r b t
|
| 131 |
+
raise pypdfium2.PdfiumError("Failed to get charbox.")
|
| 132 |
+
raw.append((bl.value, bb.value, br.value, bt.value)) # l, b, r, t
|
| 133 |
+
finally:
|
| 134 |
+
tp.close()
|
| 135 |
+
stats = {"chars": n_chars, "rotated": n_rot, "bad": n_bad}
|
| 136 |
+
if not chars:
|
| 137 |
+
return [], stats
|
| 138 |
+
# Page space -> rendered frame, all characters at once.
|
| 139 |
+
A = _device_affine(page)
|
| 140 |
+
b = np.asarray(raw, float)
|
| 141 |
+
xs = b[:, [0, 2, 0, 2]]
|
| 142 |
+
ys = b[:, [1, 1, 3, 3]]
|
| 143 |
+
px = A[0, 0] * xs + A[0, 1] * ys + A[0, 2]
|
| 144 |
+
py = A[1, 0] * xs + A[1, 1] * ys + A[1, 2]
|
| 145 |
+
boxes = np.stack([px.min(1), py.min(1), px.max(1), py.max(1)], 1)
|
| 146 |
+
# Characters outside the rendered frame (beyond a crop box) are not on the
|
| 147 |
+
# page: they are dropped, not read.
|
| 148 |
+
W, H = page.get_size()
|
| 149 |
+
cx, cy = (boxes[:, 0] + boxes[:, 2]) / 2, (boxes[:, 1] + boxes[:, 3]) / 2
|
| 150 |
+
visible = (cx >= 0) & (cx <= W) & (cy >= 0) & (cy <= H)
|
| 151 |
+
stats["off_page"] = int((~visible).sum())
|
| 152 |
+
runs: list[TextLine] = []
|
| 153 |
+
cuts = sorted(set(breaks + [len(chars)]))
|
| 154 |
+
start = 0
|
| 155 |
+
for end in cuts:
|
| 156 |
+
seg = [(chars[i], tuple(boxes[i])) for i in range(start, end) if visible[i]]
|
| 157 |
+
start = end
|
| 158 |
+
ink = [bx for c, bx in seg if not c.isspace()]
|
| 159 |
+
if not ink:
|
| 160 |
+
continue
|
| 161 |
+
box = (
|
| 162 |
+
min(v[0] for v in ink),
|
| 163 |
+
min(v[1] for v in ink),
|
| 164 |
+
max(v[2] for v in ink),
|
| 165 |
+
max(v[3] for v in ink),
|
| 166 |
+
)
|
| 167 |
+
runs.append(TextLine(box, " ".join("".join(c for c, _ in seg).split()), seg))
|
| 168 |
+
return runs, stats
|
| 169 |
+
|
| 170 |
+
|
| 171 |
+
def invisible_text_frac(page) -> float:
|
| 172 |
+
"""Share of text objects drawn invisibly (an OCR layer) on a pypdfium2 page."""
|
| 173 |
+
import pypdfium2.raw as R
|
| 174 |
+
|
| 175 |
+
n = inv = 0
|
| 176 |
+
for obj in page.get_objects(max_depth=4):
|
| 177 |
+
if obj.type == R.FPDF_PAGEOBJ_TEXT:
|
| 178 |
+
n += 1
|
| 179 |
+
inv += (
|
| 180 |
+
R.FPDFTextObj_GetTextRenderMode(obj.raw)
|
| 181 |
+
== R.FPDF_TEXTRENDERMODE_INVISIBLE
|
| 182 |
+
)
|
| 183 |
+
return inv / n if n else 0.0
|
| 184 |
+
|
| 185 |
+
|
| 186 |
+
def no_ink_frac(gray: np.ndarray, runs: list[TextLine], scale: float) -> float:
|
| 187 |
+
"""Share of runs whose box holds no dark pixel in ``gray`` (``scale`` px/pt)."""
|
| 188 |
+
ink = gray < DARK
|
| 189 |
+
H, W = ink.shape
|
| 190 |
+
empty = 0
|
| 191 |
+
for ln in runs:
|
| 192 |
+
x0, y0, x1, y1 = ln.bbox
|
| 193 |
+
r = slice(max(0, int(y0 * scale)), min(H, int(y1 * scale) + 1))
|
| 194 |
+
c = slice(max(0, int(x0 * scale)), min(W, int(x1 * scale) + 1))
|
| 195 |
+
if not ink[r, c].any():
|
| 196 |
+
empty += 1
|
| 197 |
+
return empty / max(len(runs), 1)
|
| 198 |
+
|
| 199 |
+
|
| 200 |
+
def read_text_layer(page, gray: np.ndarray, scale: float) -> TextLayer:
|
| 201 |
+
"""Text-layer runs of a page and the verdict on them.
|
| 202 |
+
|
| 203 |
+
``page`` is the open pypdfium2 page, ``gray`` a render of it at ``scale``
|
| 204 |
+
pixels per point (a low resolution is enough for the ink check)."""
|
| 205 |
+
size = page.get_size()
|
| 206 |
+
inv = invisible_text_frac(page)
|
| 207 |
+
if inv > MAX_INVISIBLE_FRAC:
|
| 208 |
+
return TextLayer([], size, False, f"invisible text {inv:.0%} (OCR layer)")
|
| 209 |
+
runs, st = extract_runs(page)
|
| 210 |
+
if not runs:
|
| 211 |
+
return TextLayer([], size, False, "no text layer", st)
|
| 212 |
+
if st["bad"] > MAX_BAD_CHAR_FRAC * max(st["chars"], 1):
|
| 213 |
+
return TextLayer(runs, size, False, BROKEN_FONT_MAP, st)
|
| 214 |
+
empty = no_ink_frac(gray, runs, scale)
|
| 215 |
+
st["no_ink"] = round(empty, 4)
|
| 216 |
+
if empty > MAX_NO_INK_LINE_FRAC:
|
| 217 |
+
return TextLayer(runs, size, False, f"{empty:.0%} of text runs have no ink", st)
|
| 218 |
+
return TextLayer(runs, size, True, "", st)
|
| 219 |
+
|
| 220 |
+
|
| 221 |
+
def order_backtracks(boxes: np.ndarray) -> int:
|
| 222 |
+
"""Moves upward in content order that do not go to a column on the right.
|
| 223 |
+
|
| 224 |
+
Reading goes down a column, then up to the top of the next column on the
|
| 225 |
+
right. Any other upward move means the content order is not reading order."""
|
| 226 |
+
if len(boxes) < 3:
|
| 227 |
+
return 0
|
| 228 |
+
h = float(np.median(boxes[:, 3] - boxes[:, 1]))
|
| 229 |
+
bt = 0
|
| 230 |
+
for a, b in zip(boxes[:-1], boxes[1:]):
|
| 231 |
+
if b[1] < a[1] - 0.5 * h and b[0] < a[2] - h:
|
| 232 |
+
bt += 1
|
| 233 |
+
return bt
|
| 234 |
+
|
| 235 |
+
|
| 236 |
+
MIN_MODEL_AGREEMENT = 0.85
|
| 237 |
+
|
| 238 |
+
|
| 239 |
+
def pdf_order_coherent(
|
| 240 |
+
boxes: np.ndarray, pair_logits: np.ndarray | None = None
|
| 241 |
+
) -> bool:
|
| 242 |
+
"""Is the content order a reading order?
|
| 243 |
+
|
| 244 |
+
Few upward moves that are not column changes, and (when the order model's
|
| 245 |
+
pairwise logits are given, ``a[i, j] > 0``: i before j) the model agrees with
|
| 246 |
+
most consecutive pairs. The second check catches two columns written line by
|
| 247 |
+
line across the gutter, which has no upward move at all."""
|
| 248 |
+
if order_backtracks(boxes) > max(1, int(0.02 * len(boxes))):
|
| 249 |
+
return False
|
| 250 |
+
if pair_logits is None or len(boxes) < 3:
|
| 251 |
+
return True
|
| 252 |
+
i = np.arange(len(boxes) - 1)
|
| 253 |
+
return float((pair_logits[i, i + 1] > 0).mean()) >= MIN_MODEL_AGREEMENT
|
OCR_INDRA/indra/indra_ocr/viewer.py
ADDED
|
@@ -0,0 +1,649 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""Local viewer: drop in a PDF or an image, see what Indra OCR reads.
|
| 2 |
+
|
| 3 |
+
indra-ocr-viewer # models from the Hugging Face repo
|
| 4 |
+
open http://localhost:8765
|
| 5 |
+
|
| 6 |
+
Each page shows its lines, layout blocks and reading order over the page (each
|
| 7 |
+
can be shown or hidden), the rendered markdown, the markdown source, the lines
|
| 8 |
+
with their confidences, the JSON the CLI writes, where the text came from (the
|
| 9 |
+
PDF text layer, OCR, or both) and the time of each stage. The pipeline options
|
| 10 |
+
(text layer or OCR, math, page headers and footers, reading order, OCR of text
|
| 11 |
+
inside images, dpi) are set in the page and apply when the document is read
|
| 12 |
+
again. Pages are read one at a time and appear as they finish. With
|
| 13 |
+
--tesseract (needs the tesseract binary and pytesseract) the page can also run
|
| 14 |
+
Tesseract on each page.
|
| 15 |
+
"""
|
| 16 |
+
|
| 17 |
+
from __future__ import annotations
|
| 18 |
+
|
| 19 |
+
import argparse
|
| 20 |
+
import base64
|
| 21 |
+
import json
|
| 22 |
+
import os
|
| 23 |
+
import tempfile
|
| 24 |
+
import threading
|
| 25 |
+
import time
|
| 26 |
+
from dataclasses import asdict
|
| 27 |
+
from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer
|
| 28 |
+
from pathlib import Path
|
| 29 |
+
|
| 30 |
+
import numpy as np
|
| 31 |
+
|
| 32 |
+
from . import __version__
|
| 33 |
+
from .cli import add_source_args, source_options
|
| 34 |
+
from .document import Converter
|
| 35 |
+
from .models import model_dir
|
| 36 |
+
from .pipeline import OrtOCR
|
| 37 |
+
|
| 38 |
+
MAX_SIDE = 1600 # longest side of the page image sent to the browser
|
| 39 |
+
TIMING_KEYS = (
|
| 40 |
+
"text_ms",
|
| 41 |
+
"det_ms",
|
| 42 |
+
"rec_ms",
|
| 43 |
+
"ocr_ms",
|
| 44 |
+
"math_ms",
|
| 45 |
+
"order_ms",
|
| 46 |
+
"total_ms",
|
| 47 |
+
)
|
| 48 |
+
MODES = ("auto", "ocr", "text")
|
| 49 |
+
|
| 50 |
+
|
| 51 |
+
def decode_image(data: bytes):
|
| 52 |
+
import cv2
|
| 53 |
+
|
| 54 |
+
img = cv2.imdecode(np.frombuffer(data, np.uint8), cv2.IMREAD_GRAYSCALE)
|
| 55 |
+
if img is None:
|
| 56 |
+
raise ValueError("not a PDF or a readable image")
|
| 57 |
+
yield 0, img, 0.0
|
| 58 |
+
|
| 59 |
+
|
| 60 |
+
class Engine:
|
| 61 |
+
"""Holds uploaded documents and reads one page per request, with the options
|
| 62 |
+
the page sends, so the browser shows each page as soon as it is done."""
|
| 63 |
+
|
| 64 |
+
def __init__(
|
| 65 |
+
self,
|
| 66 |
+
model_path: Path | None,
|
| 67 |
+
threads: int,
|
| 68 |
+
tesseract: bool,
|
| 69 |
+
tess_lang: str,
|
| 70 |
+
defaults: dict,
|
| 71 |
+
):
|
| 72 |
+
self.model_path, self.threads, self.defaults = model_path, threads, defaults
|
| 73 |
+
self.ocrs: dict[tuple, OrtOCR] = {} # (math, furniture) -> recognizer
|
| 74 |
+
self.tmp = tempfile.TemporaryDirectory(prefix="indra_ocr_viewer_")
|
| 75 |
+
self.lock = threading.Lock()
|
| 76 |
+
self.docs: dict[str, bytes] = {}
|
| 77 |
+
self.tesseract = None
|
| 78 |
+
if tesseract:
|
| 79 |
+
import pytesseract
|
| 80 |
+
|
| 81 |
+
os.environ["OMP_THREAD_LIMIT"] = str(threads)
|
| 82 |
+
self.tesseract = (pytesseract, tess_lang)
|
| 83 |
+
self._ocr(bool(defaults["math"]), bool(defaults["furniture"]))
|
| 84 |
+
|
| 85 |
+
def _ocr(self, math: bool, furniture: bool) -> OrtOCR:
|
| 86 |
+
"""The recognizer for these options; the math models download on first use."""
|
| 87 |
+
key = (math, furniture)
|
| 88 |
+
if key not in self.ocrs:
|
| 89 |
+
onnx = model_dir(self.model_path, math=math)
|
| 90 |
+
ocr = OrtOCR(
|
| 91 |
+
onnx,
|
| 92 |
+
threads=self.threads,
|
| 93 |
+
rec_int8=False,
|
| 94 |
+
math=math,
|
| 95 |
+
furniture=furniture,
|
| 96 |
+
)
|
| 97 |
+
ocr(
|
| 98 |
+
np.full((200, 200), 255, np.uint8)
|
| 99 |
+
) # warm-up: first-call allocation is not timed
|
| 100 |
+
self.ocrs[key] = ocr
|
| 101 |
+
return self.ocrs[key]
|
| 102 |
+
|
| 103 |
+
def upload(self, data: bytes) -> dict:
|
| 104 |
+
import uuid
|
| 105 |
+
|
| 106 |
+
if data[:5] == b"%PDF-":
|
| 107 |
+
import pypdfium2 as pdfium
|
| 108 |
+
|
| 109 |
+
doc = pdfium.PdfDocument(data)
|
| 110 |
+
n = len(doc)
|
| 111 |
+
doc.close()
|
| 112 |
+
else:
|
| 113 |
+
next(decode_image(data)) # raises on an unreadable image
|
| 114 |
+
n = 1
|
| 115 |
+
did = uuid.uuid4().hex[:12]
|
| 116 |
+
self.docs[did] = data
|
| 117 |
+
if data[:5] == b"%PDF-": # the converter opens the PDF from a file
|
| 118 |
+
with open(os.path.join(self.tmp.name, did + ".pdf"), "wb") as f:
|
| 119 |
+
f.write(data)
|
| 120 |
+
while len(self.docs) > 5: # keep the last few uploads only
|
| 121 |
+
old = next(iter(self.docs))
|
| 122 |
+
self.docs.pop(old)
|
| 123 |
+
try:
|
| 124 |
+
os.remove(os.path.join(self.tmp.name, old + ".pdf"))
|
| 125 |
+
except OSError:
|
| 126 |
+
pass
|
| 127 |
+
return {"id": did, "pages": n}
|
| 128 |
+
|
| 129 |
+
def page(self, did: str, n: int, opts: dict) -> dict:
|
| 130 |
+
import cv2
|
| 131 |
+
|
| 132 |
+
data = self.docs[did]
|
| 133 |
+
mode = opts.get("mode", "auto")
|
| 134 |
+
if mode not in MODES:
|
| 135 |
+
raise ValueError(f"mode must be one of {MODES}")
|
| 136 |
+
math, furniture = bool(opts.get("math")), bool(opts.get("furniture"))
|
| 137 |
+
dpi = float(opts.get("dpi") or 200.0)
|
| 138 |
+
with self.lock:
|
| 139 |
+
ocr = self._ocr(math, furniture)
|
| 140 |
+
conv = Converter(
|
| 141 |
+
ocr,
|
| 142 |
+
dpi=dpi,
|
| 143 |
+
mode=mode,
|
| 144 |
+
pdf_order=not opts.get("model_order"),
|
| 145 |
+
image_ocr=opts.get("image_ocr", True),
|
| 146 |
+
)
|
| 147 |
+
if data[:5] == b"%PDF-":
|
| 148 |
+
import pypdfium2 as pdfium
|
| 149 |
+
|
| 150 |
+
path = os.path.join(self.tmp.name, did + ".pdf")
|
| 151 |
+
doc = pdfium.PdfDocument(path)
|
| 152 |
+
try:
|
| 153 |
+
t0 = time.perf_counter()
|
| 154 |
+
# the converter's own scale, so the boxes match the picture
|
| 155 |
+
arr = (
|
| 156 |
+
doc[n]
|
| 157 |
+
.render(scale=conv.page_scale(doc[n]), grayscale=True)
|
| 158 |
+
.to_numpy()
|
| 159 |
+
)
|
| 160 |
+
render_ms = 1000 * (time.perf_counter() - t0)
|
| 161 |
+
page = conv.pdf_page(path, doc, n)
|
| 162 |
+
finally:
|
| 163 |
+
doc.close()
|
| 164 |
+
gray = np.ascontiguousarray(arr[..., 0] if arr.ndim == 3 else arr)
|
| 165 |
+
else:
|
| 166 |
+
_, gray, render_ms = next(decode_image(data))
|
| 167 |
+
page = ocr(gray)
|
| 168 |
+
if page.rotation: # the boxes are in the upright page: show it
|
| 169 |
+
gray = np.ascontiguousarray(np.rot90(gray, -(page.rotation // 90)))
|
| 170 |
+
h, w = gray.shape
|
| 171 |
+
s = min(1.0, MAX_SIDE / max(h, w))
|
| 172 |
+
small = (
|
| 173 |
+
cv2.resize(gray, None, fx=s, fy=s, interpolation=cv2.INTER_AREA)
|
| 174 |
+
if s < 1
|
| 175 |
+
else gray
|
| 176 |
+
)
|
| 177 |
+
ok, jpg = cv2.imencode(".jpg", small, [cv2.IMWRITE_JPEG_QUALITY, 82])
|
| 178 |
+
scaled = lambda b: [round(v * s, 1) for v in b[:4]] # noqa: E731
|
| 179 |
+
rec = {
|
| 180 |
+
"page": n + 1,
|
| 181 |
+
"w": small.shape[1],
|
| 182 |
+
"h": small.shape[0],
|
| 183 |
+
"img": "data:image/jpeg;base64,"
|
| 184 |
+
+ base64.b64encode(jpg.tobytes()).decode(),
|
| 185 |
+
"lines": [
|
| 186 |
+
{"box": scaled(ln.box), "text": ln.text, "conf": ln.conf}
|
| 187 |
+
for ln in page.lines
|
| 188 |
+
],
|
| 189 |
+
"blocks": [
|
| 190 |
+
{
|
| 191 |
+
"label": b["label"],
|
| 192 |
+
"box": scaled(b["bbox"]),
|
| 193 |
+
"text": b.get("text", ""),
|
| 194 |
+
}
|
| 195 |
+
for b in page.blocks
|
| 196 |
+
],
|
| 197 |
+
"markdown": page.markdown if page.markdown is not None else page.text,
|
| 198 |
+
"source": page.source,
|
| 199 |
+
"note": page.note,
|
| 200 |
+
"rotation": page.rotation,
|
| 201 |
+
"skew": page.skew,
|
| 202 |
+
"json": {
|
| 203 |
+
"page": n,
|
| 204 |
+
"source": page.source,
|
| 205 |
+
"rotation": page.rotation,
|
| 206 |
+
"skew": page.skew,
|
| 207 |
+
"note": page.note,
|
| 208 |
+
"lines": [asdict(ln) for ln in page.lines],
|
| 209 |
+
"markdown": page.markdown,
|
| 210 |
+
"blocks": page.blocks,
|
| 211 |
+
"timings": page.timings,
|
| 212 |
+
},
|
| 213 |
+
"timings": {
|
| 214 |
+
"render_ms": round(render_ms, 1),
|
| 215 |
+
**{k: 0.0 for k in TIMING_KEYS},
|
| 216 |
+
**{k: round(v, 1) for k, v in page.timings.items()},
|
| 217 |
+
},
|
| 218 |
+
}
|
| 219 |
+
if opts.get("tesseract") and self.tesseract:
|
| 220 |
+
pt, lang = self.tesseract
|
| 221 |
+
t0 = time.perf_counter()
|
| 222 |
+
try:
|
| 223 |
+
rec["tesseract"] = pt.image_to_string(
|
| 224 |
+
gray, lang=lang, config="--psm 3"
|
| 225 |
+
)
|
| 226 |
+
except Exception as e: # noqa: BLE001 (show the reason in the page)
|
| 227 |
+
rec["tesseract"] = f"[tesseract failed: {e}]"
|
| 228 |
+
rec["timings"]["tesseract_ms"] = round(
|
| 229 |
+
1000 * (time.perf_counter() - t0), 1
|
| 230 |
+
)
|
| 231 |
+
t = rec["timings"]
|
| 232 |
+
print(
|
| 233 |
+
f"page {n + 1}: {page.source}, {len(page.lines)} lines, {len(page.blocks)} blocks, "
|
| 234 |
+
f"{t['total_ms']:.0f} ms ({mode}{', math' if math else ''})",
|
| 235 |
+
flush=True,
|
| 236 |
+
)
|
| 237 |
+
return rec
|
| 238 |
+
|
| 239 |
+
|
| 240 |
+
def make_handler(engine: Engine):
|
| 241 |
+
class Handler(BaseHTTPRequestHandler):
|
| 242 |
+
def log_message(self, fmt, *args): # quiet
|
| 243 |
+
pass
|
| 244 |
+
|
| 245 |
+
def _send(self, code: int, body: bytes, ctype: str = "application/json"):
|
| 246 |
+
self.send_response(code)
|
| 247 |
+
self.send_header("Content-Type", ctype)
|
| 248 |
+
self.send_header("Content-Length", str(len(body)))
|
| 249 |
+
self.end_headers()
|
| 250 |
+
self.wfile.write(body)
|
| 251 |
+
|
| 252 |
+
def do_GET(self):
|
| 253 |
+
if self.path in ("/", "/index.html"):
|
| 254 |
+
self._send(
|
| 255 |
+
200,
|
| 256 |
+
PAGE.replace("__LOGO__", LOGO).encode(),
|
| 257 |
+
"text/html; charset=utf-8",
|
| 258 |
+
)
|
| 259 |
+
elif self.path == "/info":
|
| 260 |
+
info = {
|
| 261 |
+
"tesseract": engine.tesseract is not None,
|
| 262 |
+
"version": __version__,
|
| 263 |
+
"defaults": engine.defaults,
|
| 264 |
+
}
|
| 265 |
+
self._send(200, json.dumps(info).encode())
|
| 266 |
+
else:
|
| 267 |
+
self._send(404, b"not found", "text/plain")
|
| 268 |
+
|
| 269 |
+
def do_POST(self):
|
| 270 |
+
n = int(self.headers.get("Content-Length", "0"))
|
| 271 |
+
body = self.rfile.read(n)
|
| 272 |
+
try:
|
| 273 |
+
if self.path == "/upload":
|
| 274 |
+
res = engine.upload(body)
|
| 275 |
+
elif self.path == "/page":
|
| 276 |
+
q = json.loads(body)
|
| 277 |
+
res = engine.page(q["id"], int(q["n"]), q.get("opts", {}))
|
| 278 |
+
else:
|
| 279 |
+
return self._send(404, b"not found", "text/plain")
|
| 280 |
+
self._send(200, json.dumps(res).encode())
|
| 281 |
+
except Exception as e: # noqa: BLE001 (report the error to the page)
|
| 282 |
+
print(f"error: {e}", flush=True)
|
| 283 |
+
self._send(400, json.dumps({"error": str(e)}).encode())
|
| 284 |
+
|
| 285 |
+
return Handler
|
| 286 |
+
|
| 287 |
+
|
| 288 |
+
# The Datalab logo (assets/datalab-logo.png), for the page header.
|
| 289 |
+
LOGO = (
|
| 290 |
+
"iVBORw0KGgoAAAANSUhEUgAAAwAAAAMACAYAAACTgQCOAAAX4ElEQVR4Ae3dwVFcx9rH4dZXWkMAQALCexfsLSWA"
|
| 291 |
+
"CUAl7Y0JwFIAiABcUgCIACwH4LH2lhNABGAS0KfGl3uvL0gzwMw5p/v/PFUuubzyBtG/6bffefDpswIAAET4vwIA"
|
| 292 |
+
"AMQQAAAAEEQAAABAEAEAAABBBAAAAAQRAAAAEEQAAABAEAEAAABBBAAAAAQRAAAAEEQAAABAEAEAAABBBAAAAAQR"
|
| 293 |
+
"AAAAEEQAAABAEAEAAABBBAAAAAQRAAAAEEQAAABAEAEAAABBBAAAAAQRAAAAEEQAAABAEAEAAABBBAAAAAQRAAAA"
|
| 294 |
+
"EEQAAABAEAEAAABBBAAAAAQRAAAAEEQAAABAEAEAAABBBAAAAAQRAAAAEEQAAABAEAEAAABBBAAAAAQRAAAAEEQA"
|
| 295 |
+
"AABAEAEAAABBBAAAAAQRAAAAEEQAAABAEAEAAABBBAAAAAQRAAAAEEQAAABAEAEAAABBBAAAAAQRAAAAEEQAAABA"
|
| 296 |
+
"EAEAAABBBAAAAAQRAAAAEEQAAABAEAEAAABBBAAAAAQRAAAAEEQAAABAEAEAAABBBAAAAAQRAAAAEEQAAABAEAEA"
|
| 297 |
+
"AABBBAAAAAQRAAAAEEQAAABAEAEAAABBBAAAAAQRAAAAEEQAAABAEAEAAABBBAAAAAQRAAAAEEQAAABAEAEAAABB"
|
| 298 |
+
"BAAAAAQRAAAAEEQAAABAEAEAAABBBAAAAAQRAAAAEEQAAABAEAEAAABBBAAAAAQRAAAAEEQAAABAEAEAAABBBAAA"
|
| 299 |
+
"AAQRAAAAEEQAAABAEAEAAABBBAAAAAQRAAAAEEQAAABAEAEAAABBBAAAAAQRAAAAEEQAAABAEAEAAABBBAAAAAQR"
|
| 300 |
+
"AAAAEEQAAABAEAEAAABBBAAAAAQRAAAAEEQAAABAEAEAAABBBAAAAAQRAAAAEEQAAABAEAEAAABBBAAAAAQRAAAA"
|
| 301 |
+
"EEQAAABAEAEAAABBBAAAAAQRAAAAEEQAAABAEAEAAABBBAAAAAQRAAAAEEQAAABAEAEAAABBBAAAAAQRAAAAEEQA"
|
| 302 |
+
"AABAEAEAAABBBAAAAAQRAAAAEEQAAABAEAEAAABBBAAAAAQRAAAAEEQAAABAEAEAAABBBAAAAAQRAAAAEEQAAABA"
|
| 303 |
+
"EAEAAABBBAAAAAQRAAAAEEQAAABAEAEAAABBBAAAAAQRAAAAEEQAAABAEAEAAABBBAAAAAQRAAAAEEQAAABAEAEA"
|
| 304 |
+
"AABBHhYAuKOzs7Mym80K1+3u7pbNzc0CMDUCAIA7q4f/g4ODwnXHx8dlf3+/AEyNESAAAAgiAAAAIIgAAACAIAIA"
|
| 305 |
+
"AACCCAAAAAgiAAAAIIgAAACAIAIAAACCCAAAAAgiAAAAIIgAAACAIAIAAACCCAAAAAgiAAAAIIgAAACAIAIAAACC"
|
| 306 |
+
"CAAAAAgiAAAAIIgAAACAIAIAAACCCAAAAAgiAAAAIIgAAACAIAIAAACCCAAAAAgiAAAAIIgAAACAIAIAAACCCAAA"
|
| 307 |
+
"AAgiAAAAIIgAAACAIAIAAACCCAAAAAgiAAAAIIgAAACAIAIAAACCCAAAAAgiAAAAIIgAAACAIAIAAACCCAAAAAgi"
|
| 308 |
+
"AAAAIIgAAACAIAIAAACCCAAAAAgiAAAAIIgAAACAIAIAAACCCAAAAAgiAAAAIIgAAACAIAIAAACCCAAAAAgiAAAA"
|
| 309 |
+
"IIgAAACAIAIAAACCCAAAAAgiAAAAIIgAAACAIAIAAACCCAAAAAgiAAAAIIgAAACAIAIAAACCCAAAAAgiAAAAIIgA"
|
| 310 |
+
"AACAIAIAAACCCAAAAAjysAAw197eXuG6ra2tcnp6Wrhue3u7AEyRAABYwGw2K1xXA2BnZ6cA0A4jQAAAEEQAAABA"
|
| 311 |
+
"EAEAAABBBAAAAAQRAAAAEEQAAABAEAEAAABBBAAAAAQRAAAAEEQAAABAEAEAAABBBAAAAAQRAAAAEEQAAABAEAEA"
|
| 312 |
+
"AABBBAAAAAQRAAAAEEQAAABAEAEAAABBBAAAAAQRAAAAEEQAAABAEAEAAABBBAAAAAQRAAAAEEQAAABAEAEAAABB"
|
| 313 |
+
"BAAAAAQRAAAAEEQAAABAEAEAAABBBAAAAAQRAAAAEEQAAABAEAEAAABBBAAAAAQRAAAAEEQAAABAEAEAAABBBAAA"
|
| 314 |
+
"AAQRAAAAEEQAAABAEAEAAABBBAAAAAQRAAAAEEQAAABAEAEAAABBBAAAAAQRAAAAEEQAAABAEAEAAABBBAAAAAQR"
|
| 315 |
+
"AAAAEEQAAABAEAEAAABBBAAAAAQRAAAAEEQAAABAEAEAAABBBAAAAAQRAAAAEEQAAABAEAEAAABBBAAAAAQRAAAA"
|
| 316 |
+
"EEQAAABAEAEAAABBBAAAAAR5WADgjv76669ydnZWerK+vl7W1tYKQK8efPqsAPBVGxsbhQz7+/vl+Pi4APTKCBAA"
|
| 317 |
+
"AAQRAAAAEEQAAABAEAEAAABBBAAAAAQRAAAAEEQAAABAEAEAAABBBAAAAAQRAAAAEEQAAABAEAEAAABBBAAAAAQR"
|
| 318 |
+
"AAAAEEQAAABAEAEAAABBBAAAAAQRAAAAEEQAAABAEAEAAABBBAAAAAQRAAAAEEQAAABAEAEAAABBBAAAAAQRAAAA"
|
| 319 |
+
"EEQAAABAEAEAAABBBAAAAAQRAAAAEEQAAABAEAEAAABBBAAAAAQRAAAAEEQAAABAEAEAAABBBAAAAAQRAAAAEEQA"
|
| 320 |
+
"AABAEAEAAABBBAAAAAQRAAAAEEQAAABAEAEAAABBBAAAAAQRAAAAEEQAAABAEAEAAABBBAAAAAQRAAAAEEQAAABA"
|
| 321 |
+
"EAEAAABBBAAAAAQRAAAAEEQAAABAEAEAAABBBAAAAAQRAAAAEEQAAABAEAEAAABBBAAAAAQRAAAAEEQAAABAEAEA"
|
| 322 |
+
"AABBBAAAAAQRAAAAEEQAAABAEAEAAABBBAAAAAQRAABznJ2dFQDohQAA+Ip6+P/+++8LAPTiYQHgRh8+fCh7e3vl"
|
| 323 |
+
"4uKiAEAv3AAA3ODk5MThH4AuuQEA+B9HR0fl1atXBQB65AYA4L84/APQOzcAAP9ycHBwOfoDAD0TAEC8Ouf/9OnT"
|
| 324 |
+
"MpvNCgD0TgAA0a7WfNr1D0AKbwCAWA7/ACRyAwBEsuMfgFRuAIA4dvwDkMwNABDFmk8A0rkBAGI4/AOAGwAghB3/"
|
| 325 |
+
"APA3AQB0zY5/APgnAQB0y5pPALjOGwCgSw7/AHAzNwBAd+z4B4AvcwMAdMWOfwD4OjcAQDes+QSA+dwAAF1w+AeA"
|
| 326 |
+
"xbgBAJpnxz8ALE4AAM2y4x8Abk8AAE2y5hMA7sYbAKA5Dv8AcHduAICm2PEPAPfjBgBohh3/AHB/bgCAJljzCQDL"
|
| 327 |
+
"4QYAmDyHfwBYHjcAwKTZ8Q8AyyUAgEmy4x8AVkMAAJNjzScArI43AMCkOPwDwGq5AQAmw45/AFg9NwDAJNjxDwDD"
|
| 328 |
+
"cAMAjM6aT6Zie3u7HB4eFoCeuQEARuXwz1Q8f/68nJ6els3NzQLQMzcAwGjs+GcqXr58WZ49e1YAEggAYHB2/DMV"
|
| 329 |
+
"W1tb5fj4uOzs7BSAFAIAGJQ1n0xFnfd/8+aNkR8gjjcAwGAc/pkK8/5AMjcAwCDs+GcqzPsD6dwAACtnxz9TUOf9"
|
| 330 |
+
"66f+Dv9AOjcAwEpZ88kUmPcH+A83AMDKOPwzBeb9Af7JDQCwEnb8MwXm/QGuEwDAUtnxzxTY7w/wZQIAWBprPpkC"
|
| 331 |
+
"8/4AX+cNALAUDv9MgXl/gPncAAD3Zsc/U2DeH2AxbgCAe7Hjn7HZ7w9wO24AgDuz5pOxmfcHuD03AMCdOPwzNvP+"
|
| 332 |
+
"AHfjBgC4NTv+GZt5f4C7EwDAwuz4Z2z2+wPcnwAAFmLNJ2Mz7w+wHN4AAHM5/DM28/4Ay+MGAPgqO/4Zm3l/gOVy"
|
| 333 |
+
"AwB8kR3/jMl+f4DVcAMA3MiaT8Zk3h9gddwAANc4/DMm8/4Aq+UGAPgHO/4Zk3l/gNUTAMAlO/4Zk/3+AMMRAIA1"
|
| 334 |
+
"n4zKvD/AsLwBgHAO/4zJvD/A8NwAQDA7/hmTeX+AcbgBgFB2/DMW+/0BxuUGAAJZ88lYzPsDjM8NAIRx+Gcs5v0B"
|
| 335 |
+
"psENAASx45+xmPcHmA4BAAHs+Gcs9vsDTI8AgM5Z88lYzPsDTJM3ANAxh3/GYt4fYLrcAECn7PhnLOb9AabNDQB0"
|
| 336 |
+
"yI5/xmC/P0Ab3ABAZ6z5ZAzm/QHa4QYAOuLwzxjM+wO0xQ0AdMKOf8Zg3h+gPQIAGmfHP2Ow3x+gXQIAGmbNJ2Mw"
|
| 337 |
+
"7w/QNm8AoFEO/4zBvD9A+9wAQIPs+GcM5v0B+uAGABpjxz9Ds98foC9uAKAh1nwyNPP+AP1xAwCNcPhnaOb9Afrk"
|
| 338 |
+
"BgAaYMc/QzPvD9AvAQATZsc/Q7PfH6B/AgAmyppPhmbeHyCDNwAwQQ7/DM28P0AONwAwMXb8MzTz/gBZ3ADAhNjx"
|
| 339 |
+
"z5Ds9wfI9ODTZwUYnTWfDO39+/dGfgACuQGACXD4ZwwO/wCZvAGAkdnxDwAMSQDASOz4BwDGIABgBNZ8AgBj8QYA"
|
| 340 |
+
"BubwDwCMyQ0ADMiOfwBgbG4AYCB2/AMAU+AGAAZgzScAMBVuAGDFHP4BgClxAwArZMc/ADA1AgBWwI5/AGCqBAAs"
|
| 341 |
+
"mTWfAMCUeQMAS+TwDwBMnRsAWBI7/gGAFrgBgCWw4x8AaIUbALgnaz4BgJa4AYB7cPgHAFrjBgDuyI5/AKBFAgBu"
|
| 342 |
+
"yY5/AKBlAgBuwZpPAKB13gDAghz+AYAeuAGABdjxDwD0wg0AzGHHPwDQEzcA8BXWfAIAvXEDAF/g8A8A9MgNANzA"
|
| 343 |
+
"jn8AoFcCAP6LHf8AQO8EAPyLNZ8AQAJvAKA4/AMAOdwAEM+OfwAgiRsAotnxDwCkcQNALGs+AYBEbgCI5PAPxc0X"
|
| 344 |
+
"QCg3AMSx4x/+VgNgbW2tAJDFDQAx6mGnzvs7/MPffvvttwJAHjcARLDmE66rG7AAyOMGgO45/MPN3r17VwDIIwDo"
|
| 345 |
+
"Wv2E8/Hjxw7/cIP6czGbzQoAWQQA3bLjH+bzJgYgz4NPnxXojDWfsLj379+Xzc3NAkAGNwB0x+EfbueHH34oAOQQ"
|
| 346 |
+
"AHSl7vh3+Ifbqe8Afv755wJABiNAdKHO+T99+tSDRrij+oVgp6enZXt7uwDQNwFA86z5hOXY2tq6jADvAQD6ZgSI"
|
| 347 |
+
"pjn8w/LUn6O6Oevjx48FgH4JAJplxz8sX/15+u6777wJAOiYAKBJdvzD6tSfqxcvXly+q3EbANAfbwBojjWfMKz9"
|
| 348 |
+
"/f3LUbvd3d0CQPsEAE1x+Ifx1EfCjx49Kjs7O5fbgurmoPX19QL8/fMBrRAANGVjY6MAwNScn58XaIU3AAAAEEQA"
|
| 349 |
+
"AABAEAEAAABBBAAAAAQRAAAAEEQAAABAEAEAAABBBAAAAAQRAAAAEEQAAABAEAEAAABBBAAAAAQRAAAAEEQAAABA"
|
| 350 |
+
"EAEAAABBBAAAAAQRAAAAEEQAAABAEAEAAABBBAAAAAQRAAAAEEQAAABAEAEAAABBBAAAAAQRAAAAEEQAAABAEAEA"
|
| 351 |
+
"AABBBAAAAAQRAAAAEEQAAABAEAEAAABBBAAAAAQRAAAAEEQAAABAEAEAAABBBAAAAAQRAAAAEEQAAABAEAEAAABB"
|
| 352 |
+
"BAAAAAQRAAAAEEQAAABAkIcFABqwtrZWtre3yzfffFMePXpU1tfXL/9bdXZ2dvnPhw8fymw2KxcXFwWAmz349FmB"
|
| 353 |
+
"RmxsbBQgRz3g7+zslOfPn1/+uagaAScnJ5f/wBDOz88LtEIA0BQBADnqof/HH3/896f8d1FvBd6+fVuOjo4KrJIA"
|
| 354 |
+
"oCUCgKYIAOhfHfM5Pj6+/HNZaggcHBxc3gzAKggAWuIRMACTUT/1//XXX5d6+K+2trbK6elpOTw8LADpPAIGYBJe"
|
| 355 |
+
"vnxZnj17VlbpaqTop59+KgCpBAAAo6qfzr9+/Xrpn/p/yVVkiAAglREgAEZTD/11NGeow/+VGgF13AggkQAAYBRX"
|
| 356 |
+
"8/6bm5tlDC9evLjValGAXggAAAZX5/3rAXxsddvQfdaMArRIAAAwmDrvXz/1X/Vj30XV/583b94UgCQCAIBBjDXv"
|
| 357 |
+
"P8/VNw0DpBAAAKzc2PP+83gPACQRAACs1FTm/efxHgBIIQAAWImpzfvP4z0AkEIAALB0U533n8d7ACCBAABgqaY+"
|
| 358 |
+
"7z+P9wBA7wQAAEvTyrz/PN4DAD0TAADcW2vz/vN4DwD0TAAAcC+tzvvP4z0A0CsBAMCdtT7vP4/3AECPBAAAd9LL"
|
| 359 |
+
"vP883gMAvREAANxKb/P+83gPAPRGAACwsF7n/efxHgDoiQAAYCG9z/vP4z0A0AsBAMBcKfP+83gPAPRAAADwRWnz"
|
| 360 |
+
"/vN4DwD0QAAAcKPUef95vAcAWicAALgmfd5/Hu8BgJYJAAD+wbz/YrwHAFolAAC4ZN7/drwHAFolAAAw739H3gMA"
|
| 361 |
+
"LRIANKV+4gYsl3n/+/EeAGiNAAAIZt5/ObwHAFoiAGiK8QRYDvP+y+U9ANASAUBTjCjA/Zn3Xw3vAYBWCACa4sAC"
|
| 362 |
+
"5V7M+6+W9wBACwQATREAcHfm/YfhPQAwdQKAptQA8IsVbse8/7C8BwCmTgDQnN3d3QIsxrz/OLwHAKZMANCcJ0+e"
|
| 363 |
+
"FGA+8/7j8h4AmKoHnz4r0JCLi4vy7bffXv4J3KzO+xv5Gd/Z2Vl5/Pixv68CnJ+fF2iFGwCaU98AGAOCm5n3nxbv"
|
| 364 |
+
"AYApEgA0yeEGrjPvP03eAwBTIwBoUv2FarYW/sO8/7R5DwBMiQCgWYeHhwWw378Vvh8AmAoBQLPcApDOvH9bvAcA"
|
| 365 |
+
"pkIA0DSfqJHKvH+bvAcApkAA0LT6iZpPP0lj3r9t3gMAY/M9AHRhb2+vzGazAr2z378Pvh+gP74HgJa4AaALdRSo"
|
| 366 |
+
"3gZAr8z798V7AGBMAoAu1F+mr1+/LtAj8/598h4AGIsAoBv1cFRvAqAn5v375j0AMAZvAOjOyclJOTg4KNA68/4Z"
|
| 367 |
+
"vAfogzcAtMQNAN3Z3993E0DTzPtn8R4AGJoAoEs1AuoBysNgWmPeP5P3AMCQBADdqgeot2/fmq+lGeb9s3kPAAzF"
|
| 368 |
+
"GwAivHr1qhwdHRWYKvP+VN4DtMsbAFoiAIhRf7HWEKiPhGEqrlbYGvnhSv1Sw/rlhrRFANASI0DEqAet+jj4999/"
|
| 369 |
+
"L0+ePCkwNvP+3MR7AGDV3AAQ6+pGoH7aVv8dhlQPeHXmG76k3gLUv59ogxsAWiIA4LN3796VX375RQwwCPP+LMJ7"
|
| 370 |
+
"gLYIAFoiAOB/1F+6f/75Z/njjz/Kx48f/x0E9d/hPtbW1i7H0Iz8sKj6oYQvNmxDHS+FVggAAAAI4hEwAAAEEQAA"
|
| 371 |
+
"ABBEAAAAQBABAAAAQQQAAAAEEQAAABBEAAAAQBABAAAAQQQAAAAEEQAAABBEAAAAQBABAAAAQQQAAAAEEQAAABBE"
|
| 372 |
+
"AAAAQBABAAAAQQQAAAAEEQAAABBEAAAAQBABAAAAQQQAAAAEEQAAABBEAAAAQBABAAAAQQQAAAAEEQAAABBEAAAA"
|
| 373 |
+
"QBABAAAAQQQAAAAEEQAAABBEAAAAQBABAAAAQQQAAAAEEQAAABBEAAAAQBABAAAAQQQAAAAEEQAAABBEAAAAQBAB"
|
| 374 |
+
"AAAAQQQAAAAEEQAAABBEAAAAQBABAAAAQQQAAAAEEQAAABBEAAAAQBABAAAAQQQAAAAEEQAAABBEAAAAQBABAAAA"
|
| 375 |
+
"QQQAAAAEEQAAABBEAAAAQBABAAAAQQQAAAAEEQAAABBEAAAAQBABAAAAQQQAAAAEEQAAABBEAAAAQBABAAAAQQQA"
|
| 376 |
+
"AAAEEQAAABBEAAAAQBABAAAAQQQAAAAEEQAAABBEAAAAQBABAAAAQQQAAAAEEQAAABBEAAAAQBABAAAAQQQAAAAE"
|
| 377 |
+
"EQAAABBEAAAAQBABAAAAQQQAAAAEEQAAABBEAAAAQBABAAAAQQQAAAAEEQAAABBEAAAAQBABAAAAQQQAAAAEEQAA"
|
| 378 |
+
"ABBEAAAAQBABAAAAQQQAAAAEEQAAABBEAAAAQBABAAAAQQQAAAAEEQAAABBEAAAAQBABAAAAQQQAAAAEEQAAABBE"
|
| 379 |
+
"AAAAQBABAAAAQQQAAAAEEQAAABBEAAAAQBABAAAAQQQAAAAEEQAAABBEAAAAQBABAAAAQQQAAAAEEQAAABBEAAAA"
|
| 380 |
+
"QBABAAAAQQQAAAAEEQAAABBEAAAAQBABAAAAQQQAAAAEEQAAABBEAAAAQBABAAAAQQQAAAAEEQAAABBEAAAAQBAB"
|
| 381 |
+
"AAAAQQQAAAAEEQAAABBEAAAAQBABAAAAQQQAAAAEEQAAABBEAAAAQBABAAAAQQQAAAAEEQAAABBEAAAAQBABAAAA"
|
| 382 |
+
"QQQAAAAEEQAAABBEAAAAQBABAAAAQQQAAAAEEQAAABBEAAAAQBABAAAAQQQAAAAEEQAAABBEAAAAQBABAAAAQQQA"
|
| 383 |
+
"AAAEEQAAABDk/wH98CgW8PO2cwAAAABJRU5ErkJggg=="
|
| 384 |
+
)
|
| 385 |
+
|
| 386 |
+
PAGE = r"""<!doctype html>
|
| 387 |
+
<html lang="en"><head><meta charset="utf-8"><meta name="viewport" content="width=device-width,initial-scale=1">
|
| 388 |
+
<title>Indra OCR viewer · Datalab</title>
|
| 389 |
+
<link rel="preconnect" href="https://fonts.googleapis.com"><link rel="preconnect" href="https://fonts.gstatic.com" crossorigin>
|
| 390 |
+
<link href="https://fonts.googleapis.com/css2?family=Geist:wght@400;500;600&family=Geist+Mono:wght@400;500&display=swap" rel="stylesheet">
|
| 391 |
+
<style>
|
| 392 |
+
:root{
|
| 393 |
+
--font-body:"Geist",ui-sans-serif,system-ui,sans-serif;--font-mono:"Geist Mono",ui-monospace,"SF Mono",Menlo,monospace;
|
| 394 |
+
--page:oklch(98.5% .004 263);--surface:oklch(96.5% .006 263);--surface-2:oklch(94% .007 263);
|
| 395 |
+
--rule:oklch(86% .008 263);--rule-strong:oklch(78% .01 263);
|
| 396 |
+
--ink:oklch(18% .012 263);--ink-2:oklch(36% .012 263);--ink-3:oklch(48% .014 263);--ink-4:oklch(70% .014 263);
|
| 397 |
+
--accent:oklch(38% .24 263);--accent-soft:oklch(96% .04 263);--accent-strong:oklch(30% .26 263);--accent-ink:oklch(99% 0 0);
|
| 398 |
+
--ok:oklch(58% .15 145);--warn:oklch(72% .16 75);--err:oklch(56% .2 25);--focus:oklch(38% .24 263 / .35);
|
| 399 |
+
--hairline:1px;--ease:cubic-bezier(.25,1,.5,1)}
|
| 400 |
+
@media (min-resolution:2dppx){:root{--hairline:.5px}}
|
| 401 |
+
*{box-sizing:border-box}
|
| 402 |
+
body{margin:0;background:var(--page);color:var(--ink);font:14px/1.5 var(--font-body);-webkit-font-smoothing:antialiased}
|
| 403 |
+
a{color:var(--accent);text-decoration:none}a:hover{text-decoration:underline}
|
| 404 |
+
.mono{font-family:var(--font-mono)}
|
| 405 |
+
.label{font:500 10.5px/1 var(--font-mono);letter-spacing:.08em;text-transform:uppercase;color:var(--ink-3)}
|
| 406 |
+
/* top bar */
|
| 407 |
+
.top{position:sticky;top:0;z-index:5;display:flex;align-items:center;gap:18px;height:56px;padding:0 24px;background:color-mix(in oklch,var(--page) 88%,transparent);backdrop-filter:blur(8px);border-bottom:var(--hairline) solid var(--rule)}
|
| 408 |
+
.brand{display:flex;align-items:center;gap:10px;font-weight:600;letter-spacing:-.01em}
|
| 409 |
+
.mark{width:30px;height:30px;margin:0 -4px}
|
| 410 |
+
.brand .sep{color:var(--ink-4);font-weight:400}.brand .prod{font-weight:500}
|
| 411 |
+
.ver{font:11px var(--font-mono);color:var(--ink-3);border:var(--hairline) solid var(--rule);border-radius:999px;padding:2px 8px}
|
| 412 |
+
.top .sp{flex:1}.top nav{display:flex;gap:18px;font-size:13px}.top nav a{color:var(--ink-2)}
|
| 413 |
+
.cta{background:var(--accent);color:var(--accent-ink)!important;padding:7px 12px;border-radius:6px;font-weight:500}.cta:hover{background:var(--accent-strong);text-decoration:none}
|
| 414 |
+
/* layout */
|
| 415 |
+
.wrap{display:grid;grid-template-columns:300px minmax(0,1fr);min-height:calc(100vh - 56px)}
|
| 416 |
+
aside{border-right:var(--hairline) solid var(--rule);padding:20px;display:flex;flex-direction:column;gap:22px;position:sticky;top:56px;height:calc(100vh - 56px);overflow:auto;background:var(--page)}
|
| 417 |
+
main{padding:24px;display:flex;flex-direction:column;gap:24px;min-width:0}
|
| 418 |
+
@media(max-width:900px){.wrap{grid-template-columns:1fr}aside{position:static;height:auto;border-right:0;border-bottom:var(--hairline) solid var(--rule)}}
|
| 419 |
+
/* drop zone */
|
| 420 |
+
#drop{border:1px dashed var(--rule-strong);border-radius:10px;padding:18px 14px;text-align:center;cursor:pointer;background:var(--surface);transition:border-color var(--ease) .12s,background .12s}
|
| 421 |
+
#drop:hover,#drop.over{border-color:var(--accent);background:var(--accent-soft)}
|
| 422 |
+
#drop b{display:block;font-weight:500;margin-bottom:2px}#drop span{color:var(--ink-3);font-size:12.5px}
|
| 423 |
+
#fname{margin-top:8px;font:12px var(--font-mono);color:var(--ink-2);word-break:break-all}
|
| 424 |
+
/* controls */
|
| 425 |
+
.group{display:flex;flex-direction:column;gap:10px}
|
| 426 |
+
.seg{display:grid;grid-template-columns:repeat(3,1fr);border:var(--hairline) solid var(--rule-strong);border-radius:7px;overflow:hidden}
|
| 427 |
+
.seg button{border:0;background:transparent;padding:7px 4px;font:500 12.5px var(--font-body);color:var(--ink-2);cursor:pointer}
|
| 428 |
+
.seg button+button{border-left:var(--hairline) solid var(--rule)}.seg button.on{background:var(--ink);color:var(--page)}
|
| 429 |
+
.hint{font-size:12px;color:var(--ink-3);line-height:1.4}
|
| 430 |
+
.sw{display:flex;align-items:center;justify-content:space-between;gap:10px;font-size:13px;color:var(--ink-2);cursor:pointer;user-select:none}
|
| 431 |
+
.sw input{appearance:none;width:30px;height:18px;border-radius:999px;background:var(--rule-strong);position:relative;cursor:pointer;transition:background .15s;flex:none}
|
| 432 |
+
.sw input::after{content:"";position:absolute;top:2px;left:2px;width:14px;height:14px;border-radius:50%;background:#fff;transition:transform .15s var(--ease)}
|
| 433 |
+
.sw input:checked{background:var(--accent)}.sw input:checked::after{transform:translateX(12px)}
|
| 434 |
+
.sw input:focus-visible{outline:3px solid var(--focus)}
|
| 435 |
+
.row2{display:grid;grid-template-columns:1fr 1fr;gap:10px}
|
| 436 |
+
.field{display:flex;flex-direction:column;gap:5px}.field select,.field input{font:13px var(--font-body);padding:6px 8px;border:var(--hairline) solid var(--rule-strong);border-radius:6px;background:#fff;color:var(--ink)}
|
| 437 |
+
.btn{border:0;background:var(--accent);color:var(--accent-ink);font:500 13px var(--font-body);padding:9px 12px;border-radius:7px;cursor:pointer}
|
| 438 |
+
.btn:hover{background:var(--accent-strong)}.btn:disabled{background:var(--rule-strong);cursor:default}
|
| 439 |
+
.dot{display:inline-block;width:9px;height:9px;border-radius:2px;margin-right:6px;vertical-align:0}
|
| 440 |
+
#stats{font:12px/1.7 var(--font-mono);color:var(--ink-2)}#stats b{color:var(--ink);font-weight:500}
|
| 441 |
+
#status{font:12px var(--font-mono);color:var(--ink-3);min-height:18px}
|
| 442 |
+
.callout{border:var(--hairline) solid var(--rule);border-radius:10px;padding:12px;background:var(--surface);font-size:12.5px;color:var(--ink-2)}
|
| 443 |
+
/* empty state */
|
| 444 |
+
.empty{border:1px dashed var(--rule-strong);border-radius:14px;padding:64px 24px;text-align:center;color:var(--ink-3)}
|
| 445 |
+
.empty h2{font-weight:500;color:var(--ink);letter-spacing:-.015em;margin:0 0 6px;font-size:22px}
|
| 446 |
+
/* page card */
|
| 447 |
+
.card{background:#fff;border:var(--hairline) solid var(--rule);border-radius:12px;overflow:hidden}
|
| 448 |
+
.ch{display:flex;align-items:center;flex-wrap:wrap;gap:10px 16px;padding:12px 16px;border-bottom:var(--hairline) solid var(--rule)}
|
| 449 |
+
.ch h3{margin:0;font-size:15px;font-weight:600;letter-spacing:-.01em}
|
| 450 |
+
.chip{font:500 10.5px/1 var(--font-mono);letter-spacing:.06em;text-transform:uppercase;padding:5px 7px;border-radius:5px;background:var(--surface-2);color:var(--ink-2)}
|
| 451 |
+
.chip.text{background:oklch(95% .04 145);color:oklch(38% .12 145)}.chip.ocr{background:var(--accent-soft);color:var(--accent)}.chip.hybrid{background:oklch(95% .05 75);color:oklch(42% .12 75)}.chip.error{background:oklch(95% .05 25);color:var(--err)}
|
| 452 |
+
.meta{font:12px var(--font-mono);color:var(--ink-3)}.ch .sp{flex:1}
|
| 453 |
+
.tbar{display:flex;height:6px;border-radius:3px;overflow:hidden;background:var(--surface-2);width:180px}.tbar i{display:block;height:100%}
|
| 454 |
+
.body{display:grid;grid-template-columns:minmax(0,1.05fr) minmax(0,1fr);min-height:420px}
|
| 455 |
+
@media(max-width:1200px){.body{grid-template-columns:1fr}}
|
| 456 |
+
.pageview{padding:16px;background:var(--surface);border-right:var(--hairline) solid var(--rule)}
|
| 457 |
+
svg.pg{width:100%;height:auto;display:block;background:#fff;box-shadow:0 1px 2px oklch(18% .012 263 / .08),0 0 0 var(--hairline) var(--rule)}
|
| 458 |
+
.right{display:flex;flex-direction:column;min-width:0}
|
| 459 |
+
.tabs{display:flex;gap:2px;padding:10px 12px 0;border-bottom:var(--hairline) solid var(--rule)}
|
| 460 |
+
.tabs button{border:0;background:transparent;padding:8px 10px;font:500 12.5px var(--font-body);color:var(--ink-3);cursor:pointer;border-bottom:2px solid transparent;margin-bottom:-1px}
|
| 461 |
+
.tabs button.on{color:var(--ink);border-bottom-color:var(--accent)}
|
| 462 |
+
.pane{flex:1;overflow:auto;max-height:78vh;padding:14px 16px}
|
| 463 |
+
.pane.src{white-space:pre-wrap;font:12.5px/1.6 var(--font-mono);color:var(--ink-2)}
|
| 464 |
+
.preview{font-size:14px;line-height:1.6}.preview h1,.preview h2,.preview h3{letter-spacing:-.015em;line-height:1.25;margin:1.1em 0 .4em}.preview h1{font-size:20px}.preview h2{font-size:17px}.preview h3{font-size:15px}
|
| 465 |
+
.preview p{margin:.5em 0}.preview ul,.preview ol{padding-left:1.4em}.preview table{border-collapse:collapse;margin:10px 0;font-size:12.5px}
|
| 466 |
+
.preview td,.preview th{border:var(--hairline) solid var(--rule-strong);padding:4px 7px;vertical-align:top}.preview th{background:var(--surface)}
|
| 467 |
+
.preview code.m{font:12.5px var(--font-mono);background:var(--accent-soft);color:var(--accent-strong);padding:1px 4px;border-radius:4px}
|
| 468 |
+
.ln-row{display:flex;gap:10px;padding:3px 6px;border-radius:5px;font-size:13px;cursor:default}.ln-row:hover,.ln-row.on{background:var(--accent-soft)}
|
| 469 |
+
.ln-row .k{font:11px var(--font-mono);color:var(--ink-4);min-width:28px;text-align:right;padding-top:2px}.ln-row .t{flex:1;min-width:0;word-break:break-word}
|
| 470 |
+
.ln-row .c{font:11px var(--font-mono);color:var(--ink-3);padding-top:2px}.ln-row .c.low{color:var(--err)}
|
| 471 |
+
/* overlays */
|
| 472 |
+
rect.ln{fill:transparent;stroke:oklch(55% .15 263);stroke-width:1;vector-effect:non-scaling-stroke;opacity:.55}
|
| 473 |
+
rect.ln.on{fill:oklch(55% .2 263 / .18);stroke-width:2.5;opacity:1}
|
| 474 |
+
rect.bk{fill:transparent;stroke-width:2;vector-effect:non-scaling-stroke}rect.bk.on{stroke-width:3.5}
|
| 475 |
+
text.bl{font-family:var(--font-mono);font-weight:600;paint-order:stroke;stroke:#fff}
|
| 476 |
+
path.ord{fill:none;stroke:oklch(60% .2 30);stroke-width:1.4;vector-effect:non-scaling-stroke;opacity:.8}
|
| 477 |
+
circle.on-n{fill:oklch(60% .2 30)}text.n{font-family:var(--font-mono);font-weight:600;fill:#fff;text-anchor:middle;dominant-baseline:central}
|
| 478 |
+
.hide-lines rect.ln,.hide-blocks rect.bk,.hide-blocks text.bl,.hide-labels text.bl,.hide-order .ord,.hide-order circle.on-n,.hide-order text.n{display:none}
|
| 479 |
+
footer{padding:18px 24px;border-top:var(--hairline) solid var(--rule);display:flex;justify-content:space-between;flex-wrap:wrap;gap:10px;font-size:12px;color:var(--ink-3)}
|
| 480 |
+
</style></head><body>
|
| 481 |
+
<div class="top">
|
| 482 |
+
<div class="brand"><img class="mark" src="data:image/png;base64,__LOGO__" alt="Datalab">Datalab<span class="sep">/</span><span class="prod">Indra OCR</span></div><span class="ver" id="ver"></span>
|
| 483 |
+
<div class="sp"></div>
|
| 484 |
+
<nav><a href="https://github.com/datalab-to/indra" target="_blank" rel="noopener">GitHub</a><a href="https://huggingface.co/datalab-to/indra-ocr" target="_blank" rel="noopener">Models</a><a class="cta" href="https://www.datalab.to/?utm_source=indra-viewer" target="_blank" rel="noopener">Try Chandra on Datalab</a></nav>
|
| 485 |
+
</div>
|
| 486 |
+
<div class="wrap">
|
| 487 |
+
<aside>
|
| 488 |
+
<div><div id="drop"><b>Drop a PDF or an image</b><span>or click to choose a file</span><input id="file" type="file" accept=".pdf,image/*" hidden></div><div id="fname"></div></div>
|
| 489 |
+
<div class="group"><div class="label">Source</div>
|
| 490 |
+
<div class="seg" id="mode"><button data-v="auto">Auto</button><button data-v="ocr">OCR</button><button data-v="text">Text layer</button></div>
|
| 491 |
+
<div class="hint" id="modehint"></div></div>
|
| 492 |
+
<div class="group"><div class="label">Pipeline</div>
|
| 493 |
+
<label class="sw">Math as LaTeX<input type="checkbox" id="math"></label>
|
| 494 |
+
<label class="sw">Keep headers and footers<input type="checkbox" id="furniture"></label>
|
| 495 |
+
<label class="sw">PDF reading order<input type="checkbox" id="pdforder" checked></label>
|
| 496 |
+
<label class="sw">OCR text inside images<input type="checkbox" id="imageocr" checked></label>
|
| 497 |
+
<label class="sw" id="tesslab" hidden>Compare with Tesseract<input type="checkbox" id="tess"></label>
|
| 498 |
+
<div class="row2"><div class="field"><span class="label">DPI</span><select id="dpi"><option>150</option><option selected>200</option><option>300</option></select></div>
|
| 499 |
+
<div class="field"><span class="label">Max pages</span><input id="maxp" type="number" min="1" value="20"></div></div>
|
| 500 |
+
<button class="btn" id="rerun" disabled>Read again with these options</button></div>
|
| 501 |
+
<div class="group"><div class="label">Show</div>
|
| 502 |
+
<label class="sw">Lines<input type="checkbox" class="disp" data-c="lines" checked></label>
|
| 503 |
+
<label class="sw">Layout blocks<input type="checkbox" class="disp" data-c="blocks" checked></label>
|
| 504 |
+
<label class="sw">Block labels<input type="checkbox" class="disp" data-c="labels" checked></label>
|
| 505 |
+
<label class="sw">Reading order<input type="checkbox" class="disp" data-c="order"></label>
|
| 506 |
+
<div id="legend" class="hint"></div></div>
|
| 507 |
+
<div class="group"><div class="label">Run</div><div id="status"></div><div id="stats"></div></div>
|
| 508 |
+
<div class="callout">Need handwriting, math and tables at the highest accuracy? <a href="https://www.datalab.to/?utm_source=indra-viewer" target="_blank" rel="noopener">Chandra on Datalab</a> reads them, with $5 in free credits to start.</div>
|
| 509 |
+
</aside>
|
| 510 |
+
<main id="out"><div class="empty"><h2>Read a document</h2>Drop a PDF or an image on the left. Pages appear as they finish, with lines, layout blocks and reading order drawn over each page.</div></main>
|
| 511 |
+
</div>
|
| 512 |
+
<footer><span>Indra OCR · fast CPU OCR by <a href="https://www.datalab.to" target="_blank" rel="noopener">Datalab</a></span><span class="mono">Runs locally: your files stay on this machine.</span></footer>
|
| 513 |
+
<script>
|
| 514 |
+
const $=s=>document.querySelector(s),$$=s=>[...document.querySelectorAll(s)];
|
| 515 |
+
const esc=s=>String(s).replace(/[&<>"]/g,c=>({'&':'&','<':'<','>':'>','"':'"'})[c]);
|
| 516 |
+
const BLOCK={"Text":"oklch(55% .02 263)","Section-Header":"oklch(45% .22 263)","Page-Header":"oklch(65% .03 263)","Page-Footer":"oklch(65% .03 263)","List-Group":"oklch(58% .15 145)","Table":"oklch(55% .18 320)","Caption":"oklch(62% .14 200)","Footnote":"oklch(60% .12 60)","Table-Of-Contents":"oklch(55% .12 180)","Form":"oklch(50% .16 25)"};
|
| 517 |
+
const STAGES=[["text_ms","Text layer","oklch(55% .15 300)"],["det_ms","Detect","oklch(45% .22 263)"],["rec_ms","Recognize","oklch(58% .15 145)"],["ocr_ms","OCR in images","oklch(62% .12 200)"],["math_ms","Math","oklch(55% .18 320)"],["order_ms","Order","oklch(68% .15 75)"]];
|
| 518 |
+
const HINT={auto:"Reads the PDF text layer where it passes the checks, and OCRs the rest.",ocr:"Ignores the text layer and OCRs every page.",text:"Uses the text layer even when the checks fail."};
|
| 519 |
+
let mode="auto",doc=null,stats=[],runId=0;
|
| 520 |
+
function setMode(v){mode=v;$$('#mode button').forEach(b=>b.classList.toggle('on',b.dataset.v===v));$('#modehint').textContent=HINT[v]}
|
| 521 |
+
$$('#mode button').forEach(b=>b.onclick=()=>setMode(b.dataset.v));
|
| 522 |
+
$('#legend').innerHTML=Object.entries(BLOCK).filter(([k])=>k!=="Page-Footer").map(([k,c])=>`<span style="white-space:nowrap;margin-right:8px"><span class="dot" style="background:${c}"></span>${k==='Page-Header'?'Header/Footer':k.replace(/-/g,' ')}</span>`).join(' ');
|
| 523 |
+
function applyDisplay(){const o=$('#out');$$('.disp').forEach(i=>o.classList.toggle('hide-'+i.dataset.c,!i.checked))}
|
| 524 |
+
$$('.disp').forEach(i=>i.onchange=applyDisplay);applyDisplay();
|
| 525 |
+
const drop=$('#drop'),file=$('#file');
|
| 526 |
+
drop.onclick=()=>file.click();file.onchange=()=>file.files[0]&&start(file.files[0]);
|
| 527 |
+
drop.ondragover=e=>{e.preventDefault();drop.classList.add('over')};drop.ondragleave=()=>drop.classList.remove('over');
|
| 528 |
+
drop.ondrop=e=>{e.preventDefault();drop.classList.remove('over');e.dataTransfer.files[0]&&start(e.dataTransfer.files[0])};
|
| 529 |
+
$('#rerun').onclick=()=>doc&&run();
|
| 530 |
+
fetch('/info').then(r=>r.json()).then(i=>{$('#ver').textContent='v'+i.version;if(i.tesseract)$('#tesslab').hidden=false;const d=i.defaults||{};
|
| 531 |
+
setMode(d.mode||'auto');$('#math').checked=!!d.math;$('#furniture').checked=!!d.furniture;$('#pdforder').checked=d.pdf_order!==false;$('#imageocr').checked=d.image_ocr!==false;
|
| 532 |
+
if(d.dpi)$('#dpi').value=String(Math.round(d.dpi));if(d.max_pages)$('#maxp').value=d.max_pages});
|
| 533 |
+
setMode('auto');
|
| 534 |
+
async function start(f){$('#fname').textContent=f.name+' · '+(f.size/1024).toFixed(0)+' KB';$('#status').textContent='Uploading ...';
|
| 535 |
+
const u=await (await fetch('/upload',{method:'POST',body:await f.arrayBuffer()})).json();
|
| 536 |
+
if(u.error){$('#status').textContent='Error: '+u.error;return}doc={...u,name:f.name};$('#rerun').disabled=false;run()}
|
| 537 |
+
function opts(){return{mode,math:$('#math').checked,furniture:$('#furniture').checked,model_order:!$('#pdforder').checked,image_ocr:$('#imageocr').checked,dpi:+$('#dpi').value,tesseract:$('#tess').checked}}
|
| 538 |
+
async function run(){const id=++runId,o=opts(),N=Math.min(doc.pages,Math.max(1,+$('#maxp').value||1));
|
| 539 |
+
$('#out').innerHTML='';stats=[];summary();const t0=performance.now();
|
| 540 |
+
for(let n=0;n<N;n++){if(id!==runId)return;$('#status').textContent=`Reading page ${n+1} of ${N}`+(doc.pages>N?` (first ${N} of ${doc.pages})`:'')+(o.math&&n===0?' · math models load on first use':'')+' ...';
|
| 541 |
+
const p=await (await fetch('/page',{method:'POST',body:JSON.stringify({id:doc.id,n,opts:o})})).json();if(id!==runId)return;
|
| 542 |
+
if(p.error){$('#status').textContent=`Error on page ${n+1}: ${p.error}`;return}render(p);stats.push(p.timings);summary()}
|
| 543 |
+
$('#status').textContent=`${N} page${N>1?'s':''} in ${(performance.now()-t0).toFixed(0)} ms (including display)`}
|
| 544 |
+
function summary(){const el=$('#stats');if(!stats.length){el.innerHTML='';return}const n=stats.length,s=k=>stats.reduce((a,t)=>a+(t[k]||0),0),tot=s('total_ms');
|
| 545 |
+
let h=`<b>${(tot/n).toFixed(0)} ms</b> per page · <b>${(1000*n/tot).toFixed(1)}</b> pages/s<br>`+STAGES.filter(([k])=>s(k)>0).map(([k,l])=>`${l} ${(s(k)/n).toFixed(0)} ms`).join(' · ');
|
| 546 |
+
const tt=stats.filter(t=>t.tesseract_ms!=null);if(tt.length){const tm=tt.reduce((a,t)=>a+t.tesseract_ms,0)/tt.length;h+=`<br>Tesseract ${tm.toFixed(0)} ms/page (${(tm/(tot/n)).toFixed(1)}x)`}
|
| 547 |
+
el.innerHTML=h}
|
| 548 |
+
function md(src){// a small markdown renderer: headings, lists, paragraphs, HTML tables, $math$
|
| 549 |
+
const lines=(src||'').split('\n'),out=[];let list=null,para=[];
|
| 550 |
+
const inline=t=>esc(t).replace(/\$\$([^$]+)\$\$|\$([^$\n]+)\$/g,(m,a,b)=>`<code class="m">${a||b}</code>`).replace(/\*\*([^*]+)\*\*/g,'<b>$1</b>');
|
| 551 |
+
const flush=()=>{if(para.length){out.push('<p>'+para.map(inline).join(' ')+'</p>');para=[]}if(list){out.push(`</${list}>`);list=null}};
|
| 552 |
+
for(const raw of lines){const l=raw.trimEnd();
|
| 553 |
+
if(/^<table/.test(l)||/^<\/?t(r|d|h|able|head|body)/.test(l)){flush();out.push(l.replace(/<(?!\/?(table|thead|tbody|tr|td|th)\b)[^>]*>/g,''));continue}
|
| 554 |
+
const h=l.match(/^(#{1,6})\s+(.*)/);if(h){flush();const k=Math.min(3,h[1].length);out.push(`<h${k}>${inline(h[2])}</h${k}>`);continue}
|
| 555 |
+
const li=l.match(/^\s*(?:[-*•]|(\d+)[.)])\s+(.*)/);if(li){if(para.length){out.push('<p>'+para.map(inline).join(' ')+'</p>');para=[]}const t=li[1]?'ol':'ul';if(list!==t){if(list)out.push(`</${list}>`);out.push(`<${t}>`);list=t}out.push(`<li>${inline(li[2])}</li>`);continue}
|
| 556 |
+
if(!l.trim()){flush();continue}if(list){out.push(`</${list}>`);list=null}para.push(l)}
|
| 557 |
+
flush();return out.join('\n')}
|
| 558 |
+
function render(p){const c=document.createElement('div');c.className='card';const t=p.timings,tot=t.total_ms||1;
|
| 559 |
+
const u=Math.max(1,p.w/640);// page units per screen pixel, about: keeps labels and markers readable
|
| 560 |
+
const src={text:'Text layer',ocr:'OCR',hybrid:'Text layer + OCR',error:'Error'}[p.source]||p.source;
|
| 561 |
+
const bar=STAGES.map(([k,l,col])=>`<i style="width:${100*(t[k]||0)/tot}%;background:${col}" title="${l} ${t[k]||0} ms"></i>`).join('');
|
| 562 |
+
const extra=(p.rotation?` · turned ${p.rotation}°`:'')+(p.skew?` · straightened ${(+p.skew).toFixed(1)}°`:'');
|
| 563 |
+
const blocks=p.blocks.map((b,k)=>{const col=BLOCK[b.label]||'oklch(55% .02 263)',[x0,y0,x1,y1]=b.box;return `<rect class="bk" data-b="${k}" x="${x0-3}" y="${y0-3}" width="${x1-x0+6}" height="${y1-y0+6}" stroke="${col}"/><text class="bl" x="${x0-2}" y="${Math.max(11*u,y0-5*u)}" fill="${col}" font-size="${11*u}" stroke-width="${3*u}">${esc(b.label)}</text>`}).join('');
|
| 564 |
+
const lines=p.lines.map((l,k)=>`<rect class="ln" data-k="${k}" x="${l.box[0]}" y="${l.box[1]}" width="${l.box[2]-l.box[0]}" height="${l.box[3]-l.box[1]}"/>`).join('');
|
| 565 |
+
const cen=p.lines.map(l=>[(l.box[0]+l.box[2])/2,(l.box[1]+l.box[3])/2]);
|
| 566 |
+
const ord=cen.length>1?`<path class="ord" d="M${cen.map(c=>c.join(',')).join(' L')}"/>`:'';
|
| 567 |
+
const nums=cen.map((c,k)=>`<circle class="on-n" cx="${c[0]}" cy="${c[1]}" r="${7*u}"/><text class="n" x="${c[0]}" y="${c[1]}" font-size="${8*u}">${k+1}</text>`).join('');
|
| 568 |
+
const rows=p.lines.map((l,k)=>`<div class="ln-row" data-k="${k}"><span class="k">${k+1}</span><span class="t">${esc(l.text)||'<span style=\"color:var(--ink-4)\">(no text)</span>'}</span><span class="c ${l.conf<.8?'low':''}">${l.conf.toFixed(2)}</span></div>`).join('');
|
| 569 |
+
const tabs=['Preview','Markdown','Lines','JSON'].concat(p.tesseract!=null?['Tesseract']:[]);
|
| 570 |
+
c.innerHTML=`<div class="ch"><h3>Page ${p.page}</h3><span class="chip ${p.source}">${src}</span><span class="meta">${p.lines.length} lines · ${p.blocks.length} blocks${extra}</span><span class="sp"></span><div class="tbar">${bar}</div><span class="meta">${t.total_ms} ms</span></div>
|
| 571 |
+
${p.note?`<div class="meta" style="padding:6px 16px;border-bottom:var(--hairline) solid var(--rule)">${esc(p.note)}</div>`:''}
|
| 572 |
+
<div class="body"><div class="pageview"><svg class="pg" viewBox="0 0 ${p.w} ${p.h}"><image href="${p.img}" width="${p.w}" height="${p.h}"/>${blocks}${lines}${ord}${nums}</svg></div>
|
| 573 |
+
<div class="right"><div class="tabs">${tabs.map((n,i)=>`<button data-tab="${n}" class="${i?'':'on'}">${n}</button>`).join('')}</div>
|
| 574 |
+
<div class="pane preview" data-pane="Preview">${md(p.markdown)}</div>
|
| 575 |
+
<div class="pane src" data-pane="Markdown" hidden>${esc(p.markdown||'')}</div>
|
| 576 |
+
<div class="pane" data-pane="Lines" hidden>${rows}</div>
|
| 577 |
+
<div class="pane src" data-pane="JSON" hidden>${esc(JSON.stringify(p.json,null,1))}</div>
|
| 578 |
+
${p.tesseract!=null?`<div class="pane src" data-pane="Tesseract" hidden>${esc(p.tesseract)}</div>`:''}</div></div>`;
|
| 579 |
+
c.querySelectorAll('.tabs button').forEach(b=>b.onclick=()=>{c.querySelectorAll('.tabs button').forEach(x=>x.classList.toggle('on',x===b));c.querySelectorAll('.pane').forEach(x=>x.hidden=x.dataset.pane!==b.dataset.tab)});
|
| 580 |
+
const hl=k=>c.querySelectorAll('[data-k]').forEach(el=>el.classList.toggle('on',el.dataset.k==k));
|
| 581 |
+
c.querySelectorAll('.ln-row').forEach(el=>{el.onmouseenter=()=>hl(el.dataset.k);el.onmouseleave=()=>hl(-1)});
|
| 582 |
+
c.querySelectorAll('rect.ln').forEach(el=>{el.onmouseenter=()=>{hl(el.dataset.k);const r=c.querySelector(`.ln-row[data-k="${el.dataset.k}"]`);r&&r.scrollIntoView({block:'nearest'})};el.onmouseleave=()=>hl(-1)});
|
| 583 |
+
c.querySelectorAll('rect.bk').forEach(el=>{el.onmouseenter=()=>el.classList.add('on');el.onmouseleave=()=>el.classList.remove('on')});
|
| 584 |
+
$('#out').appendChild(c)}
|
| 585 |
+
</script></body></html>"""
|
| 586 |
+
|
| 587 |
+
|
| 588 |
+
def main():
|
| 589 |
+
from .models import ModelError
|
| 590 |
+
|
| 591 |
+
try:
|
| 592 |
+
_main()
|
| 593 |
+
except ModelError as e:
|
| 594 |
+
print(f"indra-ocr-viewer: {e}", flush=True)
|
| 595 |
+
raise SystemExit(1) from None
|
| 596 |
+
|
| 597 |
+
|
| 598 |
+
def _main():
|
| 599 |
+
ap = argparse.ArgumentParser(
|
| 600 |
+
description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter
|
| 601 |
+
)
|
| 602 |
+
ap.add_argument(
|
| 603 |
+
"--model-dir",
|
| 604 |
+
type=Path,
|
| 605 |
+
default=None,
|
| 606 |
+
help="folder with the models (default: download from Hugging Face)",
|
| 607 |
+
)
|
| 608 |
+
add_source_args(ap)
|
| 609 |
+
ap.add_argument("--version", action="version", version=f"indra-ocr {__version__}")
|
| 610 |
+
ap.add_argument("--port", type=int, default=8765)
|
| 611 |
+
ap.add_argument("--threads", type=int, default=4)
|
| 612 |
+
ap.add_argument(
|
| 613 |
+
"--dpi", type=float, default=200.0, help="initial PDF render resolution"
|
| 614 |
+
)
|
| 615 |
+
ap.add_argument(
|
| 616 |
+
"--max-pages", type=int, default=20, help="initial page limit per document"
|
| 617 |
+
)
|
| 618 |
+
ap.add_argument(
|
| 619 |
+
"--keep-furniture",
|
| 620 |
+
action="store_true",
|
| 621 |
+
help="initially keep page headers and footers",
|
| 622 |
+
)
|
| 623 |
+
ap.add_argument(
|
| 624 |
+
"--tesseract", action="store_true", help="allow comparing with Tesseract"
|
| 625 |
+
)
|
| 626 |
+
ap.add_argument("--tesseract-lang", default="eng")
|
| 627 |
+
ap.add_argument("--math", action="store_true", help="initially write math as LaTeX")
|
| 628 |
+
a = ap.parse_args()
|
| 629 |
+
src = source_options(a)
|
| 630 |
+
defaults = {
|
| 631 |
+
"mode": src["mode"],
|
| 632 |
+
"pdf_order": src["pdf_order"],
|
| 633 |
+
"image_ocr": src["image_ocr"],
|
| 634 |
+
"math": a.math,
|
| 635 |
+
"furniture": a.keep_furniture,
|
| 636 |
+
"dpi": a.dpi,
|
| 637 |
+
"max_pages": a.max_pages,
|
| 638 |
+
}
|
| 639 |
+
engine = Engine(a.model_dir, a.threads, a.tesseract, a.tesseract_lang, defaults)
|
| 640 |
+
srv = ThreadingHTTPServer(("127.0.0.1", a.port), make_handler(engine))
|
| 641 |
+
print(
|
| 642 |
+
f"Indra OCR viewer on http://localhost:{a.port} ({a.threads} threads)",
|
| 643 |
+
flush=True,
|
| 644 |
+
)
|
| 645 |
+
srv.serve_forever()
|
| 646 |
+
|
| 647 |
+
|
| 648 |
+
if __name__ == "__main__":
|
| 649 |
+
main()
|
OCR_INDRA/indra/indra_ocr/vocab.py
ADDED
|
@@ -0,0 +1,289 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""Token vocabulary for the line recognizer.
|
| 2 |
+
|
| 3 |
+
Class 0 is the CTC blank. Class i + 1 is ``tokens[i]``. A token is one
|
| 4 |
+
character, or one LaTeX command (``\\frac``, ``\\alpha``) inside math. Labels
|
| 5 |
+
are NFC normalized before encoding. A training line with a token outside the
|
| 6 |
+
vocabulary is dropped (``encode`` returns None), never silently altered.
|
| 7 |
+
|
| 8 |
+
``Vocab.latin()`` is the first recognizer's character set. ``Vocab.multi()``
|
| 9 |
+
adds Cyrillic, full Greek, Devanagari, Bengali and LaTeX command tokens; math
|
| 10 |
+
is written ``$...$`` in labels.
|
| 11 |
+
"""
|
| 12 |
+
|
| 13 |
+
from __future__ import annotations
|
| 14 |
+
|
| 15 |
+
import json
|
| 16 |
+
import re
|
| 17 |
+
import unicodedata
|
| 18 |
+
|
| 19 |
+
# Label canonicalization. Each rule maps characters or runs that the image
|
| 20 |
+
# cannot tell apart to one form, so neither training targets nor the eval
|
| 21 |
+
# punish a reading that is visually correct.
|
| 22 |
+
_CANON = str.maketrans(
|
| 23 |
+
{
|
| 24 |
+
# Romanian comma-below and Turkish/legacy cedilla look the same in most fonts.
|
| 25 |
+
"\u0218": "\u015e", # Ș -> Ş
|
| 26 |
+
"\u0219": "\u015f", # ș -> ş
|
| 27 |
+
"\u021a": "\u0162", # Ț -> Ţ
|
| 28 |
+
"\u021b": "\u0163", # ț -> ţ
|
| 29 |
+
# Bullet look-alikes (the font decides the drawn shape).
|
| 30 |
+
"\u25aa": "\u2022", # ▪
|
| 31 |
+
"\u25cf": "\u2022", # ●
|
| 32 |
+
"\u2219": "\u2022", # ∙
|
| 33 |
+
"\u2023": "\u2022", # ‣
|
| 34 |
+
"\u2043": "\u2022", # ⁃
|
| 35 |
+
}
|
| 36 |
+
)
|
| 37 |
+
# Runs of 4+ leader / rule characters: the count cannot be read from the image.
|
| 38 |
+
_RUNS = re.compile(r"([._\-=·*~])\1{3,}")
|
| 39 |
+
_DOUBLE_SINGLE = re.compile(r"''|``")
|
| 40 |
+
|
| 41 |
+
|
| 42 |
+
def canonical(text: str) -> str:
|
| 43 |
+
text = unicodedata.normalize("NFC", text).translate(_CANON)
|
| 44 |
+
text = _DOUBLE_SINGLE.sub('"', text)
|
| 45 |
+
return _RUNS.sub(lambda m: m.group(1) * 3, text)
|
| 46 |
+
|
| 47 |
+
|
| 48 |
+
def _range(a: int, b: int) -> list[str]:
|
| 49 |
+
return [chr(c) for c in range(a, b + 1)]
|
| 50 |
+
|
| 51 |
+
|
| 52 |
+
def _printable(chars) -> list[str]:
|
| 53 |
+
out = []
|
| 54 |
+
for c in chars:
|
| 55 |
+
cat = unicodedata.category(c)
|
| 56 |
+
if cat.startswith(("C", "Z")) and c != " ":
|
| 57 |
+
continue # control, format, unassigned, private, separators other than space
|
| 58 |
+
if cat == "Mn":
|
| 59 |
+
continue # combining marks: labels are NFC, marks come precomposed
|
| 60 |
+
out.append(c)
|
| 61 |
+
return out
|
| 62 |
+
|
| 63 |
+
|
| 64 |
+
def _script_chars(a: int, b: int) -> list[str]:
|
| 65 |
+
"""Assigned letters, marks and digits of a script block (combining marks kept:
|
| 66 |
+
Indic vowel signs and virama are separate codepoints even after NFC)."""
|
| 67 |
+
out = []
|
| 68 |
+
for c in _range(a, b):
|
| 69 |
+
cat = unicodedata.category(c)
|
| 70 |
+
if cat == "Cn" or cat.startswith(("C", "Z")):
|
| 71 |
+
continue
|
| 72 |
+
out.append(c)
|
| 73 |
+
return out
|
| 74 |
+
|
| 75 |
+
|
| 76 |
+
# LaTeX commands the recognizer emits as single tokens inside $...$.
|
| 77 |
+
LATEX_COMMANDS = (
|
| 78 |
+
"alpha beta gamma delta epsilon varepsilon zeta eta theta vartheta iota kappa lambda mu nu xi "
|
| 79 |
+
"pi varpi rho varrho sigma varsigma tau upsilon phi varphi chi psi omega "
|
| 80 |
+
"Gamma Delta Theta Lambda Xi Pi Sigma Upsilon Phi Psi Omega "
|
| 81 |
+
"frac dfrac tfrac sqrt sum prod int iint oint lim limsup liminf sup inf max min arg det "
|
| 82 |
+
"sin cos tan cot sec csc arcsin arccos arctan sinh cosh tanh log ln exp "
|
| 83 |
+
"left right big Big bigg Bigg langle rangle lfloor rfloor lceil rceil "
|
| 84 |
+
"cdot cdots ldots dots vdots ddots times div pm mp ast star circ bullet "
|
| 85 |
+
"leq geq neq le ge ne ll gg approx sim simeq cong equiv propto "
|
| 86 |
+
"in notin ni subset supset subseteq supseteq cup cap cap setminus emptyset varnothing "
|
| 87 |
+
"forall exists neg land lor wedge vee oplus otimes "
|
| 88 |
+
"to rightarrow leftarrow Rightarrow Leftarrow leftrightarrow Leftrightarrow mapsto "
|
| 89 |
+
"implies iff uparrow downarrow longrightarrow "
|
| 90 |
+
"infty partial nabla hbar ell prime perp parallel mid top angle "
|
| 91 |
+
"hat bar tilde vec dot ddot widehat widetilde overline underline "
|
| 92 |
+
"mathbb mathcal mathbf mathrm mathit mathsf mathfrak boldsymbol operatorname text "
|
| 93 |
+
"quad qquad "
|
| 94 |
+
).split()
|
| 95 |
+
LATEX_TOKENS = ["\\" + c for c in LATEX_COMMANDS] + [
|
| 96 |
+
"\\{",
|
| 97 |
+
"\\}",
|
| 98 |
+
"\\|",
|
| 99 |
+
"\\,",
|
| 100 |
+
"\\;",
|
| 101 |
+
"\\!",
|
| 102 |
+
"\\ ",
|
| 103 |
+
]
|
| 104 |
+
_TOKEN_RE = re.compile(r"\\[A-Za-z]+|\\[{}|,;! ]|.", re.S)
|
| 105 |
+
# $...$ is math only when it holds LaTeX, so prices ("$5 or $10") stay text.
|
| 106 |
+
_MATH_SPAN = re.compile(r"\$[^$]*[\\^_{][^$]*\$")
|
| 107 |
+
|
| 108 |
+
|
| 109 |
+
def build_latin_chars() -> list[str]:
|
| 110 |
+
chars: list[str] = []
|
| 111 |
+
chars += _range(0x20, 0x7E) # ASCII printable, with space
|
| 112 |
+
chars += _range(
|
| 113 |
+
0x00A1, 0x00FF
|
| 114 |
+
) # Latin-1 Supplement (letters, ¢ £ ¥ § © ° ± µ ¹²³ ½ ¼ ¾ × ÷ ...)
|
| 115 |
+
chars += _range(0x0100, 0x017F) # Latin Extended-A
|
| 116 |
+
chars += _range(
|
| 117 |
+
0x0180, 0x024F
|
| 118 |
+
) # Latin Extended-B (Ș ș Ț ț, Vietnamese Ơ ơ Ư ư, pinyin)
|
| 119 |
+
chars += _range(
|
| 120 |
+
0x1E00, 0x1EFF
|
| 121 |
+
) # Latin Extended Additional (Vietnamese precomposed)
|
| 122 |
+
chars += _range(0x0391, 0x03A9) + _range(0x03B1, 0x03C9) # basic Greek
|
| 123 |
+
chars += list("‐‑‒–—―‖‘’‚‛“”„‟†‡•‣…‰′″‹›⁄※")
|
| 124 |
+
chars += list("⁰⁴⁵⁶⁷⁸⁹⁺⁻⁽⁾ⁿ₀₁₂₃₄₅₆₇₈₉")
|
| 125 |
+
chars += list("€₹₽₩₺₪₱₫₴₦₡₸")
|
| 126 |
+
chars += list("™℃℉№ℓ℮⅓⅔⅛")
|
| 127 |
+
chars += list("←↑→↓↔⇒⇔↵")
|
| 128 |
+
chars += list("−∓∗∘∙√∞∝∠∧∨∩∪∫≈≠≡≤≥⊂⊃⊆⊇⊕⊗⊥⋅∂∆∇∈∉∋∏∑∀∃∅")
|
| 129 |
+
chars += list("■□▪▫▲△▶►▼▽◀◄◆◇○◦●★☆✓✔✗✘❖➢➤")
|
| 130 |
+
seen, out = set(), []
|
| 131 |
+
for c in _printable(chars):
|
| 132 |
+
if c not in seen and c != "": # soft hyphen is invisible
|
| 133 |
+
seen.add(c)
|
| 134 |
+
out.append(c)
|
| 135 |
+
return out
|
| 136 |
+
|
| 137 |
+
|
| 138 |
+
def build_multi_tokens() -> list[str]:
|
| 139 |
+
out = build_latin_chars()
|
| 140 |
+
seen = set(out)
|
| 141 |
+
extra = (
|
| 142 |
+
_script_chars(0x0370, 0x03FF) # Greek and Coptic (accented Greek)
|
| 143 |
+
+ _script_chars(0x0400, 0x045F) # Cyrillic (Russian, Ukrainian, Serbian, ...)
|
| 144 |
+
+ _script_chars(
|
| 145 |
+
0x0490, 0x04FF
|
| 146 |
+
) # Cyrillic extended letters (Kazakh, Uzbek, ...)
|
| 147 |
+
+ _script_chars(0x0900, 0x097F) # Devanagari (Hindi, Marathi, Nepali)
|
| 148 |
+
+ _script_chars(0x0980, 0x09FF) # Bengali
|
| 149 |
+
+ list("।॥₹")
|
| 150 |
+
+ LATEX_TOKENS
|
| 151 |
+
)
|
| 152 |
+
for t in extra:
|
| 153 |
+
if t not in seen:
|
| 154 |
+
seen.add(t)
|
| 155 |
+
out.append(t)
|
| 156 |
+
return out
|
| 157 |
+
|
| 158 |
+
|
| 159 |
+
class Vocab:
|
| 160 |
+
def __init__(self, chars: list[str]):
|
| 161 |
+
self.chars = list(chars) # tokens: characters, or LaTeX commands
|
| 162 |
+
self.index = {c: i + 1 for i, c in enumerate(self.chars)}
|
| 163 |
+
self.has_commands = any(len(c) > 1 for c in self.chars)
|
| 164 |
+
|
| 165 |
+
@classmethod
|
| 166 |
+
def latin(cls) -> "Vocab":
|
| 167 |
+
return cls(build_latin_chars())
|
| 168 |
+
|
| 169 |
+
@classmethod
|
| 170 |
+
def multi(cls) -> "Vocab":
|
| 171 |
+
return cls(build_multi_tokens())
|
| 172 |
+
|
| 173 |
+
@classmethod
|
| 174 |
+
def named(cls, name: str) -> "Vocab":
|
| 175 |
+
if name not in ("latin", "multi"):
|
| 176 |
+
raise ValueError(f"unknown vocab {name!r}")
|
| 177 |
+
return cls.multi() if name == "multi" else cls.latin()
|
| 178 |
+
|
| 179 |
+
def tokenize(self, text: str) -> list[str]:
|
| 180 |
+
"""Characters, with a LaTeX command as one token when the vocab has it.
|
| 181 |
+
|
| 182 |
+
Inside a $...$ math span spaces are dropped: they are not visible in
|
| 183 |
+
rendered math. ``decode`` puts back the one space LaTeX needs, between a
|
| 184 |
+
command and a following letter."""
|
| 185 |
+
if not self.has_commands:
|
| 186 |
+
return list(text)
|
| 187 |
+
out: list[str] = []
|
| 188 |
+
pos = 0
|
| 189 |
+
for m in list(_MATH_SPAN.finditer(text)) + [None]:
|
| 190 |
+
end = m.start() if m else len(text)
|
| 191 |
+
out += self._tokens(text[pos:end], math=False)
|
| 192 |
+
if m:
|
| 193 |
+
out += self._tokens(m.group(0), math=True)
|
| 194 |
+
pos = m.end()
|
| 195 |
+
return out
|
| 196 |
+
|
| 197 |
+
def _tokens(self, text: str, math: bool) -> list[str]:
|
| 198 |
+
out = []
|
| 199 |
+
for t in _TOKEN_RE.findall(text):
|
| 200 |
+
if math and t.isspace():
|
| 201 |
+
continue
|
| 202 |
+
out.extend([t] if t in self.index or len(t) == 1 else list(t))
|
| 203 |
+
return out
|
| 204 |
+
|
| 205 |
+
@property
|
| 206 |
+
def n_classes(self) -> int:
|
| 207 |
+
return len(self.chars) + 1 # + blank
|
| 208 |
+
|
| 209 |
+
def normalize(self, text: str) -> str:
|
| 210 |
+
return canonical(text)
|
| 211 |
+
|
| 212 |
+
def encode(self, text: str) -> list[int] | None:
|
| 213 |
+
text = self.normalize(text)
|
| 214 |
+
ids = [self.index.get(c) for c in self.tokenize(text)]
|
| 215 |
+
if not ids or any(i is None for i in ids):
|
| 216 |
+
return None
|
| 217 |
+
return ids
|
| 218 |
+
|
| 219 |
+
def oov(self, text: str) -> set[str]:
|
| 220 |
+
return {c for c in self.tokenize(self.normalize(text)) if c not in self.index}
|
| 221 |
+
|
| 222 |
+
def decode(self, ids: list[int]) -> str:
|
| 223 |
+
toks = [self.chars[i - 1] for i in ids if 0 < i <= len(self.chars)]
|
| 224 |
+
if not self.has_commands:
|
| 225 |
+
return "".join(toks)
|
| 226 |
+
out = []
|
| 227 |
+
for k, t in enumerate(toks):
|
| 228 |
+
out.append(t)
|
| 229 |
+
# "\\alpha" + "x" must not read back as "\\alphax"
|
| 230 |
+
nxt = toks[k + 1] if k + 1 < len(toks) else ""
|
| 231 |
+
if len(t) > 2 and t[0] == "\\" and t[1:].isalpha() and nxt[:1].isalpha():
|
| 232 |
+
out.append(" ")
|
| 233 |
+
return "".join(out)
|
| 234 |
+
|
| 235 |
+
def to_json(self) -> str:
|
| 236 |
+
return json.dumps({"blank": 0, "chars": self.chars}, ensure_ascii=False)
|
| 237 |
+
|
| 238 |
+
@classmethod
|
| 239 |
+
def from_json(cls, s: str) -> "Vocab":
|
| 240 |
+
d = json.loads(s)
|
| 241 |
+
if d.get("blank", 0) != 0:
|
| 242 |
+
raise ValueError("only blank=0 vocabularies are supported")
|
| 243 |
+
return cls(d["chars"])
|
| 244 |
+
|
| 245 |
+
|
| 246 |
+
# Indic syllable structure. A broken font mapping (common in Nepali and Hindi
|
| 247 |
+
# PDFs converted from legacy fonts) leaves vowel signs with no consonant before
|
| 248 |
+
# them ("िामथ", "का ा") or two vowel signs in a row ("णािी"); valid text never does.
|
| 249 |
+
_INDIC_BLOCKS = ((0x0900, 0x097F), (0x0980, 0x09FF))
|
| 250 |
+
_VIRAMA = {0x094D, 0x09CD}
|
| 251 |
+
_NUKTA = {0x093C, 0x09BC}
|
| 252 |
+
_MODIFIERS = {0x0900, 0x0901, 0x0902, 0x0903, 0x0981, 0x0982, 0x0983}
|
| 253 |
+
|
| 254 |
+
|
| 255 |
+
def _indic(c: str) -> bool:
|
| 256 |
+
o = ord(c)
|
| 257 |
+
return any(a <= o <= b for a, b in _INDIC_BLOCKS)
|
| 258 |
+
|
| 259 |
+
|
| 260 |
+
def indic_valid(text: str) -> bool:
|
| 261 |
+
"""False when a Devanagari or Bengali mark has no base to attach to."""
|
| 262 |
+
prev = ""
|
| 263 |
+
for c in text:
|
| 264 |
+
if _indic(c) and unicodedata.category(c) in ("Mn", "Mc"):
|
| 265 |
+
o = ord(c)
|
| 266 |
+
p_ok = bool(prev) and _indic(prev)
|
| 267 |
+
p_cat = unicodedata.category(prev) if prev else ""
|
| 268 |
+
if (
|
| 269 |
+
o in _MODIFIERS
|
| 270 |
+
): # anusvara, candrabindu, visarga: after a letter or a vowel sign
|
| 271 |
+
if not p_ok:
|
| 272 |
+
return False
|
| 273 |
+
elif o in _VIRAMA or o in _NUKTA: # after a consonant (or nukta)
|
| 274 |
+
if not (p_ok and (p_cat == "Lo" or ord(prev) in _NUKTA)):
|
| 275 |
+
return False
|
| 276 |
+
else: # vowel sign: after a consonant, nukta, or nothing else
|
| 277 |
+
if not (p_ok and (p_cat == "Lo" or ord(prev) in _NUKTA)):
|
| 278 |
+
return False
|
| 279 |
+
prev = c
|
| 280 |
+
return True
|
| 281 |
+
|
| 282 |
+
|
| 283 |
+
def indic_page_ok(texts: list[str], max_bad: float = 0.05) -> bool:
|
| 284 |
+
"""A page whose Indic lines break syllable structure more than ``max_bad`` of
|
| 285 |
+
the time has a broken font mapping; its other lines are suspect too."""
|
| 286 |
+
ind = [t for t in texts if any(_indic(c) for c in t)]
|
| 287 |
+
if not ind:
|
| 288 |
+
return True
|
| 289 |
+
return sum(not indic_valid(t) for t in ind) <= max_bad * len(ind)
|
OCR_INDRA/indra/pyproject.toml
ADDED
|
@@ -0,0 +1,37 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
[build-system]
|
| 2 |
+
requires = ["setuptools>=77"]
|
| 3 |
+
build-backend = "setuptools.build_meta"
|
| 4 |
+
|
| 5 |
+
[project]
|
| 6 |
+
name = "indra-ocr"
|
| 7 |
+
version = "0.3.2"
|
| 8 |
+
description = "Fast CPU OCR: line detection, CTC recognition, reading order and markdown, on ONNX Runtime"
|
| 9 |
+
readme = "README.md"
|
| 10 |
+
license = "Apache-2.0"
|
| 11 |
+
license-files = ["LICENSE"]
|
| 12 |
+
requires-python = ">=3.10"
|
| 13 |
+
dependencies = [
|
| 14 |
+
"numpy",
|
| 15 |
+
"opencv-python-headless",
|
| 16 |
+
"onnxruntime",
|
| 17 |
+
"pypdfium2",
|
| 18 |
+
"huggingface_hub",
|
| 19 |
+
]
|
| 20 |
+
|
| 21 |
+
[project.urls]
|
| 22 |
+
Homepage = "https://www.datalab.to"
|
| 23 |
+
Repository = "https://github.com/datalab-to/indra"
|
| 24 |
+
Models = "https://huggingface.co/datalab-to/indra-ocr"
|
| 25 |
+
Changelog = "https://github.com/datalab-to/indra/blob/main/CHANGELOG.md"
|
| 26 |
+
Issues = "https://github.com/datalab-to/indra/issues"
|
| 27 |
+
|
| 28 |
+
[project.optional-dependencies]
|
| 29 |
+
tesseract = ["pytesseract"]
|
| 30 |
+
test = ["pytest", "pillow", "pikepdf"]
|
| 31 |
+
|
| 32 |
+
[project.scripts]
|
| 33 |
+
indra-ocr = "indra_ocr.cli:main"
|
| 34 |
+
indra-ocr-viewer = "indra_ocr.viewer:main"
|
| 35 |
+
|
| 36 |
+
[tool.setuptools]
|
| 37 |
+
packages = ["indra_ocr"]
|
OCR_INDRA/indra/tests/test_inputs.py
ADDED
|
@@ -0,0 +1,201 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
import numpy as np
|
| 2 |
+
import pytest
|
| 3 |
+
|
| 4 |
+
from indra_ocr.document import MAX_PAGE_PIXELS, Converter
|
| 5 |
+
from indra_ocr.pipeline import InputError, Page, is_pdf, read_images
|
| 6 |
+
|
| 7 |
+
|
| 8 |
+
class StubOCR:
|
| 9 |
+
"""Counts pages; fails on pages that are all black."""
|
| 10 |
+
|
| 11 |
+
vocab = vocab_cjk = None
|
| 12 |
+
|
| 13 |
+
def __init__(self):
|
| 14 |
+
self.seen = []
|
| 15 |
+
|
| 16 |
+
def __call__(self, gray, turn=True):
|
| 17 |
+
self.seen.append(gray.shape)
|
| 18 |
+
if gray.max() == 0:
|
| 19 |
+
raise RuntimeError("stub failure")
|
| 20 |
+
return Page([])
|
| 21 |
+
|
| 22 |
+
|
| 23 |
+
def _pdf(path, sizes):
|
| 24 |
+
import pypdfium2 as pdfium
|
| 25 |
+
|
| 26 |
+
doc = pdfium.PdfDocument.new()
|
| 27 |
+
for w, h in sizes:
|
| 28 |
+
doc.new_page(w, h)
|
| 29 |
+
doc.save(str(path))
|
| 30 |
+
return path
|
| 31 |
+
|
| 32 |
+
|
| 33 |
+
def test_file_type_comes_from_the_bytes(tmp_path):
|
| 34 |
+
import cv2
|
| 35 |
+
|
| 36 |
+
pdf = _pdf(tmp_path / "no_extension", [(200, 300)])
|
| 37 |
+
png = tmp_path / "image_named.pdf"
|
| 38 |
+
cv2.imwrite(str(tmp_path / "x.png"), np.full((50, 40), 255, np.uint8))
|
| 39 |
+
(tmp_path / "x.png").rename(png)
|
| 40 |
+
assert is_pdf(pdf) and not is_pdf(png)
|
| 41 |
+
conv = Converter(StubOCR(), mode="ocr")
|
| 42 |
+
assert [p.source for _, p in conv.convert(pdf)] == ["ocr"]
|
| 43 |
+
assert len(list(conv.convert(png))) == 1
|
| 44 |
+
|
| 45 |
+
|
| 46 |
+
def test_every_tiff_frame_is_a_page(tmp_path):
|
| 47 |
+
from PIL import Image
|
| 48 |
+
|
| 49 |
+
frames = [Image.new("L", (60, 80), v) for v in (255, 200, 255)]
|
| 50 |
+
path = tmp_path / "three.tiff"
|
| 51 |
+
frames[0].save(path, save_all=True, append_images=frames[1:])
|
| 52 |
+
assert len(read_images(path)) == 3
|
| 53 |
+
|
| 54 |
+
|
| 55 |
+
def test_unreadable_files_raise_input_error(tmp_path):
|
| 56 |
+
conv = Converter(StubOCR())
|
| 57 |
+
for name, data in (
|
| 58 |
+
("garbage.pdf", b"%PDF-1.4 broken"),
|
| 59 |
+
("empty.png", b""),
|
| 60 |
+
("text.txt", b"hello"),
|
| 61 |
+
):
|
| 62 |
+
(tmp_path / name).write_bytes(data)
|
| 63 |
+
with pytest.raises(InputError):
|
| 64 |
+
list(conv.convert(tmp_path / name))
|
| 65 |
+
with pytest.raises(InputError, match="no such file"):
|
| 66 |
+
list(conv.convert(tmp_path / "missing.pdf"))
|
| 67 |
+
|
| 68 |
+
|
| 69 |
+
def test_password_protected_pdf(tmp_path):
|
| 70 |
+
pikepdf = pytest.importorskip("pikepdf")
|
| 71 |
+
src = _pdf(tmp_path / "plain.pdf", [(200, 300)])
|
| 72 |
+
locked = tmp_path / "locked.pdf"
|
| 73 |
+
pikepdf.open(src).save(
|
| 74 |
+
locked, encryption=pikepdf.Encryption(user="pw", owner="owner")
|
| 75 |
+
)
|
| 76 |
+
conv = Converter(StubOCR(), mode="ocr")
|
| 77 |
+
with pytest.raises(InputError, match="password"):
|
| 78 |
+
list(conv.convert(locked))
|
| 79 |
+
assert len(list(conv.convert(locked, password="pw"))) == 1
|
| 80 |
+
|
| 81 |
+
|
| 82 |
+
def test_a_failing_page_does_not_end_the_document(tmp_path):
|
| 83 |
+
from PIL import Image
|
| 84 |
+
|
| 85 |
+
frames = [
|
| 86 |
+
Image.new("L", (60, 80), v) for v in (255, 0, 255)
|
| 87 |
+
] # page 2 makes the stub fail
|
| 88 |
+
path = tmp_path / "pages.tiff"
|
| 89 |
+
frames[0].save(path, save_all=True, append_images=frames[1:])
|
| 90 |
+
pages = [p for _, p in Converter(StubOCR()).convert(path)]
|
| 91 |
+
assert [p.source for p in pages] == ["ocr", "error", "ocr"]
|
| 92 |
+
assert "stub failure" in pages[1].note
|
| 93 |
+
assert pages[1].markdown == "" and pages[1].timings["total_ms"] == 0.0
|
| 94 |
+
|
| 95 |
+
|
| 96 |
+
def test_huge_pages_render_below_the_pixel_cap(tmp_path):
|
| 97 |
+
import pypdfium2 as pdfium
|
| 98 |
+
|
| 99 |
+
pdf = _pdf(tmp_path / "poster.pdf", [(14400, 14400), (612, 792)])
|
| 100 |
+
conv = Converter(StubOCR(), mode="ocr")
|
| 101 |
+
doc = pdfium.PdfDocument(str(pdf))
|
| 102 |
+
big, letter = conv.page_scale(doc[0]), conv.page_scale(doc[1])
|
| 103 |
+
assert (14400 * big) ** 2 <= MAX_PAGE_PIXELS * 1.01 and letter == pytest.approx(
|
| 104 |
+
200 / 72
|
| 105 |
+
)
|
| 106 |
+
list(conv.convert(pdf))
|
| 107 |
+
assert conv.ocr.seen[0][0] * conv.ocr.seen[0][1] <= MAX_PAGE_PIXELS * 1.01
|
| 108 |
+
|
| 109 |
+
|
| 110 |
+
def test_parallel_plan_has_one_unit_per_page(tmp_path):
|
| 111 |
+
import cv2
|
| 112 |
+
|
| 113 |
+
from indra_ocr.parallel import _plan, usable_cores
|
| 114 |
+
|
| 115 |
+
pdf = _pdf(tmp_path / "five.pdf", [(200, 300)] * 5)
|
| 116 |
+
png = tmp_path / "scan.png"
|
| 117 |
+
cv2.imwrite(str(png), np.full((50, 40), 255, np.uint8))
|
| 118 |
+
bad = tmp_path / "bad.pdf"
|
| 119 |
+
bad.write_bytes(b"%PDF-1.4 broken")
|
| 120 |
+
plan = _plan([pdf, tmp_path / "missing.pdf", bad, png], max_pages=3, password=None)
|
| 121 |
+
assert [u[1] for u in plan[0]] == [0, 1, 2] # max_pages caps the units
|
| 122 |
+
assert isinstance(plan[1], InputError) and isinstance(plan[2], InputError)
|
| 123 |
+
assert plan[3] == [(str(png), None, 3)] # an image file is one unit
|
| 124 |
+
assert usable_cores() >= 1
|
| 125 |
+
|
| 126 |
+
|
| 127 |
+
def test_model_errors_are_one_clear_message(tmp_path, monkeypatch, capsys):
|
| 128 |
+
import huggingface_hub
|
| 129 |
+
|
| 130 |
+
from indra_ocr import ModelError
|
| 131 |
+
from indra_ocr.cli import main
|
| 132 |
+
from indra_ocr.models import model_dir
|
| 133 |
+
|
| 134 |
+
with pytest.raises(ModelError, match="lacks det.onnx"):
|
| 135 |
+
model_dir(tmp_path)
|
| 136 |
+
|
| 137 |
+
def offline(*a, **k):
|
| 138 |
+
raise ConnectionError("network is unreachable")
|
| 139 |
+
|
| 140 |
+
monkeypatch.setattr(huggingface_hub, "snapshot_download", offline)
|
| 141 |
+
monkeypatch.delenv("INDRA_OCR_MODEL_DIR", raising=False)
|
| 142 |
+
with pytest.raises(ModelError, match="network is unreachable"):
|
| 143 |
+
model_dir(None)
|
| 144 |
+
img = tmp_path / "a.png"
|
| 145 |
+
import cv2
|
| 146 |
+
|
| 147 |
+
cv2.imwrite(str(img), np.full((50, 40), 255, np.uint8))
|
| 148 |
+
assert main([str(img)]) == 1
|
| 149 |
+
err = capsys.readouterr().err.strip().splitlines()
|
| 150 |
+
assert len(err) == 1 and err[0].startswith(
|
| 151 |
+
"indra-ocr: could not download the models"
|
| 152 |
+
)
|
| 153 |
+
|
| 154 |
+
|
| 155 |
+
def test_version_flag(capsys):
|
| 156 |
+
from indra_ocr import __version__
|
| 157 |
+
from indra_ocr.cli import main
|
| 158 |
+
|
| 159 |
+
with pytest.raises(SystemExit):
|
| 160 |
+
main(["--version"])
|
| 161 |
+
assert capsys.readouterr().out.strip() == f"indra-ocr {__version__}"
|
| 162 |
+
|
| 163 |
+
|
| 164 |
+
def test_cached_models_are_checked_once_a_day(tmp_path, monkeypatch):
|
| 165 |
+
import os
|
| 166 |
+
import time
|
| 167 |
+
|
| 168 |
+
import huggingface_hub
|
| 169 |
+
from huggingface_hub import constants
|
| 170 |
+
|
| 171 |
+
from indra_ocr import models
|
| 172 |
+
|
| 173 |
+
monkeypatch.delenv("INDRA_OCR_MODEL_DIR", raising=False)
|
| 174 |
+
monkeypatch.setattr(constants, "HF_HUB_CACHE", str(tmp_path))
|
| 175 |
+
calls = []
|
| 176 |
+
state = {"cached": False, "online": True}
|
| 177 |
+
|
| 178 |
+
def fake(repo, local_files_only=False, **kw):
|
| 179 |
+
calls.append("local" if local_files_only else "hub")
|
| 180 |
+
if local_files_only and not state["cached"]:
|
| 181 |
+
raise FileNotFoundError("not cached")
|
| 182 |
+
if not local_files_only and not state["online"]:
|
| 183 |
+
raise ConnectionError("offline")
|
| 184 |
+
state["cached"] = True
|
| 185 |
+
return str(tmp_path / "snap")
|
| 186 |
+
|
| 187 |
+
monkeypatch.setattr(huggingface_hub, "snapshot_download", fake)
|
| 188 |
+
models.model_dir(None) # first run: downloads
|
| 189 |
+
assert calls == ["local", "hub"]
|
| 190 |
+
calls.clear()
|
| 191 |
+
models.model_dir(None) # checked under a day ago: cache only
|
| 192 |
+
assert calls == ["local"]
|
| 193 |
+
stamp = models._check_stamp(models.HF_REVISION)
|
| 194 |
+
old = time.time() - models.CHECK_EVERY_S - 60
|
| 195 |
+
os.utime(stamp, (old, old))
|
| 196 |
+
calls.clear()
|
| 197 |
+
models.model_dir(None) # a day later: checks the Hub again
|
| 198 |
+
assert calls == ["local", "hub"]
|
| 199 |
+
os.utime(stamp, (old, old))
|
| 200 |
+
state["online"] = False
|
| 201 |
+
assert models.model_dir(None) == tmp_path / "snap" # offline: the cached models
|
OCR_INDRA/indra/tests/test_textlayer.py
ADDED
|
@@ -0,0 +1,307 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
import numpy as np
|
| 2 |
+
|
| 3 |
+
from indra_ocr.textlayer import (
|
| 4 |
+
bad_char_count,
|
| 5 |
+
order_backtracks,
|
| 6 |
+
pdf_order_coherent,
|
| 7 |
+
)
|
| 8 |
+
|
| 9 |
+
|
| 10 |
+
def test_bad_chars_count_replacement_and_private_use():
|
| 11 |
+
assert bad_char_count("ok") == 0
|
| 12 |
+
assert bad_char_count("a�") == 2
|
| 13 |
+
|
| 14 |
+
|
| 15 |
+
def _box(x0, y0, x1, y1):
|
| 16 |
+
return [x0, y0, x1, y1]
|
| 17 |
+
|
| 18 |
+
|
| 19 |
+
def test_two_columns_in_reading_order_are_coherent():
|
| 20 |
+
left = [_box(50, 100 + 15 * i, 290, 110 + 15 * i) for i in range(20)]
|
| 21 |
+
right = [_box(310, 100 + 15 * i, 550, 110 + 15 * i) for i in range(20)]
|
| 22 |
+
boxes = np.array(left + right, np.float32)
|
| 23 |
+
assert order_backtracks(boxes) == 0 and pdf_order_coherent(boxes)
|
| 24 |
+
|
| 25 |
+
|
| 26 |
+
def test_interleaved_columns_are_not_coherent():
|
| 27 |
+
left = [_box(50, 100 + 15 * i, 290, 110 + 15 * i) for i in range(20)]
|
| 28 |
+
right = [_box(310, 100 + 15 * i, 550, 110 + 15 * i) for i in range(20)]
|
| 29 |
+
# line by line across the gutter: every move back to the left column goes up-left
|
| 30 |
+
boxes = np.array([b for pair in zip(left, right) for b in pair], np.float32)
|
| 31 |
+
# Geometry alone cannot see it (a table is read the same way) ...
|
| 32 |
+
assert order_backtracks(boxes) == 0
|
| 33 |
+
# ... the order model can: it puts the left column first.
|
| 34 |
+
col = np.array([0, 1] * 20)
|
| 35 |
+
row = np.repeat(np.arange(20), 2)
|
| 36 |
+
key = col * 100 + row # true reading position
|
| 37 |
+
logits = np.sign(key[None, :] - key[:, None]).astype(np.float32) * 4.0
|
| 38 |
+
assert not pdf_order_coherent(boxes, logits)
|
| 39 |
+
good = np.argsort(key)
|
| 40 |
+
assert pdf_order_coherent(boxes[good], logits[np.ix_(good, good)])
|
| 41 |
+
|
| 42 |
+
|
| 43 |
+
def test_lines_written_bottom_up_are_not_coherent():
|
| 44 |
+
boxes = np.array(
|
| 45 |
+
[_box(50, 400 - 15 * i, 500, 410 - 15 * i) for i in range(20)], np.float32
|
| 46 |
+
)
|
| 47 |
+
assert not pdf_order_coherent(boxes)
|
| 48 |
+
|
| 49 |
+
|
| 50 |
+
def _chars(text, x0=0.0, w=5.0, gaps=None):
|
| 51 |
+
out, x = [], x0
|
| 52 |
+
for i, ch in enumerate(text):
|
| 53 |
+
x += (gaps or {}).get(i, 0.0)
|
| 54 |
+
out.append({"char": ch, "bbox": [x, 0.0, x + w, 10.0]})
|
| 55 |
+
x += w
|
| 56 |
+
return out
|
| 57 |
+
|
| 58 |
+
|
| 59 |
+
def test_cli_source_flags():
|
| 60 |
+
import argparse
|
| 61 |
+
|
| 62 |
+
from indra_ocr.cli import add_source_args, source_options
|
| 63 |
+
|
| 64 |
+
ap = argparse.ArgumentParser()
|
| 65 |
+
add_source_args(ap)
|
| 66 |
+
assert source_options(ap.parse_args([])) == {
|
| 67 |
+
"mode": "auto",
|
| 68 |
+
"pdf_order": True,
|
| 69 |
+
"image_ocr": True,
|
| 70 |
+
}
|
| 71 |
+
assert source_options(ap.parse_args(["--mode", "ocr"]))["mode"] == "ocr"
|
| 72 |
+
o = source_options(ap.parse_args(["--model-order", "--no-image-ocr"]))
|
| 73 |
+
assert o["pdf_order"] is False and o["image_ocr"] is False
|
| 74 |
+
|
| 75 |
+
|
| 76 |
+
def test_runs_go_to_the_smallest_containing_line_in_x_order():
|
| 77 |
+
from indra_ocr.document import assign_runs
|
| 78 |
+
from indra_ocr.textlayer import TextLine
|
| 79 |
+
|
| 80 |
+
def run(text, x0, y0, y1, w=4.0):
|
| 81 |
+
chars = [
|
| 82 |
+
(c, (x0 + i * w, y0, x0 + (i + 1) * w, y1)) for i, c in enumerate(text)
|
| 83 |
+
]
|
| 84 |
+
return TextLine((x0, y0, x0 + w * len(text), y1), text, chars)
|
| 85 |
+
|
| 86 |
+
boxes = np.array([[0, 0, 100, 20], [70, 2, 95, 18], [0, 30, 100, 50]], np.float32)
|
| 87 |
+
runs = [
|
| 88 |
+
run("ab c", 10, 5, 15),
|
| 89 |
+
run("\u03b1", 2, 3, 13),
|
| 90 |
+
run("xy", 75, 5, 15),
|
| 91 |
+
run("far", 500, 5, 15),
|
| 92 |
+
]
|
| 93 |
+
toks, lost = assign_runs(runs, boxes, 1.0)
|
| 94 |
+
assert "".join(t[0] for t in toks[0]) == "\u03b1 ab c" # runs by x, own order kept
|
| 95 |
+
assert [t[2] for t in toks[0]][0] == 1 # content index of the run
|
| 96 |
+
assert "".join(t[0] for t in toks[1]) == "xy" # inside both boxes: the smaller wins
|
| 97 |
+
assert toks[2] == [] and lost == 3
|
| 98 |
+
|
| 99 |
+
|
| 100 |
+
def test_a_run_across_two_cells_splits_between_them():
|
| 101 |
+
from indra_ocr.document import assign_runs
|
| 102 |
+
from indra_ocr.textlayer import TextLine
|
| 103 |
+
|
| 104 |
+
text = "12 34"
|
| 105 |
+
chars = [(c, (i * 10.0, 0.0, i * 10.0 + 8, 10.0)) for i, c in enumerate(text)]
|
| 106 |
+
run = TextLine((0, 0, 48, 10), text, chars)
|
| 107 |
+
boxes = np.array([[0, 0, 19, 10], [29, 0, 50, 10]], np.float32) # two cells
|
| 108 |
+
toks, lost = assign_runs([run], boxes, 1.0)
|
| 109 |
+
assert ["".join(t[0] for t in tk).strip() for tk in toks] == [
|
| 110 |
+
"12",
|
| 111 |
+
"34",
|
| 112 |
+
] and lost == 0
|
| 113 |
+
|
| 114 |
+
|
| 115 |
+
def test_runs_of_a_right_to_left_line_go_right_to_left():
|
| 116 |
+
from indra_ocr.document import assign_runs
|
| 117 |
+
from indra_ocr.textlayer import TextLine
|
| 118 |
+
|
| 119 |
+
def run(text, x0): # logical order, drawn right to left from x0
|
| 120 |
+
chars = [
|
| 121 |
+
(c, (x0 - (i + 1) * 4.0, 5, x0 - i * 4.0, 15)) for i, c in enumerate(text)
|
| 122 |
+
]
|
| 123 |
+
return TextLine((x0 - 4.0 * len(text), 5, x0, 15), text, chars)
|
| 124 |
+
|
| 125 |
+
boxes = np.array([[0, 0, 100, 20]], np.float32)
|
| 126 |
+
# two table cells on one row: "שלום" on the right, "עולם" on the left
|
| 127 |
+
toks, _ = assign_runs([run("עולם", 30), run("שלום", 90)], boxes, 1.0)
|
| 128 |
+
assert "".join(t[0] for t in toks[0]) == "שלום עולם"
|
| 129 |
+
|
| 130 |
+
|
| 131 |
+
def test_ocr_only_reads_scripts_in_its_vocab():
|
| 132 |
+
from types import SimpleNamespace
|
| 133 |
+
|
| 134 |
+
from indra_ocr.document import readable
|
| 135 |
+
|
| 136 |
+
ocr = SimpleNamespace(
|
| 137 |
+
vocab=SimpleNamespace(index={c: 1 for c in "abcdefghijklmnopqrstuvwxyz"}),
|
| 138 |
+
vocab_cjk=None,
|
| 139 |
+
)
|
| 140 |
+
line = lambda t: SimpleNamespace(text=t) # noqa: E731
|
| 141 |
+
assert readable(ocr, [line("hello world 123")])
|
| 142 |
+
assert not readable(ocr, [line("مرحبا بالعالم"), line("ok")])
|
| 143 |
+
assert readable(ocr, [line("1234 ...")]) # no letters: nothing to rule out
|
| 144 |
+
assert not readable(ocr, [line("中文文本")])
|
| 145 |
+
ocr.vocab_cjk = SimpleNamespace(index={c: 1 for c in "中文文本"})
|
| 146 |
+
assert readable(ocr, [line("中文文本")])
|
| 147 |
+
|
| 148 |
+
|
| 149 |
+
def test_lines_with_hanzi_go_to_the_cjk_recognizer():
|
| 150 |
+
from indra_ocr.pipeline import OrtOCR
|
| 151 |
+
from indra_ocr.vocab import Vocab
|
| 152 |
+
|
| 153 |
+
class Rec: # emits one fixed token id per line, at every other timestep
|
| 154 |
+
def __init__(self, tok):
|
| 155 |
+
self.tok = tok
|
| 156 |
+
|
| 157 |
+
def run(self, _, feed):
|
| 158 |
+
b, w = feed["x"].shape[0], feed["x"].shape[-1] // 4
|
| 159 |
+
best = np.zeros((b, w), np.int64)
|
| 160 |
+
best[:, ::2] = self.tok
|
| 161 |
+
return best, np.ones((b, w), np.float32), np.full(b, w)
|
| 162 |
+
|
| 163 |
+
class Script: # lines wider than 300 px "hold hanzi"
|
| 164 |
+
def run(self, _, feed):
|
| 165 |
+
return [(feed["widths"] > 300).astype(np.float32)]
|
| 166 |
+
|
| 167 |
+
ocr = OrtOCR.__new__(OrtOCR)
|
| 168 |
+
ocr.vocab, ocr.vocab_cjk = Vocab(list("ab")), Vocab(list("中b"))
|
| 169 |
+
ocr.rec, ocr.rec_cjk, ocr.script, ocr.mask, ocr.height = (
|
| 170 |
+
Rec(1),
|
| 171 |
+
Rec(1),
|
| 172 |
+
Script(),
|
| 173 |
+
None,
|
| 174 |
+
32,
|
| 175 |
+
)
|
| 176 |
+
gray = np.full((200, 800), 255, np.uint8)
|
| 177 |
+
boxes = np.array([[10, 10, 100, 42, 1], [10, 100, 700, 132, 1]], np.float32)
|
| 178 |
+
(t0, _, k0), (t1, _, k1) = ocr.recognize(gray, boxes)
|
| 179 |
+
assert set(t0) == {"a"} and isinstance(k0[0][0], int) # main recognizer: ids
|
| 180 |
+
assert set(t1) == {"中"} and k1[0][0] == "中" # CJK recognizer: characters
|
| 181 |
+
assert ocr._decode(k1) == t1
|
| 182 |
+
|
| 183 |
+
|
| 184 |
+
def test_a_turned_page_is_turned_upright_before_detection():
|
| 185 |
+
from indra_ocr.pipeline import OrtOCR
|
| 186 |
+
|
| 187 |
+
class Orient: # says: content turned one quarter turn counter-clockwise
|
| 188 |
+
def run(self, _, feed):
|
| 189 |
+
assert feed["x"].shape == (1, 1, 768, 768)
|
| 190 |
+
return [np.array([[0.01, 0.97, 0.01, 0.01]], np.float32)]
|
| 191 |
+
|
| 192 |
+
seen = {}
|
| 193 |
+
ocr = OrtOCR.__new__(OrtOCR)
|
| 194 |
+
ocr.orient = Orient()
|
| 195 |
+
ocr.detect = lambda g: (
|
| 196 |
+
seen.setdefault("shape", g.shape) and np.zeros((0, 5), np.float32)
|
| 197 |
+
)
|
| 198 |
+
ocr.recognize = lambda g, b: []
|
| 199 |
+
ocr.assemble = lambda wh, b, r, t: __import__("indra_ocr.pipeline").pipeline.Page(
|
| 200 |
+
[], t
|
| 201 |
+
)
|
| 202 |
+
page = ocr(np.full((300, 500), 255, np.uint8)) # wide: a turned portrait page
|
| 203 |
+
assert seen["shape"] == (500, 300) and page.rotation == 90
|
| 204 |
+
ocr.orient = None
|
| 205 |
+
assert ocr.orientation(np.full((300, 500), 255, np.uint8)) == 0
|
| 206 |
+
|
| 207 |
+
|
| 208 |
+
def test_fast_average_linkage_matches_the_brute_force():
|
| 209 |
+
from indra_ocr.order_post import _cluster
|
| 210 |
+
|
| 211 |
+
def brute(logits):
|
| 212 |
+
n = logits.shape[0]
|
| 213 |
+
S = logits.astype(np.float64).copy()
|
| 214 |
+
np.fill_diagonal(S, 0.0)
|
| 215 |
+
size, alive, members = np.ones(n), np.ones(n, bool), [[i] for i in range(n)]
|
| 216 |
+
for _ in range(n - 1):
|
| 217 |
+
mean = S / np.outer(size, size)
|
| 218 |
+
mean[~alive] = -np.inf
|
| 219 |
+
mean[:, ~alive] = -np.inf
|
| 220 |
+
np.fill_diagonal(mean, -np.inf)
|
| 221 |
+
a, b = np.unravel_index(int(np.argmax(mean)), mean.shape)
|
| 222 |
+
if mean[a, b] <= 0:
|
| 223 |
+
break
|
| 224 |
+
S[a] += S[b]
|
| 225 |
+
S[:, a] += S[:, b]
|
| 226 |
+
S[a, a] = 0.0
|
| 227 |
+
size[a] += size[b]
|
| 228 |
+
alive[b] = False
|
| 229 |
+
members[a] += members[b]
|
| 230 |
+
return sorted(tuple(sorted(members[i])) for i in range(n) if alive[i])
|
| 231 |
+
|
| 232 |
+
rng = np.random.default_rng(1)
|
| 233 |
+
for _ in range(100):
|
| 234 |
+
n = int(rng.integers(1, 40))
|
| 235 |
+
g = rng.integers(0, max(1, n // 4), n)
|
| 236 |
+
L = np.where(g[:, None] == g[None], 2.0, -2.0) + rng.normal(0, 1.5, (n, n))
|
| 237 |
+
L = (L + L.T) / 2
|
| 238 |
+
assert sorted(tuple(c) for c in _cluster(L)) == brute(L)
|
| 239 |
+
|
| 240 |
+
|
| 241 |
+
def test_skew_estimate_recovers_a_tilt():
|
| 242 |
+
from indra_ocr.pipeline import estimate_skew, rotate_page
|
| 243 |
+
|
| 244 |
+
rng = np.random.default_rng(0)
|
| 245 |
+
page = np.full((2200, 1700), 255, np.uint8)
|
| 246 |
+
for y in range(150, 2050, 45): # text-like rows of ink blobs
|
| 247 |
+
x = 150
|
| 248 |
+
while x < 1550:
|
| 249 |
+
w = int(rng.integers(20, 90))
|
| 250 |
+
page[y : y + 18, x : x + w] = 30
|
| 251 |
+
x += w + int(rng.integers(10, 25))
|
| 252 |
+
assert abs(estimate_skew(page)) < 0.15
|
| 253 |
+
for tilt in (2.0, -1.3, 0.6):
|
| 254 |
+
est = estimate_skew(rotate_page(page, tilt))
|
| 255 |
+
assert abs(est + tilt) <= 0.15, (tilt, est) # the estimate undoes the tilt
|
| 256 |
+
assert estimate_skew(np.full((800, 600), 255, np.uint8)) == 0.0
|
| 257 |
+
|
| 258 |
+
|
| 259 |
+
def test_cjk_lines_join_without_a_space_and_bare_bullets_start_items():
|
| 260 |
+
from indra_ocr.layout_post import _ITEM, join_lines
|
| 261 |
+
|
| 262 |
+
assert (
|
| 263 |
+
join_lines(["或其中的", "任何部分,在 2020 年"])
|
| 264 |
+
== "或其中的任何部分,在 2020 年"
|
| 265 |
+
)
|
| 266 |
+
assert join_lines(["説明します。", "次に"]) == "説明します。次に"
|
| 267 |
+
assert (
|
| 268 |
+
join_lines(["한국어 문장", "다음 줄"]) == "한국어 문장 다음 줄"
|
| 269 |
+
) # Korean keeps the space
|
| 270 |
+
assert join_lines(["tumori-", "genesis is"]) == "tumorigenesis is"
|
| 271 |
+
assert join_lines(["first line", "second"]) == "first line second"
|
| 272 |
+
assert _ITEM.match("•") and _ITEM.match("• text") and _ITEM.match("12. text")
|
| 273 |
+
assert not _ITEM.match("•text") and not _ITEM.match("12.5 kg")
|
| 274 |
+
|
| 275 |
+
|
| 276 |
+
def test_unrotate_box_maps_a_deskewed_box_back():
|
| 277 |
+
from indra_ocr.pipeline import rotate_page, unrotate_box
|
| 278 |
+
|
| 279 |
+
page = np.full((400, 600), 255, np.uint8)
|
| 280 |
+
page[180:220, 100:500] = 0 # a long dark bar
|
| 281 |
+
tilted = rotate_page(page, -3.0)
|
| 282 |
+
ys, xs = np.nonzero(tilted < 128)
|
| 283 |
+
got = unrotate_box([100, 180, 500, 220], 3.0, (600, 400))
|
| 284 |
+
# the mapped box covers the tilted bar in the input page
|
| 285 |
+
assert got[0] <= xs.min() + 2 and got[2] >= xs.max() - 2
|
| 286 |
+
assert got[1] <= ys.min() + 2 and got[3] >= ys.max() - 2
|
| 287 |
+
assert unrotate_box([10, 10, 20, 20], 0.0, (600, 400)) == [10, 10, 20, 20]
|
| 288 |
+
|
| 289 |
+
|
| 290 |
+
def test_math_tokens_back_to_text():
|
| 291 |
+
from indra_ocr.math import detokenize
|
| 292 |
+
|
| 293 |
+
toks = (
|
| 294 |
+
list("we have ")
|
| 295 |
+
+ ["$", "x", "_", "{", "i", "}", "\\leq", "1", "$"]
|
| 296 |
+
+ list(" here")
|
| 297 |
+
)
|
| 298 |
+
assert detokenize(toks) == "we have $x _ { i } \\leq 1$ here"
|
| 299 |
+
assert detokenize(["$", "\\alpha"]) == "$\\alpha$" # an open span closes
|
| 300 |
+
assert detokenize([" the", " bound"]) == "the bound" # word tokens keep their space
|
| 301 |
+
|
| 302 |
+
|
| 303 |
+
def test_math_models_are_optional(tmp_path):
|
| 304 |
+
from indra_ocr.math import MathReader
|
| 305 |
+
|
| 306 |
+
(tmp_path / "math_enc.onnx").write_bytes(b"") # only some of the files
|
| 307 |
+
assert MathReader.load(tmp_path, None, None) is None
|
OCR_INDRA/logs/indra_crop_fp32_6000_20261023.log
ADDED
|
@@ -0,0 +1,16 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
nohup: ignoring input
|
| 2 |
+
init <- /home/sakshyam/ocr/OCR_INDRA/models/indra-ocr-v3.1.1/model.safetensors; device=cuda
|
| 3 |
+
train lines: 125017 skipped: {'oov_or_empty': 0, 'target_too_long': 0, 'error': 0}
|
| 4 |
+
step 500: val CER 0.0220
|
| 5 |
+
step 1000: val CER 0.0188
|
| 6 |
+
step 1500: val CER 0.0206
|
| 7 |
+
step 2000: val CER 0.0170
|
| 8 |
+
step 2500: val CER 0.0176
|
| 9 |
+
step 3000: val CER 0.0169
|
| 10 |
+
step 3500: val CER 0.0170
|
| 11 |
+
step 4000: val CER 0.0166
|
| 12 |
+
step 4500: val CER 0.0162
|
| 13 |
+
step 5000: val CER 0.0165
|
| 14 |
+
step 5500: val CER 0.0163
|
| 15 |
+
step 6000: val CER 0.0162
|
| 16 |
+
done: 6000 steps, best val CER 0.0162; log=runs/indra_crop_fp32_6000_20261023/train_log.jsonl
|
OCR_INDRA/logs/investigation.md
ADDED
|
@@ -0,0 +1,168 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
# Indra investigation log
|
| 2 |
+
|
| 3 |
+
All work in this directory. Remote machine `khalti-t4` (202.51.0.7) stayed
|
| 4 |
+
inaccessible the whole session: SSH key `khalti-t4-ocr-team` was offered and
|
| 5 |
+
the server answered `Permission denied (publickey)` — so nothing below touched
|
| 6 |
+
that machine, its datasets, or its eval recipes.
|
| 7 |
+
|
| 8 |
+
## Phase 1 — source audit (local)
|
| 9 |
+
|
| 10 |
+
- Cloned/used repo: `indra/` — `datalab-to/indra`, commit `3d773ae` (v0.3.2).
|
| 11 |
+
- Read all 17 files of `indra/indra_ocr/` (~3,850 lines): `pipeline.py`,
|
| 12 |
+
`vocab.py`, `rec_preproc.py`, `det_post.py`, `models.py`, `line_mask.py`,
|
| 13 |
+
`order_post.py`, `layout_post.py`, `document.py`, `textlayer.py`,
|
| 14 |
+
`math.py`, `cli.py`, `parallel.py`, `__init__.py`, `viewer.py`,
|
| 15 |
+
`tests/`, `bench/`, CI workflow.
|
| 16 |
+
- Grep verdict: **no training code anywhere** (only docstrings that name a
|
| 17 |
+
closed training repo: `det_model.py`, `order_model.py`, `rec_preproc.py:5–10`,
|
| 18 |
+
dataloader comments). `pyproject.toml` has no torch dependency; CI publishes a
|
| 19 |
+
PyPI wheel only.
|
| 20 |
+
|
| 21 |
+
## Phase 2 — model artifacts (local)
|
| 22 |
+
|
| 23 |
+
- Downloaded `datalab-to/indra-ocr` @ tag `v3.1.1` → `models/indra-ocr-v3.1.1/`
|
| 24 |
+
(16 files, ~46 MB: det/rec/rec_cjk/order/orient/script + math×4 ONNX,
|
| 25 |
+
meta.json, config.json, model.safetensors, LICENSE (OpenRAIL-M), README).
|
| 26 |
+
File set matches `main` (checked via Hub API).
|
| 27 |
+
- Environment: deps into `/tmp/opencode/indra-lib` via
|
| 28 |
+
`pip install --target` (numpy, opencv-python-headless, onnxruntime 1.31.0,
|
| 29 |
+
onnx, pypdfium2, huggingface_hub, pillow 13 (with raqm), safetensors).
|
| 30 |
+
|
| 31 |
+
### Verified facts (evidence scripts all in `tools/indra_eval/`)
|
| 32 |
+
|
| 33 |
+
- `inspect_vocab.py` — `meta.json` vocab == `Vocab.multi()` = 1,677 tokens,
|
| 34 |
+
blank class 0, head 1,678; **Devanagari 128/128 assigned codepoints**
|
| 35 |
+
(U+0900–U+097F, incl. ॐ । ॥ ०–९ ़ ₹), Bengali block (96) also present.
|
| 36 |
+
- `inspect_models.py` — I/O of all 5 ONNX graphs + param audit:
|
| 37 |
+
rec.onnx in `x [B,1,32,W] f32, widths [B] int64` → out
|
| 38 |
+
**`ids [B,T] int64, conf [B,T] f32, lengths [B] int64`**;
|
| 39 |
+
det `x [1,1,H,W] → [N,3,h,w]`; orient `x [B,1,768,768] → p [B,4]`;
|
| 40 |
+
order `feats [1,N,16], mask [1,N] → a [1,N,N], cls [1,N,10],
|
| 41 |
+
same [1,N,N], tab [1,3,N,N]`; script `x [B,1,32,W], widths → p [B]`.
|
| 42 |
+
`model.safetensors`: 108 F32 tensors, all `rec/`-prefixed, **1,236,366
|
| 43 |
+
params** — exactly equals rec.onnx initializers; depthwise convs stored as
|
| 44 |
+
`rec/onnx::Conv_369/…/397` pairs ([128,1,9] w + [128] b), alongside clean
|
| 45 |
+
PyTorch-style names for everything else (`rec/stem.*`, `rec/stem_bn.*`,
|
| 46 |
+
`rec/blocks.<i>.{dw?,expand,se,project}.*`, `rec.head.*`).
|
| 47 |
+
- `inspect_rec_nodes.py` — 201-node dump + numeric run (`x [2,1,32,128]`,
|
| 48 |
+
`widths [128,64]`): 52 Conv, 11 HardSwish, 10 Relu, 10 Sigmoid (SE gates),
|
| 49 |
+
32 Mul, 12 ReduceSum, 1 BatchNormalization (stem only), 1 ArgMax; **no
|
| 50 |
+
RNN/attention anywhere**. `T=32` for `W=128`; `lengths=[32,16]`.
|
| 51 |
+
- `inspect_rec_mask.py` (closes decoding of the model) —
|
| 52 |
+
- `lengths = floor((widths+3)/4) = ceil(widths/4)`, clipped to `W/4`
|
| 53 |
+
(Clip max = Shape(x)[2]/4... the feature length). Numeric table:
|
| 54 |
+
widths 1,2,3,4→1; 5→2; 9→3; 65→17; 129→32 (clip); 200→32.
|
| 55 |
+
*Supersedes the earlier note “ceil((widths−1)/4)” and the
|
| 56 |
+
`WIDTH_MULTIPLE=8 "timestep stride"` comment — the true timestep stride
|
| 57 |
+
is 4.*
|
| 58 |
+
- `Cast_4 = (arange(W/4) < lengths).float()` `[B,1,W']`, a **dynamic
|
| 59 |
+
padding mask** applied (i) post-stem, (ii) after every block's residual
|
| 60 |
+
add, (iii) inside every SE pool as the normalizer
|
| 61 |
+
(`ReduceSum(HW×mask)/max(ReduceSum(mask),1)` → channel means over valid
|
| 62 |
+
steps only).
|
| 63 |
+
- Proof run: ink across all of `W=128` with `widths=64` → final-trunk
|
| 64 |
+
features ≥ t=16 are exactly 0.0 (masked) vs |f|=15.6 unmasked; head on the
|
| 65 |
+
zero vector → blank, conf ≈ 0.78 → **callers must truncate by lengths**
|
| 66 |
+
(pipeline.py:297 does).
|
| 67 |
+
|
| 68 |
+
## Phase 3 — runnable harness (local)
|
| 69 |
+
|
| 70 |
+
- `tools/indra_eval/make_synth_page.py` — renders a 1,240×1,754 synthetic
|
| 71 |
+
Nepali page (Pillow 13 + fonttools-free raqm verified: `features.check("raqm")
|
| 72 |
+
→ True`) using the OFL Noto Sans Devanagari variable font
|
| 73 |
+
(`tools/indra_eval/testdata/NotoSansDevanagari.ttf`, downloaded from
|
| 74 |
+
google/fonts); writes png + GT boxes jsonl + GT record jsonl.
|
| 75 |
+
- `tools/indra_eval/eval_indra.py` — `predict` (full pipeline or GT-boxes
|
| 76 |
+
rec-only, normalized JSON record) + `cer` (NFC-normalized, codepoint
|
| 77 |
+
Levenshtein micro/macro CER + word WER + per-line detail).
|
| 78 |
+
|
| 79 |
+
### Smoke results (`results/smoke/`)
|
| 80 |
+
|
| 81 |
+
| run | CER micro | WER | notes |
|
| 82 |
+
|---|---|---|---|
|
| 83 |
+
| rec-only (GT boxes) | 0.0349 | 0.158 | mean_conf 0.962 |
|
| 84 |
+
| full pipeline (det+rec+order) | 0.0262 | 0.132 | 10/10 lines found, mean_conf 0.963 |
|
| 85 |
+
|
| 86 |
+
Zero-shot observations (small 32px print): nukta `ऱ` → `र` dropped;
|
| 87 |
+
`ॐ` → `ँ`/`S` (misread as its glyph lookalike); `₹` → `र`;
|
| 88 |
+
`सङ्केत` → `सङ्गेत`/`सड्लेत`; `ऋतु` → `तऋहतु` in rec-only. Rec outputs are
|
| 89 |
+
sensitive to crop y-offset (textbbox vs det crop flips borderline glyphs),
|
| 90 |
+
which anchors the identical-crop evaluation design.
|
| 91 |
+
|
| 92 |
+
Environment caveat: timings on this WSL box are meaningless (det took
|
| 93 |
+
≈18.6 min on one page — starved CPU/container I/O; /mnt/c is slow). Never
|
| 94 |
+
bench here; bench on the T4.
|
| 95 |
+
|
| 96 |
+
## Phase 4 — docs (local)
|
| 97 |
+
|
| 98 |
+
- `docs/INDRA_INVESTIGATION.md` — architecture + evidence, component
|
| 99 |
+
trainability map, [U] closure table, licensing (weights: AI PUBS Open
|
| 100 |
+
RAIL-M *modified* — fine-tuning explicitly licensed §II ¶36; code:
|
| 101 |
+
Apache-2.0).
|
| 102 |
+
- `docs/INDRA_EXPERIMENTS.md` — Experiments A–E, leakage review, licensing
|
| 103 |
+
matrix, deployment comparison, decision tree.
|
| 104 |
+
|
| 105 |
+
- `tools/indra_eval/make_docx.py` — generates the complete human-readable
|
| 106 |
+
report `INDRA_OCR_NEPALI_COMPLETE_REPORT.docx` (63 KiB: cover + executive
|
| 107 |
+
summary + Parts II–VI = the three MD docs in full, auto-generated
|
| 108 |
+
per-line smoke tables from the result files, artifact inventory,
|
| 109 |
+
reproduce commands, dependency versions, licensing table).
|
| 110 |
+
- Unicode nuance discovered while validating the DOCX: **NFC composes
|
| 111 |
+
र + ़ (U+0930 U+093C) into ऱ (U+0931, DEVANAGARI LETTER RRA)** — so every
|
| 112 |
+
NFC-normalized side (GT, Indra's `vocab.canonical`, our CER scorer)
|
| 113 |
+
represents the nukta form of र as the single codepoint ऱ. The smoke
|
| 114 |
+
"nukta drop" therefore scores as ऱ→र, a 1-edit gap either way; the vocab
|
| 115 |
+
contains ऱ (it is inside U+0900–U+097F), so no new token is needed — the
|
| 116 |
+
recognizer simply needs to learn to emit it (distribution gap, Experiment B).
|
| 117 |
+
|
| 118 |
+
## Phase 5 — fine-tune toolkit + experiment bundle (2026-10-20)
|
| 119 |
+
|
| 120 |
+
Goal: make `indra_nepali_exp.bundle` remotely runnable and prove the whole
|
| 121 |
+
Experiment A/B toolchain offline before any dataset is touched.
|
| 122 |
+
|
| 123 |
+
- **Bundle integrity (verified):** `git bundle verify` OK, complete history,
|
| 124 |
+
clone reproduces `indra/` (v0.3.2), `models/indra-ocr-v3.1.1/` (16 artifacts,
|
| 125 |
+
byte-identical), `tools/`, `docs/`, `results/smoke/`; executable bits kept.
|
| 126 |
+
- **Torch re-implementation verified end-to-end** (`tools/exp/torch_rec.py`):
|
| 127 |
+
`model.safetensors` → `IndraRec` (1,236,366 tensors) → ONNX export vs shipped
|
| 128 |
+
`rec.onnx`: **0 id mismatches, 0 lengths mismatches, max |Δconf| 7.1e-6** on a
|
| 129 |
+
1,590-valid-step battery (gate 1e-3) → the fine-tune starting point is exactly
|
| 130 |
+
the shipped recognizer (arch + 108-tensor weight mapping confirmed).
|
| 131 |
+
- **Training chain verified** (`train_rec.py` → `export_rec_onnx.py` →
|
| 132 |
+
`exactness_check.py`): mechanics run overfits 64 synthetic noise lines
|
| 133 |
+
(loss 8.1 → 1.65, val CER 0.97 → 0.58); post-train gate (torch `.pt` vs its
|
| 134 |
+
own ONNX) passes (0 id mismatches, Δconf 1.3e-5); `run_rec.py` ONNX and
|
| 135 |
+
`--torch` runs produce identical texts; `best.safetensors` reloads with the
|
| 136 |
+
param assert. Experiment-A orchestrator (`run_a.sh`) and -B (`run_b.sh`)
|
| 137 |
+
complete on a local fake set.
|
| 138 |
+
- **Bugs found & fixed** (would each have broken the remote run):
|
| 139 |
+
`torch.onnx.export` on torch ≥ 2.6 defaults to the torch.export exporter,
|
| 140 |
+
which requires `onnxscript` → export forced to the verified legacy tracer
|
| 141 |
+
(`dynamo=False`, auto-fallback for older torch);
|
| 142 |
+
`run_a.sh`/`run_b.sh` crashed on unset `PYTHONPATH` under `set -u`;
|
| 143 |
+
`a_ppocr_baseline.py` only accepted the `"image"` field (smoke/GT files use
|
| 144 |
+
`"file"`) and did not resolve boxes-relative page paths;
|
| 145 |
+
`remote_setup.sh` wrote `probe.json` relative to the cwd;
|
| 146 |
+
`data_prep.py` hash-split could leave val empty on small corpora.
|
| 147 |
+
- **New gate:** `tools/exp/verify_bundle.sh` — one command, no data: param
|
| 148 |
+
invariant → export → exactness → smoke rec-lines CER (bit-identical 3.49 %).
|
| 149 |
+
`remote_setup.sh` now runs it automatically and stops on failure.
|
| 150 |
+
- Bundle rebuilt with all fixes; `docs/REMOTE_RUN_SHEET.md` §0 reduced to the
|
| 151 |
+
one-command gate + torch note.
|
| 152 |
+
|
| 153 |
+
## Blocked / open
|
| 154 |
+
|
| 155 |
+
1. **SSH to `khalti-t4` still denied (2026-10-20)** for key
|
| 156 |
+
`khalti-t4-ocr-team` (`Permission denied (publickey,password)`, port 22
|
| 157 |
+
open) — everything dataset-side: Experiment A on frozen sets, head-to-head
|
| 158 |
+
vs deployed PP-OCRv5 on identical crops, `docs/DATA_LICENSING.md` review,
|
| 159 |
+
dataset contract, int8/latency bench. Authorize the key or hand the bundle
|
| 160 |
+
to someone with access; commands are in `docs/REMOTE_RUN_SHEET.md`.
|
| 161 |
+
2. Fine-tune (Experiment B) is fully spec'd and toolchain-verified but awaits
|
| 162 |
+
the remote `ocr-deva-ft` data inventory (and permission for GPU work).
|
| 163 |
+
3. `model.safetensors` ships **rec weights only**; det/order adaptability is
|
| 164 |
+
architecture-possible but data/recipe/impracticality-blocked.
|
| 165 |
+
4. Any future re-export of the recognizer must reproduce: the dynamic mask,
|
| 166 |
+
per-block re-zeroing, SE valid-means, CTC decode tail
|
| 167 |
+
(`ids/conf/lengths`, `lengths = ceil(widths/4)` clip `W/4`) — pinned by
|
| 168 |
+
`verify_bundle.sh` + `exactness_check.py`.
|
OCR_INDRA/make_contact_sheet.py
ADDED
|
@@ -0,0 +1,31 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
import json, re
|
| 2 |
+
from pathlib import Path
|
| 3 |
+
from PIL import Image, ImageDraw
|
| 4 |
+
|
| 5 |
+
root = Path.home() / "ocr"
|
| 6 |
+
repo = root / "OCR_INDRA"
|
| 7 |
+
data = root / "ocr-deva-ft/data/chunks-v4/test"
|
| 8 |
+
preds = repo / "results/stratified_1500/predictions_experiment1_all_models.jsonl"
|
| 9 |
+
out = repo / "results/stratified_1500/error_contact_sheet.png"
|
| 10 |
+
|
| 11 |
+
rows = [json.loads(s) for s in preds.open()]
|
| 12 |
+
strip = lambda s: re.sub(r"\s+", "", s)
|
| 13 |
+
bad = [r for r in rows if r["model"] == "Indra-FT"
|
| 14 |
+
and not r["exact_match"]
|
| 15 |
+
and strip(r["ground_truth"]) != strip(r["prediction"])][:10]
|
| 16 |
+
|
| 17 |
+
w, h = 900, 165
|
| 18 |
+
sheet = Image.new("RGB", (w, h * len(bad)), "white")
|
| 19 |
+
draw = ImageDraw.Draw(sheet)
|
| 20 |
+
|
| 21 |
+
for i, r in enumerate(bad):
|
| 22 |
+
im = Image.open(data / r["image"]).convert("RGB")
|
| 23 |
+
im.thumbnail((880, 110))
|
| 24 |
+
y = i * h
|
| 25 |
+
sheet.paste(im, (10, y))
|
| 26 |
+
draw.text((10, y + 116), "REF: " + r["ground_truth"], fill="black")
|
| 27 |
+
draw.text((10, y + 138), "FT: " + r["prediction"], fill="blue")
|
| 28 |
+
|
| 29 |
+
sheet.save(out)
|
| 30 |
+
print("Saved:", out)
|
| 31 |
+
print("Crops:", len(bad))
|
OCR_INDRA/results/manifests_crops_20261023/dev.jsonl
ADDED
|
The diff for this file is too large to render.
See raw diff
|
|
|
OCR_INDRA/results/manifests_crops_20261023/dev_smoke.jsonl
ADDED
|
@@ -0,0 +1,64 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{"image": "dev/images/dev_0000000.png", "text": "वर्ष १२४ अङ्क ३१६"}
|
| 2 |
+
{"image": "dev/images/dev_0000001.png", "text": "२०८१ साल चैत १२ गते"}
|
| 3 |
+
{"image": "dev/images/dev_0000002.png", "text": "मङ्गलबार चैत्र कृष्ण एकादशी नेपाल संवत् ११४५"}
|
| 4 |
+
{"image": "dev/images/dev_0000003.png", "text": "चिल्लागा ११"}
|
| 5 |
+
{"image": "dev/images/dev_0000004.png", "text": "पृष्ठ : १२+४+४"}
|
| 6 |
+
{"image": "dev/images/dev_0000005.png", "text": "मूल्य : दस रुपियाँ"}
|
| 7 |
+
{"image": "dev/images/dev_0000006.png", "text": "वर्ष १२४ अङ्क ३१६ २०८१ साल चैत १२"}
|
| 8 |
+
{"image": "dev/images/dev_0000007.png", "text": "गते मङ्गलबार चैत्र कृष्ण"}
|
| 9 |
+
{"image": "dev/images/dev_0000008.png", "text": "एकादशी नेपाल संवत् ११४५ चिल्लागा ११"}
|
| 10 |
+
{"image": "dev/images/dev_0000009.png", "text": "पृष्ठ : १२+४+४"}
|
| 11 |
+
{"image": "dev/images/dev_0000010.png", "text": "मूल्य : दस रुपियाँ"}
|
| 12 |
+
{"image": "dev/images/dev_0000011.png", "text": "गणतन्त्रको"}
|
| 13 |
+
{"image": "dev/images/dev_0000012.png", "text": "विकल्प"}
|
| 14 |
+
{"image": "dev/images/dev_0000013.png", "text": "सोच्न सकिँदैन : राष्ट्रपति"}
|
| 15 |
+
{"image": "dev/images/dev_0000014.png", "text": "सोलुखुम्बु, चैत ११ गते"}
|
| 16 |
+
{"image": "dev/images/dev_0000015.png", "text": "राष्ट्रपति"}
|
| 17 |
+
{"image": "dev/images/dev_0000016.png", "text": "रामचन्द्र"}
|
| 18 |
+
{"image": "dev/images/dev_0000017.png", "text": "पौ डे लले सङ्घीय"}
|
| 19 |
+
{"image": "dev/images/dev_0000018.png", "text": "लोकतान्त्रिक गणतन्त्र नेपाली जनताले"}
|
| 20 |
+
{"image": "dev/images/dev_0000019.png", "text": "रगतसँग"}
|
| 21 |
+
{"image": "dev/images/dev_0000020.png", "text": "साटेको उपलब्धि भएको भन्दै यसको"}
|
| 22 |
+
{"image": "dev/images/dev_0000021.png", "text": "विकल्प सोच्न"}
|
| 23 |
+
{"image": "dev/images/dev_0000022.png", "text": "नसकिने बताउनुभएको छ ।"}
|
| 24 |
+
{"image": "dev/images/dev_0000023.png", "text": "सोलुखुम्बुको नेचासल्यान गाउँपालिकाको"}
|
| 25 |
+
{"image": "dev/images/dev_0000024.png", "text": "नवनिर्मित प्रशासकीय भवन"}
|
| 26 |
+
{"image": "dev/images/dev_0000025.png", "text": "उद्घाटन समारोहलाई"}
|
| 27 |
+
{"image": "dev/images/dev_0000026.png", "text": "गणतन्त्रमा हरेक"}
|
| 28 |
+
{"image": "dev/images/dev_0000027.png", "text": "नागरिक आफ्नो सार्वभौमसत्ताको आफैँ मालिक"}
|
| 29 |
+
{"image": "dev/images/dev_0000028.png", "text": "हुने"}
|
| 30 |
+
{"image": "dev/images/dev_0000029.png", "text": "व्यवस्था भएको उल्लेख गर्नुभयो ।"}
|
| 31 |
+
{"image": "dev/images/dev_0000030.png", "text": "उहाँले"}
|
| 32 |
+
{"image": "dev/images/dev_0000031.png", "text": "भन्नुभयो, “शासनव्यवस्थालाई सुदृढ"}
|
| 33 |
+
{"image": "dev/images/dev_0000032.png", "text": "बनाउन र"}
|
| 34 |
+
{"image": "dev/images/dev_0000033.png", "text": "यही व्यवस्थाबाट नेपाली जनताको भाग्य र भविष्य"}
|
| 35 |
+
{"image": "dev/images/dev_0000034.png", "text": "सुनिश्चित छ"}
|
| 36 |
+
{"image": "dev/images/dev_0000035.png", "text": "सरकार, राजनीतिक दललगायत हामी सबैको"}
|
| 37 |
+
{"image": "dev/images/dev_0000036.png", "text": "हो ।”"}
|
| 38 |
+
{"image": "dev/images/dev_0000037.png", "text": "स्वतन्त्र शासनप्रणाली जनताको चाहना भएको"}
|
| 39 |
+
{"image": "dev/images/dev_0000038.png", "text": "बताउँदै उहाँले नेपाली"}
|
| 40 |
+
{"image": "dev/images/dev_0000039.png", "text": "जनतालाई आफूले बनाएको"}
|
| 41 |
+
{"image": "dev/images/dev_0000040.png", "text": "संविधानबाहिर कुनै पनि निरङ्कुशता र अराजकता"}
|
| 42 |
+
{"image": "dev/images/dev_0000041.png", "text": "स्वीकार्य नहुने"}
|
| 43 |
+
{"image": "dev/images/dev_0000042.png", "text": "बताउनुभयो । “इतिहास साक्षी छ ।"}
|
| 44 |
+
{"image": "dev/images/dev_0000043.png", "text": "१०४ वर्षीय जहानियाँ"}
|
| 45 |
+
{"image": "dev/images/dev_0000044.png", "text": "राणाशासन"}
|
| 46 |
+
{"image": "dev/images/dev_0000045.png", "text": "पञ्चायत वा सक्रिय राजतन्त्र"}
|
| 47 |
+
{"image": "dev/images/dev_0000046.png", "text": "नेपाली जनताले कुनै"}
|
| 48 |
+
{"image": "dev/images/dev_0000047.png", "text": "पनि निरङ्कुशताका नाम र अवशेषलाई रुचाएनन्,”"}
|
| 49 |
+
{"image": "dev/images/dev_0000048.png", "text": "राष्ट्रपति पौडेलले भन्नुभयो, “सङ्घीय लोकतान्त्रिक"}
|
| 50 |
+
{"image": "dev/images/dev_0000049.png", "text": "गणतन्त्र हामीले"}
|
| 51 |
+
{"image": "dev/images/dev_0000050.png", "text": "त्यसै प्राप्त गरेका होइनौँ । यो"}
|
| 52 |
+
{"image": "dev/images/dev_0000051.png", "text": "हजारौँको त्याग,"}
|
| 53 |
+
{"image": "dev/images/dev_0000052.png", "text": "सङ्घर्ष र बलिदानबाट प्राप्त"}
|
| 54 |
+
{"image": "dev/images/dev_0000053.png", "text": "उपलब्धि हो ।”"}
|
| 55 |
+
{"image": "dev/images/dev_0000054.png", "text": "राष्ट्रपति पौडेलले आफू"}
|
| 56 |
+
{"image": "dev/images/dev_0000055.png", "text": "सभामुख हुँदा पैदल"}
|
| 57 |
+
{"image": "dev/images/dev_0000056.png", "text": "डाँडाकाँडा"}
|
| 58 |
+
{"image": "dev/images/dev_0000057.png", "text": "पुगेको र"}
|
| 59 |
+
{"image": "dev/images/dev_0000058.png", "text": "जनतासँग साक्षात्कार गरेको स्मरण"}
|
| 60 |
+
{"image": "dev/images/dev_0000059.png", "text": "गर्नुभयो । उहाँले त्यतिबेला आफू"}
|
| 61 |
+
{"image": "dev/images/dev_0000060.png", "text": "टिम्बुरबोटेको"}
|
| 62 |
+
{"image": "dev/images/dev_0000061.png", "text": "ओडारमा रुझ्दै र भिज्दै"}
|
| 63 |
+
{"image": "dev/images/dev_0000062.png", "text": "पुगेको बताउनुभयो ।"}
|
| 64 |
+
{"image": "dev/images/dev_0000063.png", "text": "टिम्बुबोटेको"}
|
OCR_INDRA/results/manifests_crops_20261023/train.jsonl
ADDED
|
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
version https://git-lfs.github.com/spec/v1
|
| 2 |
+
oid sha256:5a07aeae5a4ca1d75b28099cda22e6a98e564b8f7a8e556472b30cf1ea5d4a81
|
| 3 |
+
size 14757807
|
OCR_INDRA/results/padding_variants/predictions.jsonl
ADDED
|
The diff for this file is too large to render.
See raw diff
|
|
|
OCR_INDRA/results/probe.json
ADDED
|
@@ -0,0 +1,17 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{
|
| 2 |
+
"python": "3.10.12",
|
| 3 |
+
"numpy": "2.2.6",
|
| 4 |
+
"cv2": "5.0.0",
|
| 5 |
+
"pillow": "12.3.0",
|
| 6 |
+
"torch": "2.14.1+cu130",
|
| 7 |
+
"cuda": false,
|
| 8 |
+
"cuda_device": null,
|
| 9 |
+
"torch_amp_gradscaler": true,
|
| 10 |
+
"onnxruntime": "1.23.2",
|
| 11 |
+
"ort_providers": [
|
| 12 |
+
"AzureExecutionProvider",
|
| 13 |
+
"CPUExecutionProvider"
|
| 14 |
+
],
|
| 15 |
+
"onnx": "1.23.2",
|
| 16 |
+
"indra_vocab_classes": 1678
|
| 17 |
+
}
|
OCR_INDRA/results/smoke/cer_full.json
ADDED
|
@@ -0,0 +1,18 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{
|
| 2 |
+
"gt": "results/smoke/synth_nepali_record.jsonl",
|
| 3 |
+
"pred": "results/smoke/full_run.jsonl",
|
| 4 |
+
"cer_micro": 0.026200873362445413,
|
| 5 |
+
"cer_macro": 0.026200873362445413,
|
| 6 |
+
"wer_micro": 0.13157894736842105,
|
| 7 |
+
"chars": 229,
|
| 8 |
+
"words": 38,
|
| 9 |
+
"files": [
|
| 10 |
+
{
|
| 11 |
+
"image": "/mnt/c/Users/Dell/Desktop/OCR_INDRA/results/smoke/synth_nepali_page.png",
|
| 12 |
+
"cer": 0.026200873362445413,
|
| 13 |
+
"edits": 6,
|
| 14 |
+
"chars": 229,
|
| 15 |
+
"wer": 0.13157894736842105
|
| 16 |
+
}
|
| 17 |
+
]
|
| 18 |
+
}
|
OCR_INDRA/results/smoke/cer_rec_only.json
ADDED
|
@@ -0,0 +1,18 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{
|
| 2 |
+
"gt": "results/smoke/synth_nepali_record.jsonl",
|
| 3 |
+
"pred": "results/smoke/rec_only.jsonl",
|
| 4 |
+
"cer_micro": 0.034934497816593885,
|
| 5 |
+
"cer_macro": 0.034934497816593885,
|
| 6 |
+
"wer_micro": 0.15789473684210525,
|
| 7 |
+
"chars": 229,
|
| 8 |
+
"words": 38,
|
| 9 |
+
"files": [
|
| 10 |
+
{
|
| 11 |
+
"image": "/mnt/c/Users/Dell/Desktop/OCR_INDRA/results/smoke/synth_nepali_page.png",
|
| 12 |
+
"cer": 0.034934497816593885,
|
| 13 |
+
"edits": 8,
|
| 14 |
+
"chars": 229,
|
| 15 |
+
"wer": 0.15789473684210525
|
| 16 |
+
}
|
| 17 |
+
]
|
| 18 |
+
}
|