File size: 3,188 Bytes
f9bc0c1
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
"""Shared helpers for the experiment tools (matches tools/indra_eval/eval_indra.py conventions).

Used by: train_rec.py, run_rec.py, exactness_check.py, adapters.  Everything here
is deliberately dependency-light: numpy + one of cv2 / PIL / pypdfium2.

Imports the *cloned* `indra/` package (Apache-2.0) for Vocab + rec_preproc
so training and inference run through one code path — the whole reason
zero-shot and fine-tuned numbers are comparable (docs/INDRA_EXPERIMENTS.md §11).
"""
from __future__ import annotations

import json
import unicodedata
from pathlib import Path

REPO = Path(__file__).resolve().parents[2]
for p in (str(REPO / "indra"),):
    if Path(p).is_dir():
        import sys

        sys.path.insert(0, p)


def load_gray(path: Path, dpi: float = 200.0):
    """Grayscale page as uint8 ndarray; PDF first page only (matches eval_indra)."""
    import numpy as np

    if path.suffix.lower() == ".pdf":
        import pypdfium2 as pdfium

        pdf = pdfium.PdfDocument(str(path))
        img = pdf[0].render(scale=dpi / 72.0).to_pil()
        pdf.close()
        return np.asarray(img.convert("L"))
    import cv2

    img = cv2.imread(str(path), cv2.IMREAD_GRAYSCALE)
    if img is None:
        from PIL import Image

        img = np.asarray(Image.open(path).convert("L"))
    return img


def norm(s: str) -> str:
    return unicodedata.normalize("NFC", s)


def read_jsonl(path: Path | str) -> list[dict]:
    path = Path(path)
    return [json.loads(ln) for ln in path.read_text(encoding="utf-8").splitlines() if ln.strip()]


def write_jsonl(path: Path | str, records: list[dict]) -> None:
    path = Path(path)
    path.parent.mkdir(parents=True, exist_ok=True)
    path.write_text(
        "".join(json.dumps(r, ensure_ascii=False) + "\n" for r in records),
        encoding="utf-8",
    )


def append_jsonl(path: Path | str, record: dict) -> None:
    path = Path(path)
    path.parent.mkdir(parents=True, exist_ok=True)
    with open(path, "a", encoding="utf-8") as f:
        f.write(json.dumps(record, ensure_ascii=False) + "\n")


def lev(a: str, b: str) -> int:
    """Codepoint Levenshtein, two-row DP (identical to eval_indra.py's scorer)."""
    if a == b:
        return 0
    prev = list(range(len(b) + 1))
    for i, ca in enumerate(a, 1):
        cur = [i] + [0] * len(b)
        for j, cb in enumerate(b, 1):
            cur[j] = min(prev[j] + 1, cur[j - 1] + 1, prev[j - 1] + (ca != cb))
        prev = cur
    return prev[-1]


def vocab():
    """The recognizer's Vocab (1,677 tokens + blank 0).  NFC normalize is inside encode()."""
    from indra_ocr.vocab import Vocab

    return Vocab.multi()


def collapseIds(ids: list[int]) -> list[int]:
    """CTC greedy collapse: drop blank(0), skip repeats (pipeline.py:299 semantics)."""
    out: list[int] = []
    prev = -1
    for t in ids:
        if t != 0 and t != prev:
            out.append(int(t))
        prev = t
    return out


def mean_conf(pmax, lengths) -> list[tuple[float]]:
    """Per-line mean conf over VALID timesteps; None-safe for B=1 wrappers."""
    mins = [float(pmax[k, : int(n)].mean()) if int(n) > 0 else 0.0 for k, n in enumerate(lengths)]
    return mins