File size: 9,843 Bytes
dab7db2
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
"""Turn a lab report (PDF or photo) into structured results.

Pipeline:
  1. Get text lines: the PDF text layer when there is one, otherwise OCR (Tesseract).
  2. For every line, find "name  value  [H/L]  unit  range".
  3. Map the printed name to a catalog test (exact alias, then fuzzy).
  4. Snap OCR-mangled units to the closest unit the catalog knows for that test.

No LLM is involved here: extraction errors should be measurable and reproducible.
"""
from __future__ import annotations

import re
from pathlib import Path

from rapidfuzz import fuzz, process

from .catalog import load_catalog, match_test_name, normalize_unit
from .models import Result

NUM = r"[<>≤≥]?\s?\d+(?:[.,]\d+)?"
NUM_RE = re.compile(rf"^{NUM}$")
MARKERS = {"h", "l", "*", "hh", "ll", "↑", "↓", "+", "-"}


def _to_float(s: str) -> float:
    s = s.strip().lstrip("<>≤≥").strip().replace(",", ".")
    return float(s)


def _clean_ocr_line(line: str) -> str:
    line = line.replace("|", " ").replace("—", "-").replace("–", "-")
    # Common OCR confusions inside numbers: O->0 and l->1 when surrounded by digits.
    line = re.sub(r"(?<=\d)[Oo](?=[\d,.])|(?<=[\d,.])[Oo](?=\d|\b)", "0", line)
    line = re.sub(r"(?<=\d)\s+([.,])\s*(?=\d)", r"\1", line)  # "0 ,70" -> "0,70"
    return re.sub(r"\s+", " ", line).strip()


def _looks_like_unit(tok: str) -> bool:
    if re.match(r"^[\[(<>≤≥]", tok):
        return False
    if tok[0].isdigit():
        return "/" in tok or "^" in tok  # 10^3/µL, 10^9/L
    return bool(re.search(r"[A-Za-zµ%]", tok))


def parse_range(text: str) -> tuple[float | None, float | None]:
    raw = text.strip()
    # OCR often reads a closing "]" as "1": "[4.20 - 5.901" -> "[4.20 - 5.90"
    m = re.match(r"^[\[(].*?(\d+[.,](\d+))\s*-\s*(\d+[.,](\d+))$", raw)
    if m and m.group(3).endswith("1") and len(m.group(4)) > len(m.group(2)):
        raw = raw[:-1]
    t = raw.strip("()[]{} ").replace(",", ".")
    t = t.replace("–", "-").replace("—", "-").replace(" à ", "-").replace(" to ", "-")
    m = re.match(r"^(?:<|≤|<=|inf\.?\s*[aà]?)\s*(\d+(?:\.\d+)?)", t, re.I)
    if m:
        return None, float(m.group(1))
    m = re.match(r"^(?:>|≥|>=|sup\.?\s*[aà]?)\s*(\d+(?:\.\d+)?)", t, re.I)
    if m:
        return float(m.group(1)), None
    m = re.search(r"(\d+(?:\.\d+)?)\s*-\s*(\d+(?:\.\d+)?)", t)
    if m:
        lo, hi = float(m.group(1)), float(m.group(2))
        if lo <= hi:
            return lo, hi
    return None, None


# How units are usually printed; used to display snapped units nicely.
_PRETTY = {normalize_unit(u): u for u in [
    "g/dL", "g/L", "mg/dL", "mg/L", "mmol/L", "µmol/L", "mEq/L", "%", "fL", "U/L", "UI/L", "IU/L",
    "mIU/L", "µUI/mL", "µIU/mL", "ng/mL", "µg/L", "nmol/L", "10^3/µL", "10^6/µL",
    "10^9/L", "10^12/L", "/mm3", "L/L"]}
_PRETTY.update({"10^3/ul": "10^3/µL", "10^6/ul": "10^6/µL", "uui/ml": "µUI/mL", "umol/l": "µmol/L"})


def _ocr_fold_unit(u: str) -> str:
    u = normalize_unit(u)
    u = re.sub(r"^p(?=[uµ])", "µ", u)  # "pUI/mL" -> "µUI/mL"
    u = re.sub(r"10[%*°](?=\d)", "10^", u)  # "10%3/pL" -> "10^3/pL"
    u = re.sub(r"/[pu](?=l$)", "/µ", u)  # "/pL", "/uL" -> "/µL"
    return u.replace("w", "/").replace("|", "l").replace("1", "l")


def _snap_unit(test_key: str | None, unit: str | None) -> str | None:
    """Return the unit as a known spelling for this test, repairing small OCR errors."""
    if not unit or not test_key:
        return unit
    td = load_catalog()[test_key]
    known = list(td.units) + [td.canonical_unit]
    norm_known = [normalize_unit(k) for k in known]
    nu = normalize_unit(unit)
    if nu not in norm_known:
        best = process.extractOne(_ocr_fold_unit(unit), norm_known, scorer=fuzz.ratio)
        if not best or best[1] < 60:
            return unit  # leave it; flags will rely on the printed range only
        nu = best[0]
    if test_key in ("wbc", "platelets", "rbc") and nu in ("g/l", "t/l"):
        return nu.upper()  # giga/tera per litre cell counts, as French labs print them
    return _PRETTY.get(nu, known[norm_known.index(nu)])


def parse_line(line: str) -> Result | None:
    """Parse one report line. Returns None when the line has no test-like structure."""
    # Split markers glued to values: "5,36H" -> "5,36 H"
    line = re.sub(r"(?<=\d)([HhLl*])(?=[\s.,;:_]|$)", r" \1", line)
    line = line.replace("«", "<").replace("»", ">")
    tokens = [t.strip("_.,;:'\"`~") or t for t in line.split(" ")]
    tokens = [t for t in tokens if t and not re.fullmatch(r"[_.,;:'\"`~=]+", t)]
    for i in range(1, len(tokens)):
        tok = tokens[i]
        if not NUM_RE.match(tok) or tok[0] in "<>≤≥":
            continue
        key, score, name = None, 0.0, ""
        # The name is a prefix of the tokens before the value. OCR noise ("ae", "oo", "L")
        # can sit between name and value, so also try dropping short trailing tokens.
        for k in range(i, 0, -1):
            if k < i and len(tokens[k]) > 3:
                break
            cand = " ".join(tokens[:k]).strip(" :.")
            if not re.search(r"[A-Za-zÀ-ÿ؀-ۿ]", cand):
                continue
            ck, cs = match_test_name(cand)
            if ck and cs > score:
                key, score, name = ck, cs, cand
            if cs == 100.0:
                break
        if key is None:
            continue
        # OCR sometimes emits a stray short number before the real value:
        # "K 7 3.79 mmol/L" -> the value is the number directly followed by a unit.
        if (i + 2 < len(tokens) and NUM_RE.match(tokens[i + 1]) and "." not in tok and "," not in tok
                and len(tok) <= 2 and _looks_like_unit(tokens[i + 2])):
            i += 1
            tok = tokens[i]
        j = i + 1
        marker = None
        if j < len(tokens) and tokens[j].lower() in MARKERS:
            m = tokens[j].lower()
            marker = "H" if m in ("h", "hh", "↑") else "L" if m in ("l", "ll", "↓") else "*" if m == "*" else None
            j += 1
        unit = None
        if j < len(tokens) and _looks_like_unit(tokens[j]):
            unit = tokens[j]
            j += 1
        ref_low, ref_high = parse_range(" ".join(tokens[j:]))
        return Result(
            test_key=key, raw_name=name, value=_to_float(tok), unit=_snap_unit(key, unit),
            ref_low=ref_low, ref_high=ref_high,
            ref_source="report" if (ref_low is not None or ref_high is not None) else "none",
            match_score=score, source_line=line, report_marker=marker,
        )
    return None


def lines_from_pdf(path: Path) -> list[str]:
    import pdfplumber

    lines: list[str] = []
    with pdfplumber.open(str(path)) as pdf:
        for page in pdf.pages:
            lines.extend((page.extract_text() or "").splitlines())
    return lines


def preprocess_image(path: Path):
    """Grayscale, upscale, adaptive-threshold and strip table rules (incl. dotted lines).

    Dotted row separators are the main thing that breaks Tesseract on lab reports:
    without this step OCR on dotted layouts drops almost every row.
    """
    import cv2
    import numpy as np
    from PIL import Image, ImageOps

    pil = ImageOps.exif_transpose(Image.open(path)).convert("L")
    img = np.array(pil)
    if img.shape[1] < 2400:
        f = 2400 / img.shape[1]
        img = cv2.resize(img, None, fx=f, fy=f, interpolation=cv2.INTER_CUBIC)
    img = cv2.GaussianBlur(img, (3, 3), 0)
    bw = cv2.adaptiveThreshold(img, 255, cv2.ADAPTIVE_THRESH_GAUSSIAN_C, cv2.THRESH_BINARY, 31, 15)
    inv = 255 - bw
    closed = cv2.morphologyEx(inv, cv2.MORPH_CLOSE, cv2.getStructuringElement(cv2.MORPH_RECT, (9, 1)))
    rules = cv2.morphologyEx(closed, cv2.MORPH_OPEN, cv2.getStructuringElement(cv2.MORPH_RECT, (150, 1)))
    rules = cv2.dilate(rules, cv2.getStructuringElement(cv2.MORPH_RECT, (1, 3)))
    return 255 - cv2.bitwise_and(inv, 255 - rules)


def lines_from_image(path: Path, langs: str = "eng+fra") -> list[str]:
    import pytesseract

    text = pytesseract.image_to_string(preprocess_image(path), lang=langs, config="--psm 6")
    return text.splitlines()


def detect_language(lines: list[str]) -> str:
    text = " ".join(lines).lower()
    fr = sum(w in text for w in ["analyse", "résultat", "resultat", "prélèvement", "référence", "glycémie", "valeurs"])
    en = sum(w in text for w in ["result", "reference", "collection", "patient:", "test", "range"])
    return "fr" if fr > en else "en"


def extract(path: str | Path) -> tuple[list[Result], list[str], str, str]:
    """Returns (results, unmatched numeric lines, detected language, method)."""
    path = Path(path)
    method = "pdf-text"
    if path.suffix.lower() == ".pdf":
        lines = lines_from_pdf(path)
        if sum(len(line) for line in lines) < 40:  # scanned PDF without a text layer
            import subprocess
            import tempfile

            with tempfile.TemporaryDirectory() as td:
                subprocess.run(["pdftoppm", "-png", "-r", "300", str(path), f"{td}/p"], check=True)
                lines = []
                for png in sorted(Path(td).glob("p*.png")):
                    lines.extend(lines_from_image(png))
            method = "ocr"
    else:
        lines = lines_from_image(path)
        method = "ocr"

    results: list[Result] = []
    unmatched: list[str] = []
    seen: set[str] = set()
    for raw in lines:
        line = _clean_ocr_line(raw)
        if not line:
            continue
        r = parse_line(line)
        if r and r.test_key not in seen:
            seen.add(r.test_key)
            results.append(r)
        elif r is None and re.search(r"\d+[.,]\d+", line) and not re.search(r"\d{4}-\d{2}-\d{2}", line):
            unmatched.append(line)
    return results, unmatched, detect_language(lines), method