File size: 9,843 Bytes
dab7db2 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 151 152 153 154 155 156 157 158 159 160 161 162 163 164 165 166 167 168 169 170 171 172 173 174 175 176 177 178 179 180 181 182 183 184 185 186 187 188 189 190 191 192 193 194 195 196 197 198 199 200 201 202 203 204 205 206 207 208 209 210 211 212 213 214 215 216 217 218 219 220 221 222 223 224 225 226 227 228 229 230 231 232 233 234 235 236 237 238 | """Turn a lab report (PDF or photo) into structured results.
Pipeline:
1. Get text lines: the PDF text layer when there is one, otherwise OCR (Tesseract).
2. For every line, find "name value [H/L] unit range".
3. Map the printed name to a catalog test (exact alias, then fuzzy).
4. Snap OCR-mangled units to the closest unit the catalog knows for that test.
No LLM is involved here: extraction errors should be measurable and reproducible.
"""
from __future__ import annotations
import re
from pathlib import Path
from rapidfuzz import fuzz, process
from .catalog import load_catalog, match_test_name, normalize_unit
from .models import Result
NUM = r"[<>≤≥]?\s?\d+(?:[.,]\d+)?"
NUM_RE = re.compile(rf"^{NUM}$")
MARKERS = {"h", "l", "*", "hh", "ll", "↑", "↓", "+", "-"}
def _to_float(s: str) -> float:
s = s.strip().lstrip("<>≤≥").strip().replace(",", ".")
return float(s)
def _clean_ocr_line(line: str) -> str:
line = line.replace("|", " ").replace("—", "-").replace("–", "-")
# Common OCR confusions inside numbers: O->0 and l->1 when surrounded by digits.
line = re.sub(r"(?<=\d)[Oo](?=[\d,.])|(?<=[\d,.])[Oo](?=\d|\b)", "0", line)
line = re.sub(r"(?<=\d)\s+([.,])\s*(?=\d)", r"\1", line) # "0 ,70" -> "0,70"
return re.sub(r"\s+", " ", line).strip()
def _looks_like_unit(tok: str) -> bool:
if re.match(r"^[\[(<>≤≥]", tok):
return False
if tok[0].isdigit():
return "/" in tok or "^" in tok # 10^3/µL, 10^9/L
return bool(re.search(r"[A-Za-zµ%]", tok))
def parse_range(text: str) -> tuple[float | None, float | None]:
raw = text.strip()
# OCR often reads a closing "]" as "1": "[4.20 - 5.901" -> "[4.20 - 5.90"
m = re.match(r"^[\[(].*?(\d+[.,](\d+))\s*-\s*(\d+[.,](\d+))$", raw)
if m and m.group(3).endswith("1") and len(m.group(4)) > len(m.group(2)):
raw = raw[:-1]
t = raw.strip("()[]{} ").replace(",", ".")
t = t.replace("–", "-").replace("—", "-").replace(" à ", "-").replace(" to ", "-")
m = re.match(r"^(?:<|≤|<=|inf\.?\s*[aà]?)\s*(\d+(?:\.\d+)?)", t, re.I)
if m:
return None, float(m.group(1))
m = re.match(r"^(?:>|≥|>=|sup\.?\s*[aà]?)\s*(\d+(?:\.\d+)?)", t, re.I)
if m:
return float(m.group(1)), None
m = re.search(r"(\d+(?:\.\d+)?)\s*-\s*(\d+(?:\.\d+)?)", t)
if m:
lo, hi = float(m.group(1)), float(m.group(2))
if lo <= hi:
return lo, hi
return None, None
# How units are usually printed; used to display snapped units nicely.
_PRETTY = {normalize_unit(u): u for u in [
"g/dL", "g/L", "mg/dL", "mg/L", "mmol/L", "µmol/L", "mEq/L", "%", "fL", "U/L", "UI/L", "IU/L",
"mIU/L", "µUI/mL", "µIU/mL", "ng/mL", "µg/L", "nmol/L", "10^3/µL", "10^6/µL",
"10^9/L", "10^12/L", "/mm3", "L/L"]}
_PRETTY.update({"10^3/ul": "10^3/µL", "10^6/ul": "10^6/µL", "uui/ml": "µUI/mL", "umol/l": "µmol/L"})
def _ocr_fold_unit(u: str) -> str:
u = normalize_unit(u)
u = re.sub(r"^p(?=[uµ])", "µ", u) # "pUI/mL" -> "µUI/mL"
u = re.sub(r"10[%*°](?=\d)", "10^", u) # "10%3/pL" -> "10^3/pL"
u = re.sub(r"/[pu](?=l$)", "/µ", u) # "/pL", "/uL" -> "/µL"
return u.replace("w", "/").replace("|", "l").replace("1", "l")
def _snap_unit(test_key: str | None, unit: str | None) -> str | None:
"""Return the unit as a known spelling for this test, repairing small OCR errors."""
if not unit or not test_key:
return unit
td = load_catalog()[test_key]
known = list(td.units) + [td.canonical_unit]
norm_known = [normalize_unit(k) for k in known]
nu = normalize_unit(unit)
if nu not in norm_known:
best = process.extractOne(_ocr_fold_unit(unit), norm_known, scorer=fuzz.ratio)
if not best or best[1] < 60:
return unit # leave it; flags will rely on the printed range only
nu = best[0]
if test_key in ("wbc", "platelets", "rbc") and nu in ("g/l", "t/l"):
return nu.upper() # giga/tera per litre cell counts, as French labs print them
return _PRETTY.get(nu, known[norm_known.index(nu)])
def parse_line(line: str) -> Result | None:
"""Parse one report line. Returns None when the line has no test-like structure."""
# Split markers glued to values: "5,36H" -> "5,36 H"
line = re.sub(r"(?<=\d)([HhLl*])(?=[\s.,;:_]|$)", r" \1", line)
line = line.replace("«", "<").replace("»", ">")
tokens = [t.strip("_.,;:'\"`~") or t for t in line.split(" ")]
tokens = [t for t in tokens if t and not re.fullmatch(r"[_.,;:'\"`~=]+", t)]
for i in range(1, len(tokens)):
tok = tokens[i]
if not NUM_RE.match(tok) or tok[0] in "<>≤≥":
continue
key, score, name = None, 0.0, ""
# The name is a prefix of the tokens before the value. OCR noise ("ae", "oo", "L")
# can sit between name and value, so also try dropping short trailing tokens.
for k in range(i, 0, -1):
if k < i and len(tokens[k]) > 3:
break
cand = " ".join(tokens[:k]).strip(" :.")
if not re.search(r"[A-Za-zÀ-ÿ-ۿ]", cand):
continue
ck, cs = match_test_name(cand)
if ck and cs > score:
key, score, name = ck, cs, cand
if cs == 100.0:
break
if key is None:
continue
# OCR sometimes emits a stray short number before the real value:
# "K 7 3.79 mmol/L" -> the value is the number directly followed by a unit.
if (i + 2 < len(tokens) and NUM_RE.match(tokens[i + 1]) and "." not in tok and "," not in tok
and len(tok) <= 2 and _looks_like_unit(tokens[i + 2])):
i += 1
tok = tokens[i]
j = i + 1
marker = None
if j < len(tokens) and tokens[j].lower() in MARKERS:
m = tokens[j].lower()
marker = "H" if m in ("h", "hh", "↑") else "L" if m in ("l", "ll", "↓") else "*" if m == "*" else None
j += 1
unit = None
if j < len(tokens) and _looks_like_unit(tokens[j]):
unit = tokens[j]
j += 1
ref_low, ref_high = parse_range(" ".join(tokens[j:]))
return Result(
test_key=key, raw_name=name, value=_to_float(tok), unit=_snap_unit(key, unit),
ref_low=ref_low, ref_high=ref_high,
ref_source="report" if (ref_low is not None or ref_high is not None) else "none",
match_score=score, source_line=line, report_marker=marker,
)
return None
def lines_from_pdf(path: Path) -> list[str]:
import pdfplumber
lines: list[str] = []
with pdfplumber.open(str(path)) as pdf:
for page in pdf.pages:
lines.extend((page.extract_text() or "").splitlines())
return lines
def preprocess_image(path: Path):
"""Grayscale, upscale, adaptive-threshold and strip table rules (incl. dotted lines).
Dotted row separators are the main thing that breaks Tesseract on lab reports:
without this step OCR on dotted layouts drops almost every row.
"""
import cv2
import numpy as np
from PIL import Image, ImageOps
pil = ImageOps.exif_transpose(Image.open(path)).convert("L")
img = np.array(pil)
if img.shape[1] < 2400:
f = 2400 / img.shape[1]
img = cv2.resize(img, None, fx=f, fy=f, interpolation=cv2.INTER_CUBIC)
img = cv2.GaussianBlur(img, (3, 3), 0)
bw = cv2.adaptiveThreshold(img, 255, cv2.ADAPTIVE_THRESH_GAUSSIAN_C, cv2.THRESH_BINARY, 31, 15)
inv = 255 - bw
closed = cv2.morphologyEx(inv, cv2.MORPH_CLOSE, cv2.getStructuringElement(cv2.MORPH_RECT, (9, 1)))
rules = cv2.morphologyEx(closed, cv2.MORPH_OPEN, cv2.getStructuringElement(cv2.MORPH_RECT, (150, 1)))
rules = cv2.dilate(rules, cv2.getStructuringElement(cv2.MORPH_RECT, (1, 3)))
return 255 - cv2.bitwise_and(inv, 255 - rules)
def lines_from_image(path: Path, langs: str = "eng+fra") -> list[str]:
import pytesseract
text = pytesseract.image_to_string(preprocess_image(path), lang=langs, config="--psm 6")
return text.splitlines()
def detect_language(lines: list[str]) -> str:
text = " ".join(lines).lower()
fr = sum(w in text for w in ["analyse", "résultat", "resultat", "prélèvement", "référence", "glycémie", "valeurs"])
en = sum(w in text for w in ["result", "reference", "collection", "patient:", "test", "range"])
return "fr" if fr > en else "en"
def extract(path: str | Path) -> tuple[list[Result], list[str], str, str]:
"""Returns (results, unmatched numeric lines, detected language, method)."""
path = Path(path)
method = "pdf-text"
if path.suffix.lower() == ".pdf":
lines = lines_from_pdf(path)
if sum(len(line) for line in lines) < 40: # scanned PDF without a text layer
import subprocess
import tempfile
with tempfile.TemporaryDirectory() as td:
subprocess.run(["pdftoppm", "-png", "-r", "300", str(path), f"{td}/p"], check=True)
lines = []
for png in sorted(Path(td).glob("p*.png")):
lines.extend(lines_from_image(png))
method = "ocr"
else:
lines = lines_from_image(path)
method = "ocr"
results: list[Result] = []
unmatched: list[str] = []
seen: set[str] = set()
for raw in lines:
line = _clean_ocr_line(raw)
if not line:
continue
r = parse_line(line)
if r and r.test_key not in seen:
seen.add(r.test_key)
results.append(r)
elif r is None and re.search(r"\d+[.,]\d+", line) and not re.search(r"\d{4}-\d{2}-\d{2}", line):
unmatched.append(line)
return results, unmatched, detect_language(lines), method
|