LabLens / lablens /extract.py
Imene-ee's picture
Upload 19 files
dab7db2 verified
Raw History Blame Contribute Delete
9.84 kB
"""Turn a lab report (PDF or photo) into structured results.
Pipeline:
1. Get text lines: the PDF text layer when there is one, otherwise OCR (Tesseract).
2. For every line, find "name value [H/L] unit range".
3. Map the printed name to a catalog test (exact alias, then fuzzy).
4. Snap OCR-mangled units to the closest unit the catalog knows for that test.
No LLM is involved here: extraction errors should be measurable and reproducible.
"""
from __future__ import annotations
import re
from pathlib import Path
from rapidfuzz import fuzz, process
from .catalog import load_catalog, match_test_name, normalize_unit
from .models import Result
NUM = r"[<>≤≥]?\s?\d+(?:[.,]\d+)?"
NUM_RE = re.compile(rf"^{NUM}$")
MARKERS = {"h", "l", "*", "hh", "ll", "↑", "↓", "+", "-"}
def _to_float(s: str) -> float:
s = s.strip().lstrip("<>≤≥").strip().replace(",", ".")
return float(s)
def _clean_ocr_line(line: str) -> str:
line = line.replace("|", " ").replace("—", "-").replace("–", "-")
# Common OCR confusions inside numbers: O->0 and l->1 when surrounded by digits.
line = re.sub(r"(?<=\d)[Oo](?=[\d,.])|(?<=[\d,.])[Oo](?=\d|\b)", "0", line)
line = re.sub(r"(?<=\d)\s+([.,])\s*(?=\d)", r"\1", line) # "0 ,70" -> "0,70"
return re.sub(r"\s+", " ", line).strip()
def _looks_like_unit(tok: str) -> bool:
if re.match(r"^[\[(<>≤≥]", tok):
return False
if tok[0].isdigit():
return "/" in tok or "^" in tok # 10^3/µL, 10^9/L
return bool(re.search(r"[A-Za-zµ%]", tok))
def parse_range(text: str) -> tuple[float | None, float | None]:
raw = text.strip()
# OCR often reads a closing "]" as "1": "[4.20 - 5.901" -> "[4.20 - 5.90"
m = re.match(r"^[\[(].*?(\d+[.,](\d+))\s*-\s*(\d+[.,](\d+))$", raw)
if m and m.group(3).endswith("1") and len(m.group(4)) > len(m.group(2)):
raw = raw[:-1]
t = raw.strip("()[]{} ").replace(",", ".")
t = t.replace("–", "-").replace("—", "-").replace(" à ", "-").replace(" to ", "-")
m = re.match(r"^(?:<|≤|<=|inf\.?\s*[aà]?)\s*(\d+(?:\.\d+)?)", t, re.I)
if m:
return None, float(m.group(1))
m = re.match(r"^(?:>|≥|>=|sup\.?\s*[aà]?)\s*(\d+(?:\.\d+)?)", t, re.I)
if m:
return float(m.group(1)), None
m = re.search(r"(\d+(?:\.\d+)?)\s*-\s*(\d+(?:\.\d+)?)", t)
if m:
lo, hi = float(m.group(1)), float(m.group(2))
if lo <= hi:
return lo, hi
return None, None
# How units are usually printed; used to display snapped units nicely.
_PRETTY = {normalize_unit(u): u for u in [
"g/dL", "g/L", "mg/dL", "mg/L", "mmol/L", "µmol/L", "mEq/L", "%", "fL", "U/L", "UI/L", "IU/L",
"mIU/L", "µUI/mL", "µIU/mL", "ng/mL", "µg/L", "nmol/L", "10^3/µL", "10^6/µL",
"10^9/L", "10^12/L", "/mm3", "L/L"]}
_PRETTY.update({"10^3/ul": "10^3/µL", "10^6/ul": "10^6/µL", "uui/ml": "µUI/mL", "umol/l": "µmol/L"})
def _ocr_fold_unit(u: str) -> str:
u = normalize_unit(u)
u = re.sub(r"^p(?=[uµ])", "µ", u) # "pUI/mL" -> "µUI/mL"
u = re.sub(r"10[%*°](?=\d)", "10^", u) # "10%3/pL" -> "10^3/pL"
u = re.sub(r"/[pu](?=l$)", "/µ", u) # "/pL", "/uL" -> "/µL"
return u.replace("w", "/").replace("|", "l").replace("1", "l")
def _snap_unit(test_key: str | None, unit: str | None) -> str | None:
"""Return the unit as a known spelling for this test, repairing small OCR errors."""
if not unit or not test_key:
return unit
td = load_catalog()[test_key]
known = list(td.units) + [td.canonical_unit]
norm_known = [normalize_unit(k) for k in known]
nu = normalize_unit(unit)
if nu not in norm_known:
best = process.extractOne(_ocr_fold_unit(unit), norm_known, scorer=fuzz.ratio)
if not best or best[1] < 60:
return unit # leave it; flags will rely on the printed range only
nu = best[0]
if test_key in ("wbc", "platelets", "rbc") and nu in ("g/l", "t/l"):
return nu.upper() # giga/tera per litre cell counts, as French labs print them
return _PRETTY.get(nu, known[norm_known.index(nu)])
def parse_line(line: str) -> Result | None:
"""Parse one report line. Returns None when the line has no test-like structure."""
# Split markers glued to values: "5,36H" -> "5,36 H"
line = re.sub(r"(?<=\d)([HhLl*])(?=[\s.,;:_]|$)", r" \1", line)
line = line.replace("«", "<").replace("»", ">")
tokens = [t.strip("_.,;:'\"`~") or t for t in line.split(" ")]
tokens = [t for t in tokens if t and not re.fullmatch(r"[_.,;:'\"`~=]+", t)]
for i in range(1, len(tokens)):
tok = tokens[i]
if not NUM_RE.match(tok) or tok[0] in "<>≤≥":
continue
key, score, name = None, 0.0, ""
# The name is a prefix of the tokens before the value. OCR noise ("ae", "oo", "L")
# can sit between name and value, so also try dropping short trailing tokens.
for k in range(i, 0, -1):
if k < i and len(tokens[k]) > 3:
break
cand = " ".join(tokens[:k]).strip(" :.")
if not re.search(r"[A-Za-zÀ-ÿ؀-ۿ]", cand):
continue
ck, cs = match_test_name(cand)
if ck and cs > score:
key, score, name = ck, cs, cand
if cs == 100.0:
break
if key is None:
continue
# OCR sometimes emits a stray short number before the real value:
# "K 7 3.79 mmol/L" -> the value is the number directly followed by a unit.
if (i + 2 < len(tokens) and NUM_RE.match(tokens[i + 1]) and "." not in tok and "," not in tok
and len(tok) <= 2 and _looks_like_unit(tokens[i + 2])):
i += 1
tok = tokens[i]
j = i + 1
marker = None
if j < len(tokens) and tokens[j].lower() in MARKERS:
m = tokens[j].lower()
marker = "H" if m in ("h", "hh", "↑") else "L" if m in ("l", "ll", "↓") else "*" if m == "*" else None
j += 1
unit = None
if j < len(tokens) and _looks_like_unit(tokens[j]):
unit = tokens[j]
j += 1
ref_low, ref_high = parse_range(" ".join(tokens[j:]))
return Result(
test_key=key, raw_name=name, value=_to_float(tok), unit=_snap_unit(key, unit),
ref_low=ref_low, ref_high=ref_high,
ref_source="report" if (ref_low is not None or ref_high is not None) else "none",
match_score=score, source_line=line, report_marker=marker,
)
return None
def lines_from_pdf(path: Path) -> list[str]:
import pdfplumber
lines: list[str] = []
with pdfplumber.open(str(path)) as pdf:
for page in pdf.pages:
lines.extend((page.extract_text() or "").splitlines())
return lines
def preprocess_image(path: Path):
"""Grayscale, upscale, adaptive-threshold and strip table rules (incl. dotted lines).
Dotted row separators are the main thing that breaks Tesseract on lab reports:
without this step OCR on dotted layouts drops almost every row.
"""
import cv2
import numpy as np
from PIL import Image, ImageOps
pil = ImageOps.exif_transpose(Image.open(path)).convert("L")
img = np.array(pil)
if img.shape[1] < 2400:
f = 2400 / img.shape[1]
img = cv2.resize(img, None, fx=f, fy=f, interpolation=cv2.INTER_CUBIC)
img = cv2.GaussianBlur(img, (3, 3), 0)
bw = cv2.adaptiveThreshold(img, 255, cv2.ADAPTIVE_THRESH_GAUSSIAN_C, cv2.THRESH_BINARY, 31, 15)
inv = 255 - bw
closed = cv2.morphologyEx(inv, cv2.MORPH_CLOSE, cv2.getStructuringElement(cv2.MORPH_RECT, (9, 1)))
rules = cv2.morphologyEx(closed, cv2.MORPH_OPEN, cv2.getStructuringElement(cv2.MORPH_RECT, (150, 1)))
rules = cv2.dilate(rules, cv2.getStructuringElement(cv2.MORPH_RECT, (1, 3)))
return 255 - cv2.bitwise_and(inv, 255 - rules)
def lines_from_image(path: Path, langs: str = "eng+fra") -> list[str]:
import pytesseract
text = pytesseract.image_to_string(preprocess_image(path), lang=langs, config="--psm 6")
return text.splitlines()
def detect_language(lines: list[str]) -> str:
text = " ".join(lines).lower()
fr = sum(w in text for w in ["analyse", "résultat", "resultat", "prélèvement", "référence", "glycémie", "valeurs"])
en = sum(w in text for w in ["result", "reference", "collection", "patient:", "test", "range"])
return "fr" if fr > en else "en"
def extract(path: str | Path) -> tuple[list[Result], list[str], str, str]:
"""Returns (results, unmatched numeric lines, detected language, method)."""
path = Path(path)
method = "pdf-text"
if path.suffix.lower() == ".pdf":
lines = lines_from_pdf(path)
if sum(len(line) for line in lines) < 40: # scanned PDF without a text layer
import subprocess
import tempfile
with tempfile.TemporaryDirectory() as td:
subprocess.run(["pdftoppm", "-png", "-r", "300", str(path), f"{td}/p"], check=True)
lines = []
for png in sorted(Path(td).glob("p*.png")):
lines.extend(lines_from_image(png))
method = "ocr"
else:
lines = lines_from_image(path)
method = "ocr"
results: list[Result] = []
unmatched: list[str] = []
seen: set[str] = set()
for raw in lines:
line = _clean_ocr_line(raw)
if not line:
continue
r = parse_line(line)
if r and r.test_key not in seen:
seen.add(r.test_key)
results.append(r)
elif r is None and re.search(r"\d+[.,]\d+", line) and not re.search(r"\d{4}-\d{2}-\d{2}", line):
unmatched.append(line)
return results, unmatched, detect_language(lines), method