"""Turn a lab report (PDF or photo) into structured results. Pipeline: 1. Get text lines: the PDF text layer when there is one, otherwise OCR (Tesseract). 2. For every line, find "name value [H/L] unit range". 3. Map the printed name to a catalog test (exact alias, then fuzzy). 4. Snap OCR-mangled units to the closest unit the catalog knows for that test. No LLM is involved here: extraction errors should be measurable and reproducible. """ from __future__ import annotations import re from pathlib import Path from rapidfuzz import fuzz, process from .catalog import load_catalog, match_test_name, normalize_unit from .models import Result NUM = r"[<>≤≥]?\s?\d+(?:[.,]\d+)?" NUM_RE = re.compile(rf"^{NUM}$") MARKERS = {"h", "l", "*", "hh", "ll", "↑", "↓", "+", "-"} def _to_float(s: str) -> float: s = s.strip().lstrip("<>≤≥").strip().replace(",", ".") return float(s) def _clean_ocr_line(line: str) -> str: line = line.replace("|", " ").replace("—", "-").replace("–", "-") # Common OCR confusions inside numbers: O->0 and l->1 when surrounded by digits. line = re.sub(r"(?<=\d)[Oo](?=[\d,.])|(?<=[\d,.])[Oo](?=\d|\b)", "0", line) line = re.sub(r"(?<=\d)\s+([.,])\s*(?=\d)", r"\1", line) # "0 ,70" -> "0,70" return re.sub(r"\s+", " ", line).strip() def _looks_like_unit(tok: str) -> bool: if re.match(r"^[\[(<>≤≥]", tok): return False if tok[0].isdigit(): return "/" in tok or "^" in tok # 10^3/µL, 10^9/L return bool(re.search(r"[A-Za-zµ%]", tok)) def parse_range(text: str) -> tuple[float | None, float | None]: raw = text.strip() # OCR often reads a closing "]" as "1": "[4.20 - 5.901" -> "[4.20 - 5.90" m = re.match(r"^[\[(].*?(\d+[.,](\d+))\s*-\s*(\d+[.,](\d+))$", raw) if m and m.group(3).endswith("1") and len(m.group(4)) > len(m.group(2)): raw = raw[:-1] t = raw.strip("()[]{} ").replace(",", ".") t = t.replace("–", "-").replace("—", "-").replace(" à ", "-").replace(" to ", "-") m = re.match(r"^(?:<|≤|<=|inf\.?\s*[aà]?)\s*(\d+(?:\.\d+)?)", t, re.I) if m: return None, float(m.group(1)) m = re.match(r"^(?:>|≥|>=|sup\.?\s*[aà]?)\s*(\d+(?:\.\d+)?)", t, re.I) if m: return float(m.group(1)), None m = re.search(r"(\d+(?:\.\d+)?)\s*-\s*(\d+(?:\.\d+)?)", t) if m: lo, hi = float(m.group(1)), float(m.group(2)) if lo <= hi: return lo, hi return None, None # How units are usually printed; used to display snapped units nicely. _PRETTY = {normalize_unit(u): u for u in [ "g/dL", "g/L", "mg/dL", "mg/L", "mmol/L", "µmol/L", "mEq/L", "%", "fL", "U/L", "UI/L", "IU/L", "mIU/L", "µUI/mL", "µIU/mL", "ng/mL", "µg/L", "nmol/L", "10^3/µL", "10^6/µL", "10^9/L", "10^12/L", "/mm3", "L/L"]} _PRETTY.update({"10^3/ul": "10^3/µL", "10^6/ul": "10^6/µL", "uui/ml": "µUI/mL", "umol/l": "µmol/L"}) def _ocr_fold_unit(u: str) -> str: u = normalize_unit(u) u = re.sub(r"^p(?=[uµ])", "µ", u) # "pUI/mL" -> "µUI/mL" u = re.sub(r"10[%*°](?=\d)", "10^", u) # "10%3/pL" -> "10^3/pL" u = re.sub(r"/[pu](?=l$)", "/µ", u) # "/pL", "/uL" -> "/µL" return u.replace("w", "/").replace("|", "l").replace("1", "l") def _snap_unit(test_key: str | None, unit: str | None) -> str | None: """Return the unit as a known spelling for this test, repairing small OCR errors.""" if not unit or not test_key: return unit td = load_catalog()[test_key] known = list(td.units) + [td.canonical_unit] norm_known = [normalize_unit(k) for k in known] nu = normalize_unit(unit) if nu not in norm_known: best = process.extractOne(_ocr_fold_unit(unit), norm_known, scorer=fuzz.ratio) if not best or best[1] < 60: return unit # leave it; flags will rely on the printed range only nu = best[0] if test_key in ("wbc", "platelets", "rbc") and nu in ("g/l", "t/l"): return nu.upper() # giga/tera per litre cell counts, as French labs print them return _PRETTY.get(nu, known[norm_known.index(nu)]) def parse_line(line: str) -> Result | None: """Parse one report line. Returns None when the line has no test-like structure.""" # Split markers glued to values: "5,36H" -> "5,36 H" line = re.sub(r"(?<=\d)([HhLl*])(?=[\s.,;:_]|$)", r" \1", line) line = line.replace("«", "<").replace("»", ">") tokens = [t.strip("_.,;:'\"`~") or t for t in line.split(" ")] tokens = [t for t in tokens if t and not re.fullmatch(r"[_.,;:'\"`~=]+", t)] for i in range(1, len(tokens)): tok = tokens[i] if not NUM_RE.match(tok) or tok[0] in "<>≤≥": continue key, score, name = None, 0.0, "" # The name is a prefix of the tokens before the value. OCR noise ("ae", "oo", "L") # can sit between name and value, so also try dropping short trailing tokens. for k in range(i, 0, -1): if k < i and len(tokens[k]) > 3: break cand = " ".join(tokens[:k]).strip(" :.") if not re.search(r"[A-Za-zÀ-ÿ؀-ۿ]", cand): continue ck, cs = match_test_name(cand) if ck and cs > score: key, score, name = ck, cs, cand if cs == 100.0: break if key is None: continue # OCR sometimes emits a stray short number before the real value: # "K 7 3.79 mmol/L" -> the value is the number directly followed by a unit. if (i + 2 < len(tokens) and NUM_RE.match(tokens[i + 1]) and "." not in tok and "," not in tok and len(tok) <= 2 and _looks_like_unit(tokens[i + 2])): i += 1 tok = tokens[i] j = i + 1 marker = None if j < len(tokens) and tokens[j].lower() in MARKERS: m = tokens[j].lower() marker = "H" if m in ("h", "hh", "↑") else "L" if m in ("l", "ll", "↓") else "*" if m == "*" else None j += 1 unit = None if j < len(tokens) and _looks_like_unit(tokens[j]): unit = tokens[j] j += 1 ref_low, ref_high = parse_range(" ".join(tokens[j:])) return Result( test_key=key, raw_name=name, value=_to_float(tok), unit=_snap_unit(key, unit), ref_low=ref_low, ref_high=ref_high, ref_source="report" if (ref_low is not None or ref_high is not None) else "none", match_score=score, source_line=line, report_marker=marker, ) return None def lines_from_pdf(path: Path) -> list[str]: import pdfplumber lines: list[str] = [] with pdfplumber.open(str(path)) as pdf: for page in pdf.pages: lines.extend((page.extract_text() or "").splitlines()) return lines def preprocess_image(path: Path): """Grayscale, upscale, adaptive-threshold and strip table rules (incl. dotted lines). Dotted row separators are the main thing that breaks Tesseract on lab reports: without this step OCR on dotted layouts drops almost every row. """ import cv2 import numpy as np from PIL import Image, ImageOps pil = ImageOps.exif_transpose(Image.open(path)).convert("L") img = np.array(pil) if img.shape[1] < 2400: f = 2400 / img.shape[1] img = cv2.resize(img, None, fx=f, fy=f, interpolation=cv2.INTER_CUBIC) img = cv2.GaussianBlur(img, (3, 3), 0) bw = cv2.adaptiveThreshold(img, 255, cv2.ADAPTIVE_THRESH_GAUSSIAN_C, cv2.THRESH_BINARY, 31, 15) inv = 255 - bw closed = cv2.morphologyEx(inv, cv2.MORPH_CLOSE, cv2.getStructuringElement(cv2.MORPH_RECT, (9, 1))) rules = cv2.morphologyEx(closed, cv2.MORPH_OPEN, cv2.getStructuringElement(cv2.MORPH_RECT, (150, 1))) rules = cv2.dilate(rules, cv2.getStructuringElement(cv2.MORPH_RECT, (1, 3))) return 255 - cv2.bitwise_and(inv, 255 - rules) def lines_from_image(path: Path, langs: str = "eng+fra") -> list[str]: import pytesseract text = pytesseract.image_to_string(preprocess_image(path), lang=langs, config="--psm 6") return text.splitlines() def detect_language(lines: list[str]) -> str: text = " ".join(lines).lower() fr = sum(w in text for w in ["analyse", "résultat", "resultat", "prélèvement", "référence", "glycémie", "valeurs"]) en = sum(w in text for w in ["result", "reference", "collection", "patient:", "test", "range"]) return "fr" if fr > en else "en" def extract(path: str | Path) -> tuple[list[Result], list[str], str, str]: """Returns (results, unmatched numeric lines, detected language, method).""" path = Path(path) method = "pdf-text" if path.suffix.lower() == ".pdf": lines = lines_from_pdf(path) if sum(len(line) for line in lines) < 40: # scanned PDF without a text layer import subprocess import tempfile with tempfile.TemporaryDirectory() as td: subprocess.run(["pdftoppm", "-png", "-r", "300", str(path), f"{td}/p"], check=True) lines = [] for png in sorted(Path(td).glob("p*.png")): lines.extend(lines_from_image(png)) method = "ocr" else: lines = lines_from_image(path) method = "ocr" results: list[Result] = [] unmatched: list[str] = [] seen: set[str] = set() for raw in lines: line = _clean_ocr_line(raw) if not line: continue r = parse_line(line) if r and r.test_key not in seen: seen.add(r.test_key) results.append(r) elif r is None and re.search(r"\d+[.,]\d+", line) and not re.search(r"\d{4}-\d{2}-\d{2}", line): unmatched.append(line) return results, unmatched, detect_language(lines), method