Download lablens/extract.py from Imene-ee/LabLens: direct link, hf CLI and curl.
- Browser
- Download file 9.84 kB
-
https://huggingface.co/spaces/Imene-ee/LabLens/resolve/main/lablens/extract.py
- Command line
-
hf download hf://spaces/Imene-ee/LabLens/lablens/extract.py
-
curl -L -o extract.py https://huggingface.co/spaces/Imene-ee/LabLens/resolve/main/lablens/extract.py
9.84 kB
| """Turn a lab report (PDF or photo) into structured results. | |
| Pipeline: | |
| 1. Get text lines: the PDF text layer when there is one, otherwise OCR (Tesseract). | |
| 2. For every line, find "name value [H/L] unit range". | |
| 3. Map the printed name to a catalog test (exact alias, then fuzzy). | |
| 4. Snap OCR-mangled units to the closest unit the catalog knows for that test. | |
| No LLM is involved here: extraction errors should be measurable and reproducible. | |
| """ | |
| from __future__ import annotations | |
| import re | |
| from pathlib import Path | |
| from rapidfuzz import fuzz, process | |
| from .catalog import load_catalog, match_test_name, normalize_unit | |
| from .models import Result | |
| NUM = r"[<>≤≥]?\s?\d+(?:[.,]\d+)?" | |
| NUM_RE = re.compile(rf"^{NUM}$") | |
| MARKERS = {"h", "l", "*", "hh", "ll", "↑", "↓", "+", "-"} | |
| def _to_float(s: str) -> float: | |
| s = s.strip().lstrip("<>≤≥").strip().replace(",", ".") | |
| return float(s) | |
| def _clean_ocr_line(line: str) -> str: | |
| line = line.replace("|", " ").replace("—", "-").replace("–", "-") | |
| # Common OCR confusions inside numbers: O->0 and l->1 when surrounded by digits. | |
| line = re.sub(r"(?<=\d)[Oo](?=[\d,.])|(?<=[\d,.])[Oo](?=\d|\b)", "0", line) | |
| line = re.sub(r"(?<=\d)\s+([.,])\s*(?=\d)", r"\1", line) # "0 ,70" -> "0,70" | |
| return re.sub(r"\s+", " ", line).strip() | |
| def _looks_like_unit(tok: str) -> bool: | |
| if re.match(r"^[\[(<>≤≥]", tok): | |
| return False | |
| if tok[0].isdigit(): | |
| return "/" in tok or "^" in tok # 10^3/µL, 10^9/L | |
| return bool(re.search(r"[A-Za-zµ%]", tok)) | |
| def parse_range(text: str) -> tuple[float | None, float | None]: | |
| raw = text.strip() | |
| # OCR often reads a closing "]" as "1": "[4.20 - 5.901" -> "[4.20 - 5.90" | |
| m = re.match(r"^[\[(].*?(\d+[.,](\d+))\s*-\s*(\d+[.,](\d+))$", raw) | |
| if m and m.group(3).endswith("1") and len(m.group(4)) > len(m.group(2)): | |
| raw = raw[:-1] | |
| t = raw.strip("()[]{} ").replace(",", ".") | |
| t = t.replace("–", "-").replace("—", "-").replace(" à ", "-").replace(" to ", "-") | |
| m = re.match(r"^(?:<|≤|<=|inf\.?\s*[aà]?)\s*(\d+(?:\.\d+)?)", t, re.I) | |
| if m: | |
| return None, float(m.group(1)) | |
| m = re.match(r"^(?:>|≥|>=|sup\.?\s*[aà]?)\s*(\d+(?:\.\d+)?)", t, re.I) | |
| if m: | |
| return float(m.group(1)), None | |
| m = re.search(r"(\d+(?:\.\d+)?)\s*-\s*(\d+(?:\.\d+)?)", t) | |
| if m: | |
| lo, hi = float(m.group(1)), float(m.group(2)) | |
| if lo <= hi: | |
| return lo, hi | |
| return None, None | |
| # How units are usually printed; used to display snapped units nicely. | |
| _PRETTY = {normalize_unit(u): u for u in [ | |
| "g/dL", "g/L", "mg/dL", "mg/L", "mmol/L", "µmol/L", "mEq/L", "%", "fL", "U/L", "UI/L", "IU/L", | |
| "mIU/L", "µUI/mL", "µIU/mL", "ng/mL", "µg/L", "nmol/L", "10^3/µL", "10^6/µL", | |
| "10^9/L", "10^12/L", "/mm3", "L/L"]} | |
| _PRETTY.update({"10^3/ul": "10^3/µL", "10^6/ul": "10^6/µL", "uui/ml": "µUI/mL", "umol/l": "µmol/L"}) | |
| def _ocr_fold_unit(u: str) -> str: | |
| u = normalize_unit(u) | |
| u = re.sub(r"^p(?=[uµ])", "µ", u) # "pUI/mL" -> "µUI/mL" | |
| u = re.sub(r"10[%*°](?=\d)", "10^", u) # "10%3/pL" -> "10^3/pL" | |
| u = re.sub(r"/[pu](?=l$)", "/µ", u) # "/pL", "/uL" -> "/µL" | |
| return u.replace("w", "/").replace("|", "l").replace("1", "l") | |
| def _snap_unit(test_key: str | None, unit: str | None) -> str | None: | |
| """Return the unit as a known spelling for this test, repairing small OCR errors.""" | |
| if not unit or not test_key: | |
| return unit | |
| td = load_catalog()[test_key] | |
| known = list(td.units) + [td.canonical_unit] | |
| norm_known = [normalize_unit(k) for k in known] | |
| nu = normalize_unit(unit) | |
| if nu not in norm_known: | |
| best = process.extractOne(_ocr_fold_unit(unit), norm_known, scorer=fuzz.ratio) | |
| if not best or best[1] < 60: | |
| return unit # leave it; flags will rely on the printed range only | |
| nu = best[0] | |
| if test_key in ("wbc", "platelets", "rbc") and nu in ("g/l", "t/l"): | |
| return nu.upper() # giga/tera per litre cell counts, as French labs print them | |
| return _PRETTY.get(nu, known[norm_known.index(nu)]) | |
| def parse_line(line: str) -> Result | None: | |
| """Parse one report line. Returns None when the line has no test-like structure.""" | |
| # Split markers glued to values: "5,36H" -> "5,36 H" | |
| line = re.sub(r"(?<=\d)([HhLl*])(?=[\s.,;:_]|$)", r" \1", line) | |
| line = line.replace("«", "<").replace("»", ">") | |
| tokens = [t.strip("_.,;:'\"`~") or t for t in line.split(" ")] | |
| tokens = [t for t in tokens if t and not re.fullmatch(r"[_.,;:'\"`~=]+", t)] | |
| for i in range(1, len(tokens)): | |
| tok = tokens[i] | |
| if not NUM_RE.match(tok) or tok[0] in "<>≤≥": | |
| continue | |
| key, score, name = None, 0.0, "" | |
| # The name is a prefix of the tokens before the value. OCR noise ("ae", "oo", "L") | |
| # can sit between name and value, so also try dropping short trailing tokens. | |
| for k in range(i, 0, -1): | |
| if k < i and len(tokens[k]) > 3: | |
| break | |
| cand = " ".join(tokens[:k]).strip(" :.") | |
| if not re.search(r"[A-Za-zÀ-ÿ-ۿ]", cand): | |
| continue | |
| ck, cs = match_test_name(cand) | |
| if ck and cs > score: | |
| key, score, name = ck, cs, cand | |
| if cs == 100.0: | |
| break | |
| if key is None: | |
| continue | |
| # OCR sometimes emits a stray short number before the real value: | |
| # "K 7 3.79 mmol/L" -> the value is the number directly followed by a unit. | |
| if (i + 2 < len(tokens) and NUM_RE.match(tokens[i + 1]) and "." not in tok and "," not in tok | |
| and len(tok) <= 2 and _looks_like_unit(tokens[i + 2])): | |
| i += 1 | |
| tok = tokens[i] | |
| j = i + 1 | |
| marker = None | |
| if j < len(tokens) and tokens[j].lower() in MARKERS: | |
| m = tokens[j].lower() | |
| marker = "H" if m in ("h", "hh", "↑") else "L" if m in ("l", "ll", "↓") else "*" if m == "*" else None | |
| j += 1 | |
| unit = None | |
| if j < len(tokens) and _looks_like_unit(tokens[j]): | |
| unit = tokens[j] | |
| j += 1 | |
| ref_low, ref_high = parse_range(" ".join(tokens[j:])) | |
| return Result( | |
| test_key=key, raw_name=name, value=_to_float(tok), unit=_snap_unit(key, unit), | |
| ref_low=ref_low, ref_high=ref_high, | |
| ref_source="report" if (ref_low is not None or ref_high is not None) else "none", | |
| match_score=score, source_line=line, report_marker=marker, | |
| ) | |
| return None | |
| def lines_from_pdf(path: Path) -> list[str]: | |
| import pdfplumber | |
| lines: list[str] = [] | |
| with pdfplumber.open(str(path)) as pdf: | |
| for page in pdf.pages: | |
| lines.extend((page.extract_text() or "").splitlines()) | |
| return lines | |
| def preprocess_image(path: Path): | |
| """Grayscale, upscale, adaptive-threshold and strip table rules (incl. dotted lines). | |
| Dotted row separators are the main thing that breaks Tesseract on lab reports: | |
| without this step OCR on dotted layouts drops almost every row. | |
| """ | |
| import cv2 | |
| import numpy as np | |
| from PIL import Image, ImageOps | |
| pil = ImageOps.exif_transpose(Image.open(path)).convert("L") | |
| img = np.array(pil) | |
| if img.shape[1] < 2400: | |
| f = 2400 / img.shape[1] | |
| img = cv2.resize(img, None, fx=f, fy=f, interpolation=cv2.INTER_CUBIC) | |
| img = cv2.GaussianBlur(img, (3, 3), 0) | |
| bw = cv2.adaptiveThreshold(img, 255, cv2.ADAPTIVE_THRESH_GAUSSIAN_C, cv2.THRESH_BINARY, 31, 15) | |
| inv = 255 - bw | |
| closed = cv2.morphologyEx(inv, cv2.MORPH_CLOSE, cv2.getStructuringElement(cv2.MORPH_RECT, (9, 1))) | |
| rules = cv2.morphologyEx(closed, cv2.MORPH_OPEN, cv2.getStructuringElement(cv2.MORPH_RECT, (150, 1))) | |
| rules = cv2.dilate(rules, cv2.getStructuringElement(cv2.MORPH_RECT, (1, 3))) | |
| return 255 - cv2.bitwise_and(inv, 255 - rules) | |
| def lines_from_image(path: Path, langs: str = "eng+fra") -> list[str]: | |
| import pytesseract | |
| text = pytesseract.image_to_string(preprocess_image(path), lang=langs, config="--psm 6") | |
| return text.splitlines() | |
| def detect_language(lines: list[str]) -> str: | |
| text = " ".join(lines).lower() | |
| fr = sum(w in text for w in ["analyse", "résultat", "resultat", "prélèvement", "référence", "glycémie", "valeurs"]) | |
| en = sum(w in text for w in ["result", "reference", "collection", "patient:", "test", "range"]) | |
| return "fr" if fr > en else "en" | |
| def extract(path: str | Path) -> tuple[list[Result], list[str], str, str]: | |
| """Returns (results, unmatched numeric lines, detected language, method).""" | |
| path = Path(path) | |
| method = "pdf-text" | |
| if path.suffix.lower() == ".pdf": | |
| lines = lines_from_pdf(path) | |
| if sum(len(line) for line in lines) < 40: # scanned PDF without a text layer | |
| import subprocess | |
| import tempfile | |
| with tempfile.TemporaryDirectory() as td: | |
| subprocess.run(["pdftoppm", "-png", "-r", "300", str(path), f"{td}/p"], check=True) | |
| lines = [] | |
| for png in sorted(Path(td).glob("p*.png")): | |
| lines.extend(lines_from_image(png)) | |
| method = "ocr" | |
| else: | |
| lines = lines_from_image(path) | |
| method = "ocr" | |
| results: list[Result] = [] | |
| unmatched: list[str] = [] | |
| seen: set[str] = set() | |
| for raw in lines: | |
| line = _clean_ocr_line(raw) | |
| if not line: | |
| continue | |
| r = parse_line(line) | |
| if r and r.test_key not in seen: | |
| seen.add(r.test_key) | |
| results.append(r) | |
| elif r is None and re.search(r"\d+[.,]\d+", line) and not re.search(r"\d{4}-\d{2}-\d{2}", line): | |
| unmatched.append(line) | |
| return results, unmatched, detect_language(lines), method | |