Upload 19 files
Browse files- .gitattributes +2 -0
- .github/workflows/ci.yml +24 -0
- benchmark/results.json +86 -0
- benchmark/results.md +8 -0
- lablens/__init__.py +0 -0
- lablens/catalog.py +144 -0
- lablens/data/catalog.yaml +407 -0
- lablens/evaluate.py +122 -0
- lablens/explain.py +186 -0
- lablens/extract.py +237 -0
- lablens/flagger.py +84 -0
- lablens/i18n.py +77 -0
- lablens/models.py +44 -0
- lablens/synth/__init__.py +0 -0
- lablens/synth/generator.py +320 -0
- samples/sample_en_report.pdf +0 -0
- samples/sample_fr_photo.png +3 -0
- samples/sample_fr_urgent.png +3 -0
- tests/__init__.py +0 -0
- tests/test_core.py +196 -0
.gitattributes
CHANGED
|
@@ -33,3 +33,5 @@ saved_model/**/* filter=lfs diff=lfs merge=lfs -text
|
|
| 33 |
*.zip filter=lfs diff=lfs merge=lfs -text
|
| 34 |
*.zst filter=lfs diff=lfs merge=lfs -text
|
| 35 |
*tfevents* filter=lfs diff=lfs merge=lfs -text
|
|
|
|
|
|
|
|
|
| 33 |
*.zip filter=lfs diff=lfs merge=lfs -text
|
| 34 |
*.zst filter=lfs diff=lfs merge=lfs -text
|
| 35 |
*tfevents* filter=lfs diff=lfs merge=lfs -text
|
| 36 |
+
samples/sample_fr_photo.png filter=lfs diff=lfs merge=lfs -text
|
| 37 |
+
samples/sample_fr_urgent.png filter=lfs diff=lfs merge=lfs -text
|
.github/workflows/ci.yml
ADDED
|
@@ -0,0 +1,24 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
name: tests
|
| 2 |
+
|
| 3 |
+
on:
|
| 4 |
+
push:
|
| 5 |
+
pull_request:
|
| 6 |
+
|
| 7 |
+
jobs:
|
| 8 |
+
test:
|
| 9 |
+
runs-on: ubuntu-latest
|
| 10 |
+
steps:
|
| 11 |
+
- uses: actions/checkout@v4
|
| 12 |
+
- name: System packages (OCR + PDF tools)
|
| 13 |
+
run: |
|
| 14 |
+
sudo apt-get update
|
| 15 |
+
sudo apt-get install -y tesseract-ocr tesseract-ocr-eng tesseract-ocr-fra tesseract-ocr-ara poppler-utils fonts-dejavu-core
|
| 16 |
+
- uses: actions/setup-python@v5
|
| 17 |
+
with:
|
| 18 |
+
python-version: "3.12"
|
| 19 |
+
- run: pip install -r requirements.txt pytest
|
| 20 |
+
- run: pytest -q
|
| 21 |
+
- name: Mini benchmark (20 reports)
|
| 22 |
+
run: |
|
| 23 |
+
python -m lablens.synth.generator --out /tmp/bench -n 20 --seed 1000
|
| 24 |
+
python -m lablens.evaluate --bench /tmp/bench --out /tmp/results.json
|
benchmark/results.json
ADDED
|
@@ -0,0 +1,86 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{
|
| 2 |
+
"pdf/all": {
|
| 3 |
+
"reports": 200,
|
| 4 |
+
"rows": 1989,
|
| 5 |
+
"recall": 1.0,
|
| 6 |
+
"precision": 1.0,
|
| 7 |
+
"value_acc": 1.0,
|
| 8 |
+
"unit_acc": 1.0,
|
| 9 |
+
"range_acc": 1.0,
|
| 10 |
+
"flag_acc": 1.0,
|
| 11 |
+
"critical_recall": 1.0,
|
| 12 |
+
"critical_rows": 59,
|
| 13 |
+
"dangerous_misses": 0,
|
| 14 |
+
"abnormal_rows": 584
|
| 15 |
+
},
|
| 16 |
+
"pdf/en": {
|
| 17 |
+
"reports": 101,
|
| 18 |
+
"rows": 1009,
|
| 19 |
+
"recall": 1.0,
|
| 20 |
+
"precision": 1.0,
|
| 21 |
+
"value_acc": 1.0,
|
| 22 |
+
"unit_acc": 1.0,
|
| 23 |
+
"range_acc": 1.0,
|
| 24 |
+
"flag_acc": 1.0,
|
| 25 |
+
"critical_recall": 1.0,
|
| 26 |
+
"critical_rows": 25,
|
| 27 |
+
"dangerous_misses": 0,
|
| 28 |
+
"abnormal_rows": 289
|
| 29 |
+
},
|
| 30 |
+
"pdf/fr": {
|
| 31 |
+
"reports": 99,
|
| 32 |
+
"rows": 980,
|
| 33 |
+
"recall": 1.0,
|
| 34 |
+
"precision": 1.0,
|
| 35 |
+
"value_acc": 1.0,
|
| 36 |
+
"unit_acc": 1.0,
|
| 37 |
+
"range_acc": 1.0,
|
| 38 |
+
"flag_acc": 1.0,
|
| 39 |
+
"critical_recall": 1.0,
|
| 40 |
+
"critical_rows": 34,
|
| 41 |
+
"dangerous_misses": 0,
|
| 42 |
+
"abnormal_rows": 295
|
| 43 |
+
},
|
| 44 |
+
"photo/all": {
|
| 45 |
+
"reports": 200,
|
| 46 |
+
"rows": 1989,
|
| 47 |
+
"recall": 0.9316,
|
| 48 |
+
"precision": 1.0,
|
| 49 |
+
"value_acc": 0.9595,
|
| 50 |
+
"unit_acc": 0.9709,
|
| 51 |
+
"range_acc": 0.9687,
|
| 52 |
+
"flag_acc": 0.8924,
|
| 53 |
+
"critical_recall": 0.8475,
|
| 54 |
+
"critical_rows": 59,
|
| 55 |
+
"dangerous_misses": 2,
|
| 56 |
+
"abnormal_rows": 584
|
| 57 |
+
},
|
| 58 |
+
"photo/en": {
|
| 59 |
+
"reports": 101,
|
| 60 |
+
"rows": 1009,
|
| 61 |
+
"recall": 0.9217,
|
| 62 |
+
"precision": 1.0,
|
| 63 |
+
"value_acc": 0.9624,
|
| 64 |
+
"unit_acc": 0.9796,
|
| 65 |
+
"range_acc": 0.9753,
|
| 66 |
+
"flag_acc": 0.885,
|
| 67 |
+
"critical_recall": 0.88,
|
| 68 |
+
"critical_rows": 25,
|
| 69 |
+
"dangerous_misses": 1,
|
| 70 |
+
"abnormal_rows": 289
|
| 71 |
+
},
|
| 72 |
+
"photo/fr": {
|
| 73 |
+
"reports": 99,
|
| 74 |
+
"rows": 980,
|
| 75 |
+
"recall": 0.9418,
|
| 76 |
+
"precision": 1.0,
|
| 77 |
+
"value_acc": 0.9567,
|
| 78 |
+
"unit_acc": 0.9621,
|
| 79 |
+
"range_acc": 0.9621,
|
| 80 |
+
"flag_acc": 0.9,
|
| 81 |
+
"critical_recall": 0.8235,
|
| 82 |
+
"critical_rows": 34,
|
| 83 |
+
"dangerous_misses": 1,
|
| 84 |
+
"abnormal_rows": 295
|
| 85 |
+
}
|
| 86 |
+
}
|
benchmark/results.md
ADDED
|
@@ -0,0 +1,8 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
| split | reports | rows | recall | precision | value_acc | unit_acc | range_acc | flag_acc | critical_recall | dangerous_misses |
|
| 2 |
+
|---|---|---|---|---|---|---|---|---|---|---|
|
| 3 |
+
| pdf/all | 200 | 1989 | 100.0% | 100.0% | 100.0% | 100.0% | 100.0% | 100.0% | 100.0% | 0 |
|
| 4 |
+
| pdf/en | 101 | 1009 | 100.0% | 100.0% | 100.0% | 100.0% | 100.0% | 100.0% | 100.0% | 0 |
|
| 5 |
+
| pdf/fr | 99 | 980 | 100.0% | 100.0% | 100.0% | 100.0% | 100.0% | 100.0% | 100.0% | 0 |
|
| 6 |
+
| photo/all | 200 | 1989 | 93.2% | 100.0% | 96.0% | 97.1% | 96.9% | 89.2% | 84.8% | 2 |
|
| 7 |
+
| photo/en | 101 | 1009 | 92.2% | 100.0% | 96.2% | 98.0% | 97.5% | 88.5% | 88.0% | 1 |
|
| 8 |
+
| photo/fr | 99 | 980 | 94.2% | 100.0% | 95.7% | 96.2% | 96.2% | 90.0% | 82.3% | 1 |
|
lablens/__init__.py
ADDED
|
File without changes
|
lablens/catalog.py
ADDED
|
@@ -0,0 +1,144 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""Load the test catalog and resolve names and units against it."""
|
| 2 |
+
from __future__ import annotations
|
| 3 |
+
|
| 4 |
+
import re
|
| 5 |
+
import unicodedata
|
| 6 |
+
from dataclasses import dataclass, field
|
| 7 |
+
from functools import lru_cache
|
| 8 |
+
from pathlib import Path
|
| 9 |
+
|
| 10 |
+
import yaml
|
| 11 |
+
from rapidfuzz import fuzz, process
|
| 12 |
+
|
| 13 |
+
CATALOG_PATH = Path(__file__).parent / "data" / "catalog.yaml"
|
| 14 |
+
|
| 15 |
+
|
| 16 |
+
def normalize_text(s: str) -> str:
|
| 17 |
+
"""Lowercase, strip accents (Latin only), collapse whitespace."""
|
| 18 |
+
s = s.strip().lower()
|
| 19 |
+
# Strip Latin diacritics but keep Arabic letters intact.
|
| 20 |
+
decomposed = unicodedata.normalize("NFKD", s)
|
| 21 |
+
s = "".join(c for c in decomposed if not (unicodedata.combining(c) and ord(c) < 0x0600))
|
| 22 |
+
s = s.replace("μ", "µ")
|
| 23 |
+
return re.sub(r"\s+", " ", s)
|
| 24 |
+
|
| 25 |
+
|
| 26 |
+
def normalize_unit(u: str) -> str:
|
| 27 |
+
u = normalize_text(u or "").replace(" ", "")
|
| 28 |
+
u = u.replace("μ", "µ").replace("×", "").replace("x10", "10")
|
| 29 |
+
u = u.replace("10e", "10^").replace("10*", "10^")
|
| 30 |
+
u = re.sub(r"10\^?(\d)", r"10^\1", u)
|
| 31 |
+
return u
|
| 32 |
+
|
| 33 |
+
|
| 34 |
+
@dataclass
|
| 35 |
+
class TestDef:
|
| 36 |
+
key: str
|
| 37 |
+
display: dict[str, str]
|
| 38 |
+
category: str
|
| 39 |
+
canonical_unit: str
|
| 40 |
+
units: dict[str, float]
|
| 41 |
+
typical_range: tuple[float, float]
|
| 42 |
+
critical: dict[str, float]
|
| 43 |
+
plausible: tuple[float, float]
|
| 44 |
+
aliases: dict[str, list[str]]
|
| 45 |
+
kb: dict[str, str]
|
| 46 |
+
all_aliases: list[str] = field(default_factory=list)
|
| 47 |
+
|
| 48 |
+
def to_canonical(self, value: float, unit: str | None) -> float | None:
|
| 49 |
+
"""Convert a value to the canonical unit. Returns None if the unit is unknown."""
|
| 50 |
+
if not unit:
|
| 51 |
+
return value # assume canonical when no unit printed
|
| 52 |
+
nu = normalize_unit(unit)
|
| 53 |
+
for spelled, factor in self.units.items():
|
| 54 |
+
if normalize_unit(spelled) == nu:
|
| 55 |
+
return value * factor
|
| 56 |
+
if nu == normalize_unit(self.canonical_unit):
|
| 57 |
+
return value
|
| 58 |
+
return None
|
| 59 |
+
|
| 60 |
+
|
| 61 |
+
@lru_cache(maxsize=1)
|
| 62 |
+
def load_catalog(path: str | None = None) -> dict[str, TestDef]:
|
| 63 |
+
raw = yaml.safe_load(Path(path or CATALOG_PATH).read_text(encoding="utf-8"))
|
| 64 |
+
out: dict[str, TestDef] = {}
|
| 65 |
+
for key, t in raw["tests"].items():
|
| 66 |
+
td = TestDef(
|
| 67 |
+
key=key,
|
| 68 |
+
display=t["display"],
|
| 69 |
+
category=t["category"],
|
| 70 |
+
canonical_unit=t["canonical_unit"],
|
| 71 |
+
units=t["units"],
|
| 72 |
+
typical_range=tuple(t["typical_range"]),
|
| 73 |
+
critical=t.get("critical") or {},
|
| 74 |
+
plausible=tuple(t["plausible"]),
|
| 75 |
+
aliases=t["aliases"],
|
| 76 |
+
kb=t["kb"],
|
| 77 |
+
)
|
| 78 |
+
td.all_aliases = sorted(
|
| 79 |
+
{normalize_text(a) for lang in td.aliases.values() for a in lang}, key=len, reverse=True
|
| 80 |
+
)
|
| 81 |
+
out[key] = td
|
| 82 |
+
return out
|
| 83 |
+
|
| 84 |
+
|
| 85 |
+
@lru_cache(maxsize=1)
|
| 86 |
+
def _alias_index() -> dict[str, str]:
|
| 87 |
+
idx: dict[str, str] = {}
|
| 88 |
+
for key, td in load_catalog().items():
|
| 89 |
+
for a in td.all_aliases:
|
| 90 |
+
idx.setdefault(a, key)
|
| 91 |
+
return idx
|
| 92 |
+
|
| 93 |
+
|
| 94 |
+
# Characters Tesseract commonly confuses, folded to one representative before fuzzy
|
| 95 |
+
# matching: "Hemiogiouine" and "hemoglobine" end up much closer.
|
| 96 |
+
_OCR_FOLD = str.maketrans({"l": "i", "1": "i", "|": "i", "!": "i", "j": "i", "t": "i",
|
| 97 |
+
"0": "o", "a": "o", "e": "o", "c": "o", "u": "o", "b": "o", "g": "o"})
|
| 98 |
+
|
| 99 |
+
|
| 100 |
+
def _fold(s: str) -> str:
|
| 101 |
+
s = s.replace("rn", "m").replace("-", " ").replace("_", " ")
|
| 102 |
+
return re.sub(r"\s+", " ", s.translate(_OCR_FOLD)).strip()
|
| 103 |
+
|
| 104 |
+
|
| 105 |
+
@lru_cache(maxsize=1)
|
| 106 |
+
def _folded_index() -> dict[str, set[str]]:
|
| 107 |
+
out: dict[str, set[str]] = {}
|
| 108 |
+
for a, k in _alias_index().items():
|
| 109 |
+
if len(a) >= 4:
|
| 110 |
+
out.setdefault(_fold(a), set()).add(k)
|
| 111 |
+
return out
|
| 112 |
+
|
| 113 |
+
|
| 114 |
+
def match_test_name(name: str, min_score: int = 80, margin: float = 6.0) -> tuple[str | None, float]:
|
| 115 |
+
"""Map a printed test name to a catalog key.
|
| 116 |
+
|
| 117 |
+
Exact alias first. Otherwise fuzzy matching on OCR-folded strings, accepted only when
|
| 118 |
+
the best test beats the runner-up *test* by a margin. HDL vs LDL differ by one letter
|
| 119 |
+
and have opposite meanings, so an ambiguous match is rejected rather than guessed.
|
| 120 |
+
"""
|
| 121 |
+
n = normalize_text(name)
|
| 122 |
+
n = re.sub(r"[:.\-–_]+$", "", n).strip()
|
| 123 |
+
idx = _alias_index()
|
| 124 |
+
if n in idx:
|
| 125 |
+
return idx[n], 100.0
|
| 126 |
+
if len(n) < 4:
|
| 127 |
+
# Short aliases (hb, k, na, a1c) only match exactly after OCR folding, and only
|
| 128 |
+
# when the folded form is unambiguous ("Alc" -> a1c).
|
| 129 |
+
if len(n) < 3:
|
| 130 |
+
return None, 0.0 # never guess 1-2 letter tokens ("No", "N°" sample numbers)
|
| 131 |
+
keys = {k for a, k in idx.items() if len(a) < 4 and _fold(a) == _fold(n)}
|
| 132 |
+
return (keys.pop(), 90.0) if len(keys) == 1 else (None, 0.0)
|
| 133 |
+
folded = _folded_index()
|
| 134 |
+
hits = process.extract(_fold(n), list(folded), scorer=fuzz.ratio, limit=12)
|
| 135 |
+
best_per_key: dict[str, float] = {}
|
| 136 |
+
for alias, score, _ in hits:
|
| 137 |
+
for k in folded[alias]:
|
| 138 |
+
best_per_key[k] = max(best_per_key.get(k, 0.0), score)
|
| 139 |
+
ranked = sorted(best_per_key.items(), key=lambda kv: kv[1], reverse=True)
|
| 140 |
+
if not ranked or ranked[0][1] < min_score:
|
| 141 |
+
return None, ranked[0][1] if ranked else 0.0
|
| 142 |
+
if len(ranked) > 1 and ranked[0][1] - ranked[1][1] < margin:
|
| 143 |
+
return None, ranked[0][1]
|
| 144 |
+
return ranked[0][0], float(ranked[0][1])
|
lablens/data/catalog.yaml
ADDED
|
@@ -0,0 +1,407 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
# LabLens test catalog.
|
| 2 |
+
#
|
| 3 |
+
# Each test has:
|
| 4 |
+
# canonical_unit unit used internally for critical thresholds
|
| 5 |
+
# units accepted unit spellings -> multiply factor to canonical unit
|
| 6 |
+
# typical_range adult reference interval in canonical unit (used ONLY when the
|
| 7 |
+
# report prints no range of its own)
|
| 8 |
+
# critical life-threatening thresholds in canonical unit (deterministic escalation)
|
| 9 |
+
# plausible physiologically possible bounds; values outside are treated as a
|
| 10 |
+
# likely misread (e.g. a dropped decimal point) and sent for review
|
| 11 |
+
# aliases names as printed on reports (en / fr / ar), matched case-insensitively
|
| 12 |
+
# kb plain-language knowledge used to ground explanations
|
| 13 |
+
#
|
| 14 |
+
# Critical thresholds are conservative values drawn from commonly published hospital
|
| 15 |
+
# critical-value lists. They are a safety net, not a diagnosis, and every lab sets its own.
|
| 16 |
+
|
| 17 |
+
tests:
|
| 18 |
+
hemoglobin:
|
| 19 |
+
display: {en: Hemoglobin, fr: Hémoglobine, ar: الهيموغلوبين, darija: "الهيموغلوبين"}
|
| 20 |
+
category: blood_count
|
| 21 |
+
canonical_unit: g/dL
|
| 22 |
+
units: {"g/dl": 1, "g/l": 0.1, "mmol/l": 1.611}
|
| 23 |
+
typical_range: [12.0, 17.0]
|
| 24 |
+
critical: {low: 7.0, high: 20.0}
|
| 25 |
+
plausible: [2, 25]
|
| 26 |
+
aliases:
|
| 27 |
+
en: [hemoglobin, haemoglobin, hgb, hb]
|
| 28 |
+
fr: [hémoglobine, hemoglobine]
|
| 29 |
+
ar: [الهيموغلوبين, هيموغلوبين, خضاب الدم]
|
| 30 |
+
kb:
|
| 31 |
+
what: The protein in red blood cells that carries oxygen from the lungs to the rest of the body.
|
| 32 |
+
low: Often seen with anemia, which can come from iron deficiency, blood loss, low vitamin B12 or folate, or chronic illness. People may feel tired or short of breath.
|
| 33 |
+
high: Can be seen with dehydration, smoking, living at high altitude, or conditions where the body makes too many red cells.
|
| 34 |
+
|
| 35 |
+
hematocrit:
|
| 36 |
+
display: {en: Hematocrit, fr: Hématocrite, ar: الهيماتوكريت, darija: "الهيماتوكريت"}
|
| 37 |
+
category: blood_count
|
| 38 |
+
canonical_unit: "%"
|
| 39 |
+
units: {"%": 1, "l/l": 100}
|
| 40 |
+
typical_range: [36.0, 50.0]
|
| 41 |
+
critical: {low: 20.0, high: 60.0}
|
| 42 |
+
plausible: [8, 75]
|
| 43 |
+
aliases:
|
| 44 |
+
en: [hematocrit, haematocrit, hct]
|
| 45 |
+
fr: [hématocrite, hematocrite]
|
| 46 |
+
ar: [الهيماتوكريت, هيماتوكريت]
|
| 47 |
+
kb:
|
| 48 |
+
what: The share of your blood volume made up of red blood cells.
|
| 49 |
+
low: Usually moves together with hemoglobin and can point to anemia or blood loss.
|
| 50 |
+
high: Can reflect dehydration or an excess of red blood cells.
|
| 51 |
+
|
| 52 |
+
rbc:
|
| 53 |
+
display: {en: Red blood cells, fr: Globules rouges, ar: كريات الدم الحمراء, darija: "الكريات الحمرا"}
|
| 54 |
+
category: blood_count
|
| 55 |
+
canonical_unit: "10^6/µL"
|
| 56 |
+
units: {"10^6/µl": 1, "10^6/ul": 1, "10^6/mm3": 1, "10^12/l": 1, "t/l": 1, "m/µl": 1, "/mm3": 0.000001}
|
| 57 |
+
typical_range: [4.2, 5.9]
|
| 58 |
+
critical: {}
|
| 59 |
+
plausible: [1, 9]
|
| 60 |
+
aliases:
|
| 61 |
+
en: [red blood cells, red blood cell count, rbc, erythrocytes]
|
| 62 |
+
fr: [globules rouges, hématies, hematies, érythrocytes]
|
| 63 |
+
ar: [كريات الدم الحمراء, الكريات الحمراء]
|
| 64 |
+
kb:
|
| 65 |
+
what: The number of red blood cells, which carry oxygen.
|
| 66 |
+
low: Can be part of anemia.
|
| 67 |
+
high: Can be seen with dehydration or conditions that raise red cell production.
|
| 68 |
+
|
| 69 |
+
mcv:
|
| 70 |
+
display: {en: Mean corpuscular volume, fr: VGM, ar: متوسط حجم الكرية, darija: "حجم الكريات (VGM)"}
|
| 71 |
+
category: blood_count
|
| 72 |
+
canonical_unit: fL
|
| 73 |
+
units: {"fl": 1, "µm3": 1, "um3": 1}
|
| 74 |
+
typical_range: [80.0, 100.0]
|
| 75 |
+
critical: {}
|
| 76 |
+
plausible: [50, 140]
|
| 77 |
+
aliases:
|
| 78 |
+
en: [mcv, mean corpuscular volume, mean cell volume]
|
| 79 |
+
fr: [vgm, volume globulaire moyen]
|
| 80 |
+
ar: [متوسط حجم الكرية, حجم الكرية الوسطي]
|
| 81 |
+
kb:
|
| 82 |
+
what: The average size of your red blood cells.
|
| 83 |
+
low: Small red cells are often linked to iron deficiency or inherited conditions such as thalassemia trait.
|
| 84 |
+
high: Large red cells can be linked to low vitamin B12 or folate, alcohol use, some medicines, or thyroid and liver issues.
|
| 85 |
+
|
| 86 |
+
wbc:
|
| 87 |
+
display: {en: White blood cells, fr: Globules blancs, ar: كريات الدم البيضاء, darija: "الكريات البيض"}
|
| 88 |
+
category: blood_count
|
| 89 |
+
canonical_unit: "10^3/µL"
|
| 90 |
+
units: {"10^3/µl": 1, "10^3/ul": 1, "10^3/mm3": 1, "10^9/l": 1, "g/l": 1, "k/µl": 1, "/mm3": 0.001, "/µl": 0.001}
|
| 91 |
+
typical_range: [4.0, 11.0]
|
| 92 |
+
critical: {low: 2.0, high: 30.0}
|
| 93 |
+
plausible: [0.1, 300]
|
| 94 |
+
aliases:
|
| 95 |
+
en: [white blood cells, white blood cell count, wbc, leukocytes, leucocytes]
|
| 96 |
+
fr: [globules blancs, leucocytes]
|
| 97 |
+
ar: [كريات الدم البيضاء, الكريات البيضاء]
|
| 98 |
+
kb:
|
| 99 |
+
what: The cells of your immune system that fight infection.
|
| 100 |
+
low: Can follow viral infections, some medicines, or problems with bone-marrow production; very low counts raise infection risk.
|
| 101 |
+
high: Often a response to infection, inflammation, stress or steroids; less often a blood disorder.
|
| 102 |
+
|
| 103 |
+
platelets:
|
| 104 |
+
display: {en: Platelets, fr: Plaquettes, ar: الصفائح الدموية, darija: "الصفائح (plaquettes)"}
|
| 105 |
+
category: blood_count
|
| 106 |
+
canonical_unit: "10^3/µL"
|
| 107 |
+
units: {"10^3/µl": 1, "10^3/ul": 1, "10^3/mm3": 1, "10^9/l": 1, "g/l": 1, "k/µl": 1, "/mm3": 0.001, "/µl": 0.001}
|
| 108 |
+
typical_range: [150.0, 400.0]
|
| 109 |
+
critical: {low: 50.0, high: 1000.0}
|
| 110 |
+
plausible: [2, 2500]
|
| 111 |
+
aliases:
|
| 112 |
+
en: [platelets, platelet count, plt]
|
| 113 |
+
fr: [plaquettes, numération plaquettaire]
|
| 114 |
+
ar: [الصفائح الدموية, الصفائح]
|
| 115 |
+
kb:
|
| 116 |
+
what: Small cell fragments that help your blood clot.
|
| 117 |
+
low: Can follow infections, some medicines, or immune or bone-marrow conditions; very low counts raise bleeding risk.
|
| 118 |
+
high: Often a reaction to inflammation, infection or iron deficiency.
|
| 119 |
+
|
| 120 |
+
glucose:
|
| 121 |
+
display: {en: Fasting glucose, fr: Glycémie à jeun, ar: سكر الدم الصائم, darija: "السكر فالدم (على الريق)"}
|
| 122 |
+
category: metabolic
|
| 123 |
+
canonical_unit: mg/dL
|
| 124 |
+
units: {"mg/dl": 1, "g/l": 100, "mmol/l": 18.016}
|
| 125 |
+
typical_range: [70.0, 99.0]
|
| 126 |
+
critical: {low: 50.0, high: 400.0}
|
| 127 |
+
plausible: [10, 1500]
|
| 128 |
+
aliases:
|
| 129 |
+
en: [fasting glucose, glucose, fasting blood sugar, blood glucose]
|
| 130 |
+
fr: [glycémie à jeun, glycemie a jeun, glycémie, glycemie, glucose]
|
| 131 |
+
ar: [سكر الدم الصائم, سكر الدم, الغلوكوز]
|
| 132 |
+
kb:
|
| 133 |
+
what: The amount of sugar in your blood after not eating, usually for 8 hours or more.
|
| 134 |
+
low: Can happen with some diabetes medicines, long fasting or, rarely, hormone problems.
|
| 135 |
+
high: Can point to prediabetes or diabetes, especially if repeated; stress, illness and some medicines can also raise it.
|
| 136 |
+
|
| 137 |
+
hba1c:
|
| 138 |
+
display: {en: HbA1c, fr: Hémoglobine glyquée, ar: الهيموغلوبين السكري, darija: "السكر التراكمي (HbA1c)"}
|
| 139 |
+
category: metabolic
|
| 140 |
+
canonical_unit: "%"
|
| 141 |
+
units: {"%": 1}
|
| 142 |
+
typical_range: [4.0, 5.6]
|
| 143 |
+
critical: {}
|
| 144 |
+
plausible: [3, 20]
|
| 145 |
+
aliases:
|
| 146 |
+
en: [hba1c, a1c, glycated hemoglobin, glycated haemoglobin]
|
| 147 |
+
fr: [hémoglobine glyquée, hemoglobine glyquee, hba1c]
|
| 148 |
+
ar: [الهيموغلوبين السكري, السكر التراكمي]
|
| 149 |
+
kb:
|
| 150 |
+
what: Your average blood sugar over the past two to three months.
|
| 151 |
+
low: Rarely a concern on its own; can be affected by anemia or recent blood loss.
|
| 152 |
+
high: 5.7 to 6.4% is commonly called the prediabetes range and 6.5% or above is a threshold used for diagnosing diabetes, which a doctor confirms.
|
| 153 |
+
|
| 154 |
+
creatinine:
|
| 155 |
+
display: {en: Creatinine, fr: Créatinine, ar: الكرياتينين, darija: "الكرياتينين"}
|
| 156 |
+
category: kidney
|
| 157 |
+
canonical_unit: mg/dL
|
| 158 |
+
units: {"mg/dl": 1, "µmol/l": 0.01131, "umol/l": 0.01131, "mg/l": 0.1}
|
| 159 |
+
typical_range: [0.6, 1.2]
|
| 160 |
+
critical: {}
|
| 161 |
+
plausible: [0.1, 25]
|
| 162 |
+
aliases:
|
| 163 |
+
en: [creatinine, serum creatinine]
|
| 164 |
+
fr: [créatinine, creatinine, créatininémie]
|
| 165 |
+
ar: [الكرياتينين, كرياتينين]
|
| 166 |
+
kb:
|
| 167 |
+
what: A waste product from muscles that the kidneys filter out; it is used to estimate kidney function.
|
| 168 |
+
low: Usually not a concern; often seen with low muscle mass.
|
| 169 |
+
high: Can mean the kidneys are filtering less well; dehydration, high muscle mass and some medicines can also raise it.
|
| 170 |
+
|
| 171 |
+
urea:
|
| 172 |
+
display: {en: Urea, fr: Urée, ar: اليوريا, darija: "اليوريا (urée)"}
|
| 173 |
+
category: kidney
|
| 174 |
+
canonical_unit: mg/dL
|
| 175 |
+
units: {"mg/dl": 1, "g/l": 100, "mmol/l": 6.006}
|
| 176 |
+
typical_range: [15.0, 45.0]
|
| 177 |
+
critical: {}
|
| 178 |
+
plausible: [2, 400]
|
| 179 |
+
aliases:
|
| 180 |
+
en: [urea, blood urea]
|
| 181 |
+
fr: [urée, uree, urémie]
|
| 182 |
+
ar: [اليوريا, البولة]
|
| 183 |
+
kb:
|
| 184 |
+
what: A waste product from protein breakdown that the kidneys remove.
|
| 185 |
+
low: Can be seen with low protein intake or liver conditions.
|
| 186 |
+
high: Can reflect dehydration, a high-protein diet, or reduced kidney function.
|
| 187 |
+
|
| 188 |
+
sodium:
|
| 189 |
+
display: {en: Sodium, fr: Sodium, ar: الصوديوم, darija: "الصوديوم"}
|
| 190 |
+
category: electrolytes
|
| 191 |
+
canonical_unit: mmol/L
|
| 192 |
+
units: {"mmol/l": 1, "meq/l": 1}
|
| 193 |
+
typical_range: [135.0, 145.0]
|
| 194 |
+
critical: {low: 120.0, high: 160.0}
|
| 195 |
+
plausible: [95, 190]
|
| 196 |
+
aliases:
|
| 197 |
+
en: [sodium, na]
|
| 198 |
+
fr: [sodium, natrémie, natremie, na+]
|
| 199 |
+
ar: [الصوديوم]
|
| 200 |
+
kb:
|
| 201 |
+
what: A salt that keeps your body's water balance and nerves working.
|
| 202 |
+
low: Can come from drinking a lot of water, some diuretics, vomiting or hormone and heart conditions.
|
| 203 |
+
high: Usually reflects dehydration or not drinking enough.
|
| 204 |
+
|
| 205 |
+
potassium:
|
| 206 |
+
display: {en: Potassium, fr: Potassium, ar: البوتاسيوم, darija: "البوتاسيوم"}
|
| 207 |
+
category: electrolytes
|
| 208 |
+
canonical_unit: mmol/L
|
| 209 |
+
units: {"mmol/l": 1, "meq/l": 1}
|
| 210 |
+
typical_range: [3.5, 5.1]
|
| 211 |
+
critical: {low: 2.8, high: 6.2}
|
| 212 |
+
plausible: [1.2, 10]
|
| 213 |
+
aliases:
|
| 214 |
+
en: [potassium, k]
|
| 215 |
+
fr: [potassium, kaliémie, kaliemie, k+]
|
| 216 |
+
ar: [البوتاسيوم]
|
| 217 |
+
kb:
|
| 218 |
+
what: A mineral essential for your heart rhythm and muscles.
|
| 219 |
+
low: Can come from vomiting, diarrhea or some diuretics.
|
| 220 |
+
high: Can come from kidney problems or some medicines; a sample that was handled roughly can also read falsely high.
|
| 221 |
+
|
| 222 |
+
calcium:
|
| 223 |
+
display: {en: Calcium, fr: Calcium, ar: الكالسيوم, darija: "الكالسيوم"}
|
| 224 |
+
category: electrolytes
|
| 225 |
+
canonical_unit: mg/dL
|
| 226 |
+
units: {"mg/dl": 1, "mmol/l": 4.008, "mg/l": 0.1}
|
| 227 |
+
typical_range: [8.5, 10.5]
|
| 228 |
+
critical: {low: 6.5, high: 13.0}
|
| 229 |
+
plausible: [3, 20]
|
| 230 |
+
aliases:
|
| 231 |
+
en: [calcium, total calcium, ca]
|
| 232 |
+
fr: [calcium, calcémie, calcemie]
|
| 233 |
+
ar: [الكالسيوم]
|
| 234 |
+
kb:
|
| 235 |
+
what: A mineral for bones, muscles, nerves and the heart.
|
| 236 |
+
low: Can be linked to low vitamin D, low albumin or parathyroid conditions.
|
| 237 |
+
high: Can be linked to parathyroid conditions, too much vitamin D or calcium supplements, or other conditions a doctor should look at.
|
| 238 |
+
|
| 239 |
+
cholesterol_total:
|
| 240 |
+
display: {en: Total cholesterol, fr: Cholestérol total, ar: الكوليسترول الكلي, darija: "الكوليسترول الكل"}
|
| 241 |
+
category: lipids
|
| 242 |
+
canonical_unit: mg/dL
|
| 243 |
+
units: {"mg/dl": 1, "g/l": 100, "mmol/l": 38.67}
|
| 244 |
+
typical_range: [0.0, 200.0]
|
| 245 |
+
critical: {}
|
| 246 |
+
plausible: [40, 1000]
|
| 247 |
+
aliases:
|
| 248 |
+
en: [total cholesterol, cholesterol]
|
| 249 |
+
fr: [cholestérol total, cholesterol total, cholestérol]
|
| 250 |
+
ar: [الكوليسترول الكلي, الكوليسترول]
|
| 251 |
+
kb:
|
| 252 |
+
what: All the cholesterol carried in your blood.
|
| 253 |
+
low: Rarely a concern.
|
| 254 |
+
high: Raises long-term heart and blood-vessel risk, especially together with high LDL; diet, activity, genetics and medicines all play a role.
|
| 255 |
+
|
| 256 |
+
ldl:
|
| 257 |
+
display: {en: LDL cholesterol, fr: LDL-cholestérol, ar: الكوليسترول الضار, darija: "الكوليسترول الخايب (LDL)"}
|
| 258 |
+
category: lipids
|
| 259 |
+
canonical_unit: mg/dL
|
| 260 |
+
units: {"mg/dl": 1, "g/l": 100, "mmol/l": 38.67}
|
| 261 |
+
typical_range: [0.0, 130.0]
|
| 262 |
+
critical: {}
|
| 263 |
+
plausible: [10, 600]
|
| 264 |
+
aliases:
|
| 265 |
+
en: [ldl cholesterol, ldl, ldl-c]
|
| 266 |
+
fr: [ldl-cholestérol, ldl cholestérol, ldl-cholesterol, cholestérol ldl, ldl]
|
| 267 |
+
ar: [الكوليسترول الضار, كوليسترول ldl]
|
| 268 |
+
kb:
|
| 269 |
+
what: Often called "bad" cholesterol because it can build up in artery walls.
|
| 270 |
+
low: Generally fine.
|
| 271 |
+
high: Raises heart and stroke risk over time; the right target depends on your overall risk, which a doctor assesses.
|
| 272 |
+
|
| 273 |
+
hdl:
|
| 274 |
+
display: {en: HDL cholesterol, fr: HDL-cholestérol, ar: الكوليسترول النافع, darija: "الكوليسترول الباهي (HDL)"}
|
| 275 |
+
category: lipids
|
| 276 |
+
canonical_unit: mg/dL
|
| 277 |
+
units: {"mg/dl": 1, "g/l": 100, "mmol/l": 38.67}
|
| 278 |
+
typical_range: [40.0, 100.0]
|
| 279 |
+
critical: {}
|
| 280 |
+
plausible: [5, 150]
|
| 281 |
+
aliases:
|
| 282 |
+
en: [hdl cholesterol, hdl, hdl-c]
|
| 283 |
+
fr: [hdl-cholestérol, hdl cholestérol, hdl-cholesterol, cholestérol hdl, hdl]
|
| 284 |
+
ar: [الكوليسترول النافع, كوليسترول hdl]
|
| 285 |
+
kb:
|
| 286 |
+
what: Often called "good" cholesterol because it helps carry cholesterol away from arteries.
|
| 287 |
+
low: Associated with higher heart risk; exercise and not smoking tend to raise it.
|
| 288 |
+
high: Generally considered favorable.
|
| 289 |
+
|
| 290 |
+
triglycerides:
|
| 291 |
+
display: {en: Triglycerides, fr: Triglycérides, ar: الدهون الثلاثية, darija: "التريغليسيريد"}
|
| 292 |
+
category: lipids
|
| 293 |
+
canonical_unit: mg/dL
|
| 294 |
+
units: {"mg/dl": 1, "g/l": 100, "mmol/l": 88.57}
|
| 295 |
+
typical_range: [0.0, 150.0]
|
| 296 |
+
critical: {high: 1000.0}
|
| 297 |
+
plausible: [10, 10000]
|
| 298 |
+
aliases:
|
| 299 |
+
en: [triglycerides, tg]
|
| 300 |
+
fr: [triglycérides, triglycerides]
|
| 301 |
+
ar: [الدهون الثلاثية, الشحوم الثلاثية]
|
| 302 |
+
kb:
|
| 303 |
+
what: A type of fat in the blood that stores energy from food.
|
| 304 |
+
low: Rarely a concern.
|
| 305 |
+
high: Can be linked to diet, alcohol, weight, diabetes or genetics; very high levels raise the risk of pancreatitis.
|
| 306 |
+
|
| 307 |
+
alt:
|
| 308 |
+
display: {en: ALT, fr: ALAT (TGP), ar: إنزيم ALT, darija: "ALAT (الكبدة)"}
|
| 309 |
+
category: liver
|
| 310 |
+
canonical_unit: U/L
|
| 311 |
+
units: {"u/l": 1, "ui/l": 1, "iu/l": 1}
|
| 312 |
+
typical_range: [7.0, 55.0]
|
| 313 |
+
critical: {}
|
| 314 |
+
plausible: [1, 5000]
|
| 315 |
+
aliases:
|
| 316 |
+
en: [alt, alanine aminotransferase, sgpt]
|
| 317 |
+
fr: [alat, tgp, alanine aminotransférase, alat (tgp)]
|
| 318 |
+
ar: [إنزيم alt, ناقلة أمين الألانين]
|
| 319 |
+
kb:
|
| 320 |
+
what: A liver enzyme; it leaks into the blood when liver cells are irritated or damaged.
|
| 321 |
+
low: Not a concern.
|
| 322 |
+
high: Can be linked to fatty liver, alcohol, some medicines, hepatitis or recent intense exercise.
|
| 323 |
+
|
| 324 |
+
ast:
|
| 325 |
+
display: {en: AST, fr: ASAT (TGO), ar: إنزيم AST, darija: "ASAT (الكبدة)"}
|
| 326 |
+
category: liver
|
| 327 |
+
canonical_unit: U/L
|
| 328 |
+
units: {"u/l": 1, "ui/l": 1, "iu/l": 1}
|
| 329 |
+
typical_range: [8.0, 48.0]
|
| 330 |
+
critical: {}
|
| 331 |
+
plausible: [1, 5000]
|
| 332 |
+
aliases:
|
| 333 |
+
en: [ast, aspartate aminotransferase, sgot]
|
| 334 |
+
fr: [asat, tgo, aspartate aminotransférase, asat (tgo)]
|
| 335 |
+
ar: [إنزيم ast, ناقلة أمين الأسبارتات]
|
| 336 |
+
kb:
|
| 337 |
+
what: An enzyme found in the liver, heart and muscles.
|
| 338 |
+
low: Not a concern.
|
| 339 |
+
high: Can be linked to liver irritation, muscle injury or strenuous exercise; it is read together with ALT.
|
| 340 |
+
|
| 341 |
+
tsh:
|
| 342 |
+
display: {en: TSH, fr: TSH, ar: الهرمون المنبه للدرقية, darija: "TSH (الغدة الدرقية)"}
|
| 343 |
+
category: thyroid
|
| 344 |
+
canonical_unit: mIU/L
|
| 345 |
+
units: {"miu/l": 1, "µui/ml": 1, "uui/ml": 1, "µiu/ml": 1, "uiu/ml": 1, "mui/l": 1}
|
| 346 |
+
typical_range: [0.4, 4.0]
|
| 347 |
+
critical: {}
|
| 348 |
+
plausible: [0.001, 200]
|
| 349 |
+
aliases:
|
| 350 |
+
en: [tsh, thyroid stimulating hormone, thyrotropin]
|
| 351 |
+
fr: [tsh, thyréostimuline, tsh ultrasensible, tshus]
|
| 352 |
+
ar: [الهرمون المنبه للدرقية, هرمون tsh]
|
| 353 |
+
kb:
|
| 354 |
+
what: A hormone from the brain that tells the thyroid how hard to work.
|
| 355 |
+
low: Can point to an overactive thyroid or too high a dose of thyroid medicine.
|
| 356 |
+
high: Can point to an underactive thyroid.
|
| 357 |
+
|
| 358 |
+
ferritin:
|
| 359 |
+
display: {en: Ferritin, fr: Ferritine, ar: الفيريتين, darija: "الفيريتين (مخزون الحديد)"}
|
| 360 |
+
category: iron
|
| 361 |
+
canonical_unit: ng/mL
|
| 362 |
+
units: {"ng/ml": 1, "µg/l": 1, "ug/l": 1}
|
| 363 |
+
typical_range: [15.0, 200.0]
|
| 364 |
+
critical: {}
|
| 365 |
+
plausible: [1, 10000]
|
| 366 |
+
aliases:
|
| 367 |
+
en: [ferritin, serum ferritin]
|
| 368 |
+
fr: [ferritine, ferritinémie]
|
| 369 |
+
ar: [الفيريتين, مخزون الحديد]
|
| 370 |
+
kb:
|
| 371 |
+
what: Reflects how much iron your body has in storage.
|
| 372 |
+
low: The most common sign of iron deficiency, even before anemia appears.
|
| 373 |
+
high: Can be raised by inflammation, infection, liver conditions or iron overload.
|
| 374 |
+
|
| 375 |
+
crp:
|
| 376 |
+
display: {en: C-reactive protein, fr: CRP, ar: البروتين المتفاعل C, darija: "CRP (الالتهاب)"}
|
| 377 |
+
category: inflammation
|
| 378 |
+
canonical_unit: mg/L
|
| 379 |
+
units: {"mg/l": 1, "mg/dl": 10}
|
| 380 |
+
typical_range: [0.0, 5.0]
|
| 381 |
+
critical: {}
|
| 382 |
+
plausible: [0, 600]
|
| 383 |
+
aliases:
|
| 384 |
+
en: [c-reactive protein, crp, c reactive protein]
|
| 385 |
+
fr: [crp, protéine c-réactive, protéine c réactive]
|
| 386 |
+
ar: [البروتين المتفاعل c, البروتين التفاعلي c]
|
| 387 |
+
kb:
|
| 388 |
+
what: A protein the liver makes when there is inflammation in the body.
|
| 389 |
+
low: Normal.
|
| 390 |
+
high: Signals inflammation or infection somewhere in the body; it does not say where or why.
|
| 391 |
+
|
| 392 |
+
vitamin_d:
|
| 393 |
+
display: {en: Vitamin D, fr: Vitamine D, ar: فيتامين د, darija: "فيتامين D"}
|
| 394 |
+
category: vitamins
|
| 395 |
+
canonical_unit: ng/mL
|
| 396 |
+
units: {"ng/ml": 1, "nmol/l": 0.4006, "µg/l": 1, "ug/l": 1}
|
| 397 |
+
typical_range: [30.0, 100.0]
|
| 398 |
+
critical: {}
|
| 399 |
+
plausible: [2, 250]
|
| 400 |
+
aliases:
|
| 401 |
+
en: [vitamin d, 25-oh vitamin d, 25(oh)d, 25-hydroxyvitamin d]
|
| 402 |
+
fr: [vitamine d, 25-oh vitamine d, 25(oh)d, vitamine d totale]
|
| 403 |
+
ar: [فيتامين د, فيتامين d]
|
| 404 |
+
kb:
|
| 405 |
+
what: A vitamin your skin makes in sunlight; it helps your body use calcium for bones.
|
| 406 |
+
low: Very common, including in sunny countries; can affect bones and muscles.
|
| 407 |
+
high: Usually from taking too many supplements.
|
lablens/evaluate.py
ADDED
|
@@ -0,0 +1,122 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""Benchmark the extraction + flagging pipeline against synthetic ground truth.
|
| 2 |
+
|
| 3 |
+
Metrics (per input type: PDF text layer vs degraded photo, and per locale):
|
| 4 |
+
recall share of true test rows that were found
|
| 5 |
+
precision share of extracted rows that are real rows of the report
|
| 6 |
+
value_acc exact value match among found rows
|
| 7 |
+
unit_acc unit understood as the same unit (same conversion factor)
|
| 8 |
+
range_acc reference range read exactly
|
| 9 |
+
flag_acc end-to-end flag equals the ground-truth flag (missing rows count as wrong)
|
| 10 |
+
critical_recall critical rows that came out as critical (the safety metric)
|
| 11 |
+
dangerous_misses abnormal rows reported as NORMAL (worse than missing: false reassurance)
|
| 12 |
+
"""
|
| 13 |
+
from __future__ import annotations
|
| 14 |
+
|
| 15 |
+
import argparse
|
| 16 |
+
import json
|
| 17 |
+
from collections import defaultdict
|
| 18 |
+
from pathlib import Path
|
| 19 |
+
|
| 20 |
+
from .catalog import load_catalog
|
| 21 |
+
from .extract import extract
|
| 22 |
+
from .flagger import flag_all
|
| 23 |
+
|
| 24 |
+
|
| 25 |
+
def _same_unit(key: str, a: str | None, b: str | None) -> bool:
|
| 26 |
+
td = load_catalog()[key]
|
| 27 |
+
fa, fb = td.to_canonical(1.0, a), td.to_canonical(1.0, b)
|
| 28 |
+
return fa is not None and fb is not None and abs(fa - fb) < 1e-9
|
| 29 |
+
|
| 30 |
+
|
| 31 |
+
def score_report(truth: dict, path: Path) -> dict:
|
| 32 |
+
results, _, _, _ = extract(path)
|
| 33 |
+
analysis = flag_all(results)
|
| 34 |
+
got = {r.test_key: r for r in analysis.results}
|
| 35 |
+
true_keys = {row["test_key"] for row in truth["rows"]}
|
| 36 |
+
|
| 37 |
+
c = defaultdict(int)
|
| 38 |
+
c["rows"] = len(truth["rows"])
|
| 39 |
+
c["extracted"] = len(got)
|
| 40 |
+
c["true_positive_keys"] = len(true_keys & set(got))
|
| 41 |
+
for row in truth["rows"]:
|
| 42 |
+
k, exp = row["test_key"], row["expected_flag"]
|
| 43 |
+
crit = exp.startswith("critical")
|
| 44 |
+
c["critical_rows"] += crit
|
| 45 |
+
c["abnormal_rows"] += exp != "normal"
|
| 46 |
+
g = got.get(k)
|
| 47 |
+
if g is None:
|
| 48 |
+
continue
|
| 49 |
+
c["found"] += 1
|
| 50 |
+
c["value_ok"] += abs(g.value - row["value"]) < 1e-9
|
| 51 |
+
c["unit_ok"] += _same_unit(k, g.unit, row["unit"])
|
| 52 |
+
c["range_ok"] += (g.ref_low, g.ref_high) == (row["ref_low"], row["ref_high"]) or (
|
| 53 |
+
g.ref_source == "catalog" and row["ref_low"] is None and row["ref_high"] is None)
|
| 54 |
+
c["flag_ok"] += g.flag.value == exp
|
| 55 |
+
c["critical_caught"] += crit and g.flag.value.startswith("critical")
|
| 56 |
+
c["dangerous_miss"] += exp != "normal" and g.flag.value == "normal"
|
| 57 |
+
return dict(c)
|
| 58 |
+
|
| 59 |
+
|
| 60 |
+
def summarize(counts: list[dict]) -> dict:
|
| 61 |
+
t = defaultdict(int)
|
| 62 |
+
for c in counts:
|
| 63 |
+
for k, v in c.items():
|
| 64 |
+
t[k] += v
|
| 65 |
+
|
| 66 |
+
def ratio(a, b):
|
| 67 |
+
return round(t[a] / t[b], 4) if t[b] else None
|
| 68 |
+
|
| 69 |
+
return {
|
| 70 |
+
"reports": len(counts), "rows": t["rows"],
|
| 71 |
+
"recall": ratio("found", "rows"),
|
| 72 |
+
"precision": ratio("true_positive_keys", "extracted"),
|
| 73 |
+
"value_acc": ratio("value_ok", "found"),
|
| 74 |
+
"unit_acc": ratio("unit_ok", "found"),
|
| 75 |
+
"range_acc": ratio("range_ok", "found"),
|
| 76 |
+
"flag_acc": ratio("flag_ok", "rows"),
|
| 77 |
+
"critical_recall": ratio("critical_caught", "critical_rows"),
|
| 78 |
+
"critical_rows": t["critical_rows"],
|
| 79 |
+
"dangerous_misses": t["dangerous_miss"],
|
| 80 |
+
"abnormal_rows": t["abnormal_rows"],
|
| 81 |
+
}
|
| 82 |
+
|
| 83 |
+
|
| 84 |
+
def run(bench_dir: Path) -> dict:
|
| 85 |
+
groups: dict[str, list[dict]] = defaultdict(list)
|
| 86 |
+
for jf in sorted(bench_dir.glob("rpt_*.json")):
|
| 87 |
+
truth = json.loads(jf.read_text(encoding="utf-8"))
|
| 88 |
+
for kind, field in (("pdf", "pdf"), ("photo", "image")):
|
| 89 |
+
if field not in truth:
|
| 90 |
+
continue
|
| 91 |
+
c = score_report(truth, bench_dir / truth[field])
|
| 92 |
+
groups[f"{kind}/all"].append(c)
|
| 93 |
+
groups[f"{kind}/{truth['locale']}"].append(c)
|
| 94 |
+
return {g: summarize(v) for g, v in sorted(groups.items())}
|
| 95 |
+
|
| 96 |
+
|
| 97 |
+
def to_markdown(summary: dict) -> str:
|
| 98 |
+
cols = ["reports", "rows", "recall", "precision", "value_acc", "unit_acc", "range_acc",
|
| 99 |
+
"flag_acc", "critical_recall", "dangerous_misses"]
|
| 100 |
+
out = ["| split | " + " | ".join(cols) + " |", "|" + "---|" * (len(cols) + 1)]
|
| 101 |
+
for g, s in summary.items():
|
| 102 |
+
cells = []
|
| 103 |
+
for k in cols:
|
| 104 |
+
v = s[k]
|
| 105 |
+
if isinstance(v, float) and k not in ("reports", "rows", "dangerous_misses"):
|
| 106 |
+
cells.append(f"{v:.1%}")
|
| 107 |
+
elif k == "critical_recall" and v is None:
|
| 108 |
+
cells.append("n/a")
|
| 109 |
+
else:
|
| 110 |
+
cells.append(str(v))
|
| 111 |
+
out.append(f"| {g} | " + " | ".join(cells) + " |")
|
| 112 |
+
return "\n".join(out)
|
| 113 |
+
|
| 114 |
+
|
| 115 |
+
if __name__ == "__main__":
|
| 116 |
+
ap = argparse.ArgumentParser()
|
| 117 |
+
ap.add_argument("--bench", default="benchmark/reports")
|
| 118 |
+
ap.add_argument("--out", default="benchmark/results.json")
|
| 119 |
+
a = ap.parse_args()
|
| 120 |
+
summary = run(Path(a.bench))
|
| 121 |
+
Path(a.out).write_text(json.dumps(summary, indent=2), encoding="utf-8")
|
| 122 |
+
print(to_markdown(summary))
|
lablens/explain.py
ADDED
|
@@ -0,0 +1,186 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""Plain-language explanations: hosted open LLM, grounded and guarded, with a template fallback.
|
| 2 |
+
|
| 3 |
+
Privacy by construction: the model only ever receives test names, values, units, ranges and
|
| 4 |
+
the deterministic flags. The report image, patient name, lab, dates and any other text on
|
| 5 |
+
the page never leave the extraction step.
|
| 6 |
+
|
| 7 |
+
Safety by construction: the model does not decide what is abnormal or urgent. Flags and the
|
| 8 |
+
urgent banner are computed by `flagger.py` and rendered from fixed strings in `i18n.py`.
|
| 9 |
+
The model's output is checked for diagnosis/treatment claims and replaced by the template
|
| 10 |
+
if it fails.
|
| 11 |
+
"""
|
| 12 |
+
from __future__ import annotations
|
| 13 |
+
|
| 14 |
+
import json
|
| 15 |
+
import os
|
| 16 |
+
import re
|
| 17 |
+
|
| 18 |
+
from .catalog import load_catalog
|
| 19 |
+
from .i18n import FLAG_LABEL, HEADINGS, LANGS, TEMPLATE_QUESTION, t
|
| 20 |
+
from .models import Analysis, Flag
|
| 21 |
+
|
| 22 |
+
DEFAULT_BASE_URL = "https://router.huggingface.co/v1"
|
| 23 |
+
DEFAULT_MODEL = "Qwen/Qwen2.5-72B-Instruct"
|
| 24 |
+
|
| 25 |
+
LANG_INSTRUCTIONS = {
|
| 26 |
+
"en": "Write in simple English, at the level of a 12-year-old reader.",
|
| 27 |
+
"fr": "Écris en français simple, compréhensible par quelqu'un sans formation médicale.",
|
| 28 |
+
"ar": "اكتب بالعربية الفصحى المبسّطة التي يفهمها أي شخص.",
|
| 29 |
+
"darija": ("Write in Tunisian Arabic (Darija/Tounsi) in Arabic script, the way people speak in Tunis or "
|
| 30 |
+
"Sousse. Keep common French medical words as Tunisians say them (analyse, cholestérol). "
|
| 31 |
+
"Do not write Modern Standard Arabic."),
|
| 32 |
+
}
|
| 33 |
+
|
| 34 |
+
SYSTEM_PROMPT = """You are LabLens, a careful assistant that explains blood test results to patients in plain language.
|
| 35 |
+
|
| 36 |
+
Hard rules:
|
| 37 |
+
1. You do NOT diagnose. Never say the person "has" a disease. Use "can be linked to", "is sometimes seen with".
|
| 38 |
+
2. You do NOT recommend medicines, doses, supplements or stopping any treatment.
|
| 39 |
+
3. The status of each result (within range, low, high, very low/high, check) is FINAL and given to you. Never contradict it or re-judge a value yourself.
|
| 40 |
+
4. Only talk about the tests listed in the input. Base explanations on the provided knowledge notes; do not add other medical claims.
|
| 41 |
+
5. Be calm and kind. Abnormal results are common and often minor, but do not dismiss them.
|
| 42 |
+
6. Keep it short: one or two sentences per result.
|
| 43 |
+
|
| 44 |
+
Return ONLY valid JSON with this shape:
|
| 45 |
+
{"summary": "2-3 sentences", "results": [{"key": "<test key>", "explanation": "..."}], "questions": ["3 to 5 short questions to ask a doctor"]}
|
| 46 |
+
"""
|
| 47 |
+
|
| 48 |
+
# Patterns that indicate diagnosis or treatment advice; any hit triggers the template fallback.
|
| 49 |
+
BANNED = [
|
| 50 |
+
r"\byou have (?:a |an )?(?:diabetes|anemia|anaemia|cancer|leukemia|kidney disease|hypothyroidism|hyperthyroidism|infection)\b",
|
| 51 |
+
r"\byou are (?:diabetic|anemic|anaemic)\b",
|
| 52 |
+
r"\bvous (?:avez|êtes) (?:le |un |une |du )?(?:diabète|diabétique|anémie|anémique|cancer|leucémie|infection)\b",
|
| 53 |
+
r"\b(?:take|prenez|prends)\s+\d+\s*(?:mg|g|µg|ui|iu)\b",
|
| 54 |
+
r"\b(?:stop|arrêtez|arrête)\s+(?:taking|your|votre|ton|de prendre)\b",
|
| 55 |
+
r"\bdiagnos(?:is|ed)\s+(?:of|with)\b",
|
| 56 |
+
]
|
| 57 |
+
|
| 58 |
+
|
| 59 |
+
def _payload(analysis: Analysis, lang: str) -> list[dict]:
|
| 60 |
+
catalog = load_catalog()
|
| 61 |
+
items = []
|
| 62 |
+
for r in analysis.results:
|
| 63 |
+
td = catalog.get(r.test_key or "")
|
| 64 |
+
if not td:
|
| 65 |
+
continue
|
| 66 |
+
items.append({
|
| 67 |
+
"key": td.key,
|
| 68 |
+
"name": td.display.get(lang) or td.display["en"],
|
| 69 |
+
"value": r.value, "unit": r.unit,
|
| 70 |
+
"reference_range": [r.ref_low, r.ref_high],
|
| 71 |
+
"range_is_generic": r.ref_source == "catalog",
|
| 72 |
+
"status": r.flag.value,
|
| 73 |
+
"knowledge": {"what": td.kb["what"],
|
| 74 |
+
"if_low": td.kb["low"] if r.flag in (Flag.LOW, Flag.CRITICAL_LOW) else None,
|
| 75 |
+
"if_high": td.kb["high"] if r.flag in (Flag.HIGH, Flag.CRITICAL_HIGH) else None},
|
| 76 |
+
})
|
| 77 |
+
return items
|
| 78 |
+
|
| 79 |
+
|
| 80 |
+
def violates_guardrails(text: str) -> str | None:
|
| 81 |
+
low = text.lower()
|
| 82 |
+
for pat in BANNED:
|
| 83 |
+
if re.search(pat, low):
|
| 84 |
+
return pat
|
| 85 |
+
return None
|
| 86 |
+
|
| 87 |
+
|
| 88 |
+
def llm_available() -> bool:
|
| 89 |
+
return bool(os.getenv("LLM_API_KEY") or os.getenv("HF_TOKEN"))
|
| 90 |
+
|
| 91 |
+
|
| 92 |
+
def explain_with_llm(analysis: Analysis, lang: str, timeout: float = 45.0) -> dict:
|
| 93 |
+
from openai import OpenAI
|
| 94 |
+
|
| 95 |
+
client = OpenAI(
|
| 96 |
+
base_url=os.getenv("LLM_BASE_URL", DEFAULT_BASE_URL),
|
| 97 |
+
api_key=os.getenv("LLM_API_KEY") or os.getenv("HF_TOKEN"),
|
| 98 |
+
timeout=timeout,
|
| 99 |
+
)
|
| 100 |
+
items = _payload(analysis, lang)
|
| 101 |
+
user = (f"Language: {LANGS.get(lang, 'English')}. {LANG_INSTRUCTIONS.get(lang, '')}\n\n"
|
| 102 |
+
f"Results (JSON):\n{json.dumps(items, ensure_ascii=False)}")
|
| 103 |
+
resp = client.chat.completions.create(
|
| 104 |
+
model=os.getenv("LLM_MODEL", DEFAULT_MODEL),
|
| 105 |
+
messages=[{"role": "system", "content": SYSTEM_PROMPT}, {"role": "user", "content": user}],
|
| 106 |
+
temperature=0.2,
|
| 107 |
+
max_tokens=1400,
|
| 108 |
+
)
|
| 109 |
+
text = resp.choices[0].message.content or ""
|
| 110 |
+
m = re.search(r"\{.*\}", text, re.S)
|
| 111 |
+
if not m:
|
| 112 |
+
raise ValueError("model did not return JSON")
|
| 113 |
+
data = json.loads(m.group(0))
|
| 114 |
+
valid_keys = {i["key"] for i in items}
|
| 115 |
+
data["results"] = [x for x in data.get("results", []) if x.get("key") in valid_keys]
|
| 116 |
+
flat = " ".join([data.get("summary", "")] + [x.get("explanation", "") for x in data["results"]]
|
| 117 |
+
+ list(data.get("questions", [])))
|
| 118 |
+
hit = violates_guardrails(flat)
|
| 119 |
+
if hit:
|
| 120 |
+
raise ValueError(f"guardrail: {hit}")
|
| 121 |
+
return data
|
| 122 |
+
|
| 123 |
+
|
| 124 |
+
def explain_with_template(analysis: Analysis, lang: str) -> dict:
|
| 125 |
+
"""Deterministic fallback. Explanations come straight from the knowledge base (English)."""
|
| 126 |
+
catalog = load_catalog()
|
| 127 |
+
abnormal = [r for r in analysis.results if r.flag not in (Flag.NORMAL, Flag.UNKNOWN)]
|
| 128 |
+
results, questions = [], []
|
| 129 |
+
for r in analysis.results:
|
| 130 |
+
td = catalog.get(r.test_key or "")
|
| 131 |
+
if not td:
|
| 132 |
+
continue
|
| 133 |
+
parts = [td.kb["what"]]
|
| 134 |
+
if r.flag in (Flag.LOW, Flag.CRITICAL_LOW):
|
| 135 |
+
parts.append(td.kb["low"])
|
| 136 |
+
elif r.flag in (Flag.HIGH, Flag.CRITICAL_HIGH):
|
| 137 |
+
parts.append(td.kb["high"])
|
| 138 |
+
results.append({"key": td.key, "explanation": " ".join(parts)})
|
| 139 |
+
if r in abnormal and r.flag != Flag.CHECK and len(questions) < 5:
|
| 140 |
+
questions.append(t(TEMPLATE_QUESTION, lang).format(
|
| 141 |
+
name=td.display.get(lang) or td.display["en"],
|
| 142 |
+
flag_lower=t(FLAG_LABEL[r.flag.value], lang).lower(),
|
| 143 |
+
value=f"{r.value:g}", unit=r.unit or ""))
|
| 144 |
+
n_ab = len([r for r in abnormal if r.flag != Flag.CHECK])
|
| 145 |
+
summary = {
|
| 146 |
+
"en": f"{len(results)} results read, {n_ab} outside the reference range.",
|
| 147 |
+
"fr": f"{len(results)} résultats lus, dont {n_ab} en dehors des valeurs de référence.",
|
| 148 |
+
"ar": f"تمت قراءة {len(results)} نتيجة، منها {n_ab} خارج المعدل المرجعي.",
|
| 149 |
+
"darija": f"قريت {len(results)} نتيجة، منهم {n_ab} برّا المعدل.",
|
| 150 |
+
}
|
| 151 |
+
return {"summary": t(summary, lang), "results": results, "questions": questions}
|
| 152 |
+
|
| 153 |
+
|
| 154 |
+
def explain(analysis: Analysis, lang: str) -> tuple[dict, str]:
|
| 155 |
+
"""Returns (explanation dict, source) where source is 'llm' or 'template:<reason>'."""
|
| 156 |
+
if not llm_available():
|
| 157 |
+
return explain_with_template(analysis, lang), "template:no-api-key"
|
| 158 |
+
try:
|
| 159 |
+
return explain_with_llm(analysis, lang), "llm"
|
| 160 |
+
except Exception as e: # network, JSON, or guardrail failure -> safe fallback
|
| 161 |
+
return explain_with_template(analysis, lang), f"template:{type(e).__name__}: {str(e)[:120]}"
|
| 162 |
+
|
| 163 |
+
|
| 164 |
+
def render_markdown(analysis: Analysis, exp: dict, lang: str) -> str:
|
| 165 |
+
catalog = load_catalog()
|
| 166 |
+
by_key = {x["key"]: x["explanation"] for x in exp.get("results", [])}
|
| 167 |
+
lines = [f"### {t(HEADINGS['summary'], lang)}", exp.get("summary", ""), "",
|
| 168 |
+
f"### {t(HEADINGS['details'], lang)}"]
|
| 169 |
+
order = {Flag.CRITICAL_HIGH: 0, Flag.CRITICAL_LOW: 0, Flag.CHECK: 1, Flag.HIGH: 2, Flag.LOW: 2,
|
| 170 |
+
Flag.UNKNOWN: 3, Flag.NORMAL: 4}
|
| 171 |
+
normal_names = []
|
| 172 |
+
for r in sorted(analysis.results, key=lambda r: order[r.flag]):
|
| 173 |
+
td = catalog.get(r.test_key or "")
|
| 174 |
+
if not td:
|
| 175 |
+
continue
|
| 176 |
+
name = td.display.get(lang) or td.display["en"]
|
| 177 |
+
if r.flag == Flag.NORMAL:
|
| 178 |
+
normal_names.append(name)
|
| 179 |
+
continue
|
| 180 |
+
label = t(FLAG_LABEL[r.flag.value], lang)
|
| 181 |
+
lines.append(f"- **{name}: {r.value:g} {r.unit or ''}** — _{label}_. {by_key.get(td.key, '')}")
|
| 182 |
+
if normal_names:
|
| 183 |
+
lines += ["", f"**{t(HEADINGS['normal_list'], lang)}:** " + ", ".join(normal_names)]
|
| 184 |
+
if exp.get("questions"):
|
| 185 |
+
lines += ["", f"### {t(HEADINGS['questions'], lang)}"] + [f"{i}. {q}" for i, q in enumerate(exp["questions"], 1)]
|
| 186 |
+
return "\n".join(lines)
|
lablens/extract.py
ADDED
|
@@ -0,0 +1,237 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""Turn a lab report (PDF or photo) into structured results.
|
| 2 |
+
|
| 3 |
+
Pipeline:
|
| 4 |
+
1. Get text lines: the PDF text layer when there is one, otherwise OCR (Tesseract).
|
| 5 |
+
2. For every line, find "name value [H/L] unit range".
|
| 6 |
+
3. Map the printed name to a catalog test (exact alias, then fuzzy).
|
| 7 |
+
4. Snap OCR-mangled units to the closest unit the catalog knows for that test.
|
| 8 |
+
|
| 9 |
+
No LLM is involved here: extraction errors should be measurable and reproducible.
|
| 10 |
+
"""
|
| 11 |
+
from __future__ import annotations
|
| 12 |
+
|
| 13 |
+
import re
|
| 14 |
+
from pathlib import Path
|
| 15 |
+
|
| 16 |
+
from rapidfuzz import fuzz, process
|
| 17 |
+
|
| 18 |
+
from .catalog import load_catalog, match_test_name, normalize_unit
|
| 19 |
+
from .models import Result
|
| 20 |
+
|
| 21 |
+
NUM = r"[<>≤≥]?\s?\d+(?:[.,]\d+)?"
|
| 22 |
+
NUM_RE = re.compile(rf"^{NUM}$")
|
| 23 |
+
MARKERS = {"h", "l", "*", "hh", "ll", "↑", "↓", "+", "-"}
|
| 24 |
+
|
| 25 |
+
|
| 26 |
+
def _to_float(s: str) -> float:
|
| 27 |
+
s = s.strip().lstrip("<>≤≥").strip().replace(",", ".")
|
| 28 |
+
return float(s)
|
| 29 |
+
|
| 30 |
+
|
| 31 |
+
def _clean_ocr_line(line: str) -> str:
|
| 32 |
+
line = line.replace("|", " ").replace("—", "-").replace("–", "-")
|
| 33 |
+
# Common OCR confusions inside numbers: O->0 and l->1 when surrounded by digits.
|
| 34 |
+
line = re.sub(r"(?<=\d)[Oo](?=[\d,.])|(?<=[\d,.])[Oo](?=\d|\b)", "0", line)
|
| 35 |
+
line = re.sub(r"(?<=\d)\s+([.,])\s*(?=\d)", r"\1", line) # "0 ,70" -> "0,70"
|
| 36 |
+
return re.sub(r"\s+", " ", line).strip()
|
| 37 |
+
|
| 38 |
+
|
| 39 |
+
def _looks_like_unit(tok: str) -> bool:
|
| 40 |
+
if re.match(r"^[\[(<>≤≥]", tok):
|
| 41 |
+
return False
|
| 42 |
+
if tok[0].isdigit():
|
| 43 |
+
return "/" in tok or "^" in tok # 10^3/µL, 10^9/L
|
| 44 |
+
return bool(re.search(r"[A-Za-zµ%]", tok))
|
| 45 |
+
|
| 46 |
+
|
| 47 |
+
def parse_range(text: str) -> tuple[float | None, float | None]:
|
| 48 |
+
raw = text.strip()
|
| 49 |
+
# OCR often reads a closing "]" as "1": "[4.20 - 5.901" -> "[4.20 - 5.90"
|
| 50 |
+
m = re.match(r"^[\[(].*?(\d+[.,](\d+))\s*-\s*(\d+[.,](\d+))$", raw)
|
| 51 |
+
if m and m.group(3).endswith("1") and len(m.group(4)) > len(m.group(2)):
|
| 52 |
+
raw = raw[:-1]
|
| 53 |
+
t = raw.strip("()[]{} ").replace(",", ".")
|
| 54 |
+
t = t.replace("–", "-").replace("—", "-").replace(" à ", "-").replace(" to ", "-")
|
| 55 |
+
m = re.match(r"^(?:<|≤|<=|inf\.?\s*[aà]?)\s*(\d+(?:\.\d+)?)", t, re.I)
|
| 56 |
+
if m:
|
| 57 |
+
return None, float(m.group(1))
|
| 58 |
+
m = re.match(r"^(?:>|≥|>=|sup\.?\s*[aà]?)\s*(\d+(?:\.\d+)?)", t, re.I)
|
| 59 |
+
if m:
|
| 60 |
+
return float(m.group(1)), None
|
| 61 |
+
m = re.search(r"(\d+(?:\.\d+)?)\s*-\s*(\d+(?:\.\d+)?)", t)
|
| 62 |
+
if m:
|
| 63 |
+
lo, hi = float(m.group(1)), float(m.group(2))
|
| 64 |
+
if lo <= hi:
|
| 65 |
+
return lo, hi
|
| 66 |
+
return None, None
|
| 67 |
+
|
| 68 |
+
|
| 69 |
+
# How units are usually printed; used to display snapped units nicely.
|
| 70 |
+
_PRETTY = {normalize_unit(u): u for u in [
|
| 71 |
+
"g/dL", "g/L", "mg/dL", "mg/L", "mmol/L", "µmol/L", "mEq/L", "%", "fL", "U/L", "UI/L", "IU/L",
|
| 72 |
+
"mIU/L", "µUI/mL", "µIU/mL", "ng/mL", "µg/L", "nmol/L", "10^3/µL", "10^6/µL",
|
| 73 |
+
"10^9/L", "10^12/L", "/mm3", "L/L"]}
|
| 74 |
+
_PRETTY.update({"10^3/ul": "10^3/µL", "10^6/ul": "10^6/µL", "uui/ml": "µUI/mL", "umol/l": "µmol/L"})
|
| 75 |
+
|
| 76 |
+
|
| 77 |
+
def _ocr_fold_unit(u: str) -> str:
|
| 78 |
+
u = normalize_unit(u)
|
| 79 |
+
u = re.sub(r"^p(?=[uµ])", "µ", u) # "pUI/mL" -> "µUI/mL"
|
| 80 |
+
u = re.sub(r"10[%*°](?=\d)", "10^", u) # "10%3/pL" -> "10^3/pL"
|
| 81 |
+
u = re.sub(r"/[pu](?=l$)", "/µ", u) # "/pL", "/uL" -> "/µL"
|
| 82 |
+
return u.replace("w", "/").replace("|", "l").replace("1", "l")
|
| 83 |
+
|
| 84 |
+
|
| 85 |
+
def _snap_unit(test_key: str | None, unit: str | None) -> str | None:
|
| 86 |
+
"""Return the unit as a known spelling for this test, repairing small OCR errors."""
|
| 87 |
+
if not unit or not test_key:
|
| 88 |
+
return unit
|
| 89 |
+
td = load_catalog()[test_key]
|
| 90 |
+
known = list(td.units) + [td.canonical_unit]
|
| 91 |
+
norm_known = [normalize_unit(k) for k in known]
|
| 92 |
+
nu = normalize_unit(unit)
|
| 93 |
+
if nu not in norm_known:
|
| 94 |
+
best = process.extractOne(_ocr_fold_unit(unit), norm_known, scorer=fuzz.ratio)
|
| 95 |
+
if not best or best[1] < 60:
|
| 96 |
+
return unit # leave it; flags will rely on the printed range only
|
| 97 |
+
nu = best[0]
|
| 98 |
+
if test_key in ("wbc", "platelets", "rbc") and nu in ("g/l", "t/l"):
|
| 99 |
+
return nu.upper() # giga/tera per litre cell counts, as French labs print them
|
| 100 |
+
return _PRETTY.get(nu, known[norm_known.index(nu)])
|
| 101 |
+
|
| 102 |
+
|
| 103 |
+
def parse_line(line: str) -> Result | None:
|
| 104 |
+
"""Parse one report line. Returns None when the line has no test-like structure."""
|
| 105 |
+
# Split markers glued to values: "5,36H" -> "5,36 H"
|
| 106 |
+
line = re.sub(r"(?<=\d)([HhLl*])(?=[\s.,;:_]|$)", r" \1", line)
|
| 107 |
+
line = line.replace("«", "<").replace("»", ">")
|
| 108 |
+
tokens = [t.strip("_.,;:'\"`~") or t for t in line.split(" ")]
|
| 109 |
+
tokens = [t for t in tokens if t and not re.fullmatch(r"[_.,;:'\"`~=]+", t)]
|
| 110 |
+
for i in range(1, len(tokens)):
|
| 111 |
+
tok = tokens[i]
|
| 112 |
+
if not NUM_RE.match(tok) or tok[0] in "<>≤≥":
|
| 113 |
+
continue
|
| 114 |
+
key, score, name = None, 0.0, ""
|
| 115 |
+
# The name is a prefix of the tokens before the value. OCR noise ("ae", "oo", "L")
|
| 116 |
+
# can sit between name and value, so also try dropping short trailing tokens.
|
| 117 |
+
for k in range(i, 0, -1):
|
| 118 |
+
if k < i and len(tokens[k]) > 3:
|
| 119 |
+
break
|
| 120 |
+
cand = " ".join(tokens[:k]).strip(" :.")
|
| 121 |
+
if not re.search(r"[A-Za-zÀ-ÿ-ۿ]", cand):
|
| 122 |
+
continue
|
| 123 |
+
ck, cs = match_test_name(cand)
|
| 124 |
+
if ck and cs > score:
|
| 125 |
+
key, score, name = ck, cs, cand
|
| 126 |
+
if cs == 100.0:
|
| 127 |
+
break
|
| 128 |
+
if key is None:
|
| 129 |
+
continue
|
| 130 |
+
# OCR sometimes emits a stray short number before the real value:
|
| 131 |
+
# "K 7 3.79 mmol/L" -> the value is the number directly followed by a unit.
|
| 132 |
+
if (i + 2 < len(tokens) and NUM_RE.match(tokens[i + 1]) and "." not in tok and "," not in tok
|
| 133 |
+
and len(tok) <= 2 and _looks_like_unit(tokens[i + 2])):
|
| 134 |
+
i += 1
|
| 135 |
+
tok = tokens[i]
|
| 136 |
+
j = i + 1
|
| 137 |
+
marker = None
|
| 138 |
+
if j < len(tokens) and tokens[j].lower() in MARKERS:
|
| 139 |
+
m = tokens[j].lower()
|
| 140 |
+
marker = "H" if m in ("h", "hh", "↑") else "L" if m in ("l", "ll", "↓") else "*" if m == "*" else None
|
| 141 |
+
j += 1
|
| 142 |
+
unit = None
|
| 143 |
+
if j < len(tokens) and _looks_like_unit(tokens[j]):
|
| 144 |
+
unit = tokens[j]
|
| 145 |
+
j += 1
|
| 146 |
+
ref_low, ref_high = parse_range(" ".join(tokens[j:]))
|
| 147 |
+
return Result(
|
| 148 |
+
test_key=key, raw_name=name, value=_to_float(tok), unit=_snap_unit(key, unit),
|
| 149 |
+
ref_low=ref_low, ref_high=ref_high,
|
| 150 |
+
ref_source="report" if (ref_low is not None or ref_high is not None) else "none",
|
| 151 |
+
match_score=score, source_line=line, report_marker=marker,
|
| 152 |
+
)
|
| 153 |
+
return None
|
| 154 |
+
|
| 155 |
+
|
| 156 |
+
def lines_from_pdf(path: Path) -> list[str]:
|
| 157 |
+
import pdfplumber
|
| 158 |
+
|
| 159 |
+
lines: list[str] = []
|
| 160 |
+
with pdfplumber.open(str(path)) as pdf:
|
| 161 |
+
for page in pdf.pages:
|
| 162 |
+
lines.extend((page.extract_text() or "").splitlines())
|
| 163 |
+
return lines
|
| 164 |
+
|
| 165 |
+
|
| 166 |
+
def preprocess_image(path: Path):
|
| 167 |
+
"""Grayscale, upscale, adaptive-threshold and strip table rules (incl. dotted lines).
|
| 168 |
+
|
| 169 |
+
Dotted row separators are the main thing that breaks Tesseract on lab reports:
|
| 170 |
+
without this step OCR on dotted layouts drops almost every row.
|
| 171 |
+
"""
|
| 172 |
+
import cv2
|
| 173 |
+
import numpy as np
|
| 174 |
+
from PIL import Image, ImageOps
|
| 175 |
+
|
| 176 |
+
pil = ImageOps.exif_transpose(Image.open(path)).convert("L")
|
| 177 |
+
img = np.array(pil)
|
| 178 |
+
if img.shape[1] < 2400:
|
| 179 |
+
f = 2400 / img.shape[1]
|
| 180 |
+
img = cv2.resize(img, None, fx=f, fy=f, interpolation=cv2.INTER_CUBIC)
|
| 181 |
+
img = cv2.GaussianBlur(img, (3, 3), 0)
|
| 182 |
+
bw = cv2.adaptiveThreshold(img, 255, cv2.ADAPTIVE_THRESH_GAUSSIAN_C, cv2.THRESH_BINARY, 31, 15)
|
| 183 |
+
inv = 255 - bw
|
| 184 |
+
closed = cv2.morphologyEx(inv, cv2.MORPH_CLOSE, cv2.getStructuringElement(cv2.MORPH_RECT, (9, 1)))
|
| 185 |
+
rules = cv2.morphologyEx(closed, cv2.MORPH_OPEN, cv2.getStructuringElement(cv2.MORPH_RECT, (150, 1)))
|
| 186 |
+
rules = cv2.dilate(rules, cv2.getStructuringElement(cv2.MORPH_RECT, (1, 3)))
|
| 187 |
+
return 255 - cv2.bitwise_and(inv, 255 - rules)
|
| 188 |
+
|
| 189 |
+
|
| 190 |
+
def lines_from_image(path: Path, langs: str = "eng+fra") -> list[str]:
|
| 191 |
+
import pytesseract
|
| 192 |
+
|
| 193 |
+
text = pytesseract.image_to_string(preprocess_image(path), lang=langs, config="--psm 6")
|
| 194 |
+
return text.splitlines()
|
| 195 |
+
|
| 196 |
+
|
| 197 |
+
def detect_language(lines: list[str]) -> str:
|
| 198 |
+
text = " ".join(lines).lower()
|
| 199 |
+
fr = sum(w in text for w in ["analyse", "résultat", "resultat", "prélèvement", "référence", "glycémie", "valeurs"])
|
| 200 |
+
en = sum(w in text for w in ["result", "reference", "collection", "patient:", "test", "range"])
|
| 201 |
+
return "fr" if fr > en else "en"
|
| 202 |
+
|
| 203 |
+
|
| 204 |
+
def extract(path: str | Path) -> tuple[list[Result], list[str], str, str]:
|
| 205 |
+
"""Returns (results, unmatched numeric lines, detected language, method)."""
|
| 206 |
+
path = Path(path)
|
| 207 |
+
method = "pdf-text"
|
| 208 |
+
if path.suffix.lower() == ".pdf":
|
| 209 |
+
lines = lines_from_pdf(path)
|
| 210 |
+
if sum(len(line) for line in lines) < 40: # scanned PDF without a text layer
|
| 211 |
+
import subprocess
|
| 212 |
+
import tempfile
|
| 213 |
+
|
| 214 |
+
with tempfile.TemporaryDirectory() as td:
|
| 215 |
+
subprocess.run(["pdftoppm", "-png", "-r", "300", str(path), f"{td}/p"], check=True)
|
| 216 |
+
lines = []
|
| 217 |
+
for png in sorted(Path(td).glob("p*.png")):
|
| 218 |
+
lines.extend(lines_from_image(png))
|
| 219 |
+
method = "ocr"
|
| 220 |
+
else:
|
| 221 |
+
lines = lines_from_image(path)
|
| 222 |
+
method = "ocr"
|
| 223 |
+
|
| 224 |
+
results: list[Result] = []
|
| 225 |
+
unmatched: list[str] = []
|
| 226 |
+
seen: set[str] = set()
|
| 227 |
+
for raw in lines:
|
| 228 |
+
line = _clean_ocr_line(raw)
|
| 229 |
+
if not line:
|
| 230 |
+
continue
|
| 231 |
+
r = parse_line(line)
|
| 232 |
+
if r and r.test_key not in seen:
|
| 233 |
+
seen.add(r.test_key)
|
| 234 |
+
results.append(r)
|
| 235 |
+
elif r is None and re.search(r"\d+[.,]\d+", line) and not re.search(r"\d{4}-\d{2}-\d{2}", line):
|
| 236 |
+
unmatched.append(line)
|
| 237 |
+
return results, unmatched, detect_language(lines), method
|
lablens/flagger.py
ADDED
|
@@ -0,0 +1,84 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""Deterministic flagging. No LLM is involved in deciding what is abnormal or urgent.
|
| 2 |
+
|
| 3 |
+
Rules, in order:
|
| 4 |
+
0. Values outside physiologically possible bounds are marked CHECK (likely misread).
|
| 5 |
+
1. Critical thresholds from the catalog (converted to canonical units) always win.
|
| 6 |
+
They escalate to an "urgent" banner.
|
| 7 |
+
2. Otherwise the report's own printed reference range decides low / high / normal.
|
| 8 |
+
3. If the report printed no range, the catalog's typical adult range is used and the
|
| 9 |
+
result is marked as such, so the UI can say the range is generic.
|
| 10 |
+
4. If the unit can't be understood, critical checks are skipped and the flag uses
|
| 11 |
+
only the printed range; if there is none, the flag is UNKNOWN.
|
| 12 |
+
5. If the lab printed an H/L marker that disagrees with the computed flag, the result
|
| 13 |
+
becomes CHECK: a misread value must never come out as reassuring.
|
| 14 |
+
"""
|
| 15 |
+
from __future__ import annotations
|
| 16 |
+
|
| 17 |
+
from .catalog import load_catalog
|
| 18 |
+
from .models import Analysis, Flag, FlaggedResult, Result
|
| 19 |
+
|
| 20 |
+
|
| 21 |
+
def flag_result(r: Result) -> FlaggedResult:
|
| 22 |
+
catalog = load_catalog()
|
| 23 |
+
fr = FlaggedResult(**r.model_dump())
|
| 24 |
+
td = catalog.get(r.test_key) if r.test_key else None
|
| 25 |
+
|
| 26 |
+
canonical = td.to_canonical(r.value, r.unit) if td else None
|
| 27 |
+
fr.canonical_value = round(canonical, 4) if canonical is not None else None
|
| 28 |
+
|
| 29 |
+
# 0. Implausible values are almost always reading errors (dropped decimal, stray digit).
|
| 30 |
+
# Never call them normal and never escalate on them: ask the user to check.
|
| 31 |
+
if td and canonical is not None and not (td.plausible[0] <= canonical <= td.plausible[1]):
|
| 32 |
+
fr.flag = Flag.CHECK
|
| 33 |
+
fr.reason = "value outside what is physically possible; please check it against your report"
|
| 34 |
+
return fr
|
| 35 |
+
|
| 36 |
+
# 1. Critical values (only when we are sure of the unit).
|
| 37 |
+
if td and canonical is not None:
|
| 38 |
+
crit_low = td.critical.get("low")
|
| 39 |
+
crit_high = td.critical.get("high")
|
| 40 |
+
if crit_low is not None and canonical < crit_low:
|
| 41 |
+
fr.flag = Flag.CRITICAL_LOW
|
| 42 |
+
fr.reason = f"below critical threshold {crit_low} {td.canonical_unit}"
|
| 43 |
+
return fr
|
| 44 |
+
if crit_high is not None and canonical > crit_high:
|
| 45 |
+
fr.flag = Flag.CRITICAL_HIGH
|
| 46 |
+
fr.reason = f"above critical threshold {crit_high} {td.canonical_unit}"
|
| 47 |
+
return fr
|
| 48 |
+
|
| 49 |
+
# 2./3. Reference range: report first, catalog fallback.
|
| 50 |
+
low, high, value = r.ref_low, r.ref_high, r.value
|
| 51 |
+
if low is None and high is None:
|
| 52 |
+
if td and canonical is not None:
|
| 53 |
+
low, high = td.typical_range
|
| 54 |
+
value = canonical
|
| 55 |
+
fr.ref_source = "catalog"
|
| 56 |
+
fr.ref_low, fr.ref_high = low, high
|
| 57 |
+
else:
|
| 58 |
+
fr.flag = Flag.UNKNOWN
|
| 59 |
+
fr.ref_source = "none"
|
| 60 |
+
fr.reason = "no reference range on the report and unit not recognized"
|
| 61 |
+
return fr
|
| 62 |
+
|
| 63 |
+
if low is not None and value < low:
|
| 64 |
+
fr.flag, fr.reason = Flag.LOW, f"{value:g} is below {low:g}"
|
| 65 |
+
elif high is not None and value > high:
|
| 66 |
+
fr.flag, fr.reason = Flag.HIGH, f"{value:g} is above {high:g}"
|
| 67 |
+
else:
|
| 68 |
+
fr.flag, fr.reason = Flag.NORMAL, "within range"
|
| 69 |
+
|
| 70 |
+
# 5. Cross-check with the lab's own H/L marker. If the lab marked the value abnormal
|
| 71 |
+
# but the numbers we read say otherwise, something was misread: never reassure.
|
| 72 |
+
m = r.report_marker
|
| 73 |
+
if m and ((fr.flag == Flag.NORMAL) or (m == "H" and fr.flag == Flag.LOW) or (m == "L" and fr.flag == Flag.HIGH)):
|
| 74 |
+
fr.flag = Flag.CHECK
|
| 75 |
+
fr.reason = f"the lab marked this value '{m}' but the numbers read disagree; please check"
|
| 76 |
+
return fr
|
| 77 |
+
|
| 78 |
+
|
| 79 |
+
def flag_all(results: list[Result], language: str = "en", unmatched: list[str] | None = None,
|
| 80 |
+
method: str = "") -> Analysis:
|
| 81 |
+
flagged = [flag_result(r) for r in results]
|
| 82 |
+
urgent = any(f.flag in (Flag.CRITICAL_LOW, Flag.CRITICAL_HIGH) for f in flagged)
|
| 83 |
+
return Analysis(language=language, results=flagged, unmatched_lines=unmatched or [],
|
| 84 |
+
urgent=urgent, extraction_method=method)
|
lablens/i18n.py
ADDED
|
@@ -0,0 +1,77 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""UI strings and deterministic text in the four supported languages.
|
| 2 |
+
|
| 3 |
+
Everything safety-relevant (flags, the urgent banner, the disclaimer) comes from here,
|
| 4 |
+
never from the LLM, so it is identical on every run and can be reviewed once.
|
| 5 |
+
"""
|
| 6 |
+
from __future__ import annotations
|
| 7 |
+
|
| 8 |
+
LANGS = {"en": "English", "fr": "Français", "ar": "العربية", "darija": "Tounsi (Darija)"}
|
| 9 |
+
RTL = {"ar", "darija"}
|
| 10 |
+
|
| 11 |
+
FLAG_LABEL = {
|
| 12 |
+
"normal": {"en": "Within range", "fr": "Dans les normes", "ar": "ضمن المعدل", "darija": "ف النورمال"},
|
| 13 |
+
"low": {"en": "Low", "fr": "Bas", "ar": "منخفض", "darija": "ناقص"},
|
| 14 |
+
"high": {"en": "High", "fr": "Élevé", "ar": "مرتفع", "darija": "طالع"},
|
| 15 |
+
"critical_low": {"en": "Very low – urgent", "fr": "Très bas – urgent", "ar": "منخفض جدًا – عاجل",
|
| 16 |
+
"darija": "ناقص برشا – عاجل"},
|
| 17 |
+
"critical_high": {"en": "Very high – urgent", "fr": "Très élevé – urgent", "ar": "مرتفع جدًا – عاجل",
|
| 18 |
+
"darija": "طالع برشا – عاجل"},
|
| 19 |
+
"unknown": {"en": "Can't judge", "fr": "Non évaluable", "ar": "لا يمكن التقييم", "darija": "ما نجموش نقيّموه"},
|
| 20 |
+
"check": {"en": "Please check value", "fr": "Vérifiez la valeur", "ar": "تحقق من القيمة",
|
| 21 |
+
"darija": "ثبّت فالرقم"},
|
| 22 |
+
}
|
| 23 |
+
|
| 24 |
+
URGENT = {
|
| 25 |
+
"en": "**One or more results are in a range that can be dangerous.** Please contact a doctor today, "
|
| 26 |
+
"or go to an emergency service if you feel unwell. Lab values can be wrong, so a doctor needs to "
|
| 27 |
+
"confirm them, but don't wait.",
|
| 28 |
+
"fr": "**Un ou plusieurs résultats sont dans une zone qui peut être dangereuse.** Contactez un médecin "
|
| 29 |
+
"aujourd'hui, ou rendez-vous aux urgences si vous ne vous sentez pas bien. Un résultat peut être "
|
| 30 |
+
"erroné et doit être confirmé par un médecin, mais n'attendez pas.",
|
| 31 |
+
"ar": "**نتيجة أو أكثر في نطاق قد يكون خطيرًا.** يرجى التواصل مع طبيب اليوم، أو التوجه إلى الطوارئ إذا "
|
| 32 |
+
"شعرت بتوعك. قد تكون النتائج خاطئة ويجب أن يؤكدها الطبيب، لكن لا تنتظر.",
|
| 33 |
+
"darija": "**فما نتيجة ولا أكثر في مستوى ينجم يكون خطير.** كلّم طبيب اليوم، ولا امشي للاستعجالي كان "
|
| 34 |
+
"ماكش لاباس. النتيجة تنجم تكون غالطة والطبيب لازم يثبّتها، أما ما تستناش.",
|
| 35 |
+
}
|
| 36 |
+
|
| 37 |
+
DISCLAIMER = {
|
| 38 |
+
"en": "LabLens explains lab results in plain language. It does not diagnose and does not replace a doctor.",
|
| 39 |
+
"fr": "LabLens explique les résultats d'analyses en langage simple. Il ne pose pas de diagnostic et ne "
|
| 40 |
+
"remplace pas un médecin.",
|
| 41 |
+
"ar": "يشرح LabLens نتائج التحاليل بلغة بسيطة. لا يقدّم تشخيصًا ولا يغني عن الطبيب.",
|
| 42 |
+
"darija": "LabLens يفسّرلك نتايج التحاليل بكلام ساهل. ما يعطيكش تشخيص وما يعوّضش الطبيب.",
|
| 43 |
+
}
|
| 44 |
+
|
| 45 |
+
CHECK_NOTE = {
|
| 46 |
+
"en": "Some values look misread. Compare them with your paper report and fix them in the table, then explain again.",
|
| 47 |
+
"fr": "Certaines valeurs semblent mal lues. Comparez-les avec votre compte rendu, corrigez-les dans le tableau, puis relancez.",
|
| 48 |
+
"ar": "يبدو أن بعض القيم قُرئت بشكل خاطئ. قارنها مع التقرير الورقي وصحّحها في الجدول ثم أعد الشرح.",
|
| 49 |
+
"darija": "فما أرقام يظهر تقراو غالط. قارنهم مع الورقة متاعك، صلّحهم فالجدول، وعاود.",
|
| 50 |
+
}
|
| 51 |
+
|
| 52 |
+
GENERIC_RANGE = {
|
| 53 |
+
"en": "typical adult range (your report printed none)",
|
| 54 |
+
"fr": "valeurs usuelles adulte (aucune sur votre compte rendu)",
|
| 55 |
+
"ar": "المعدل المعتاد للبالغين (لا يوجد في تقريرك)",
|
| 56 |
+
"darija": "المعدل العادي للكبار (ما فماش فالتقرير)",
|
| 57 |
+
}
|
| 58 |
+
|
| 59 |
+
HEADINGS = {
|
| 60 |
+
"summary": {"en": "Summary", "fr": "En résumé", "ar": "الخلاصة", "darija": "باختصار"},
|
| 61 |
+
"details": {"en": "Your results explained", "fr": "Vos résultats expliqués", "ar": "شرح نتائجك",
|
| 62 |
+
"darija": "نتايجك مفسّرة"},
|
| 63 |
+
"questions": {"en": "Questions to ask your doctor", "fr": "Questions à poser à votre médecin",
|
| 64 |
+
"ar": "أسئلة لطبيبك", "darija": "أسئلة تسألهم لطبيبك"},
|
| 65 |
+
"normal_list": {"en": "Within range", "fr": "Dans les normes", "ar": "ضمن المعدل", "darija": "ف النورمال"},
|
| 66 |
+
}
|
| 67 |
+
|
| 68 |
+
TEMPLATE_QUESTION = {
|
| 69 |
+
"en": "My {name} is {flag_lower} ({value} {unit}). What could explain it, and does it need follow-up?",
|
| 70 |
+
"fr": "Mon résultat de {name} est {flag_lower} ({value} {unit}). Qu'est-ce qui peut l'expliquer, et faut-il un suivi ?",
|
| 71 |
+
"ar": "نتيجة {name} لدي {flag_lower} ({value} {unit}). ما الذي قد يفسر ذلك، وهل تحتاج إلى متابعة؟",
|
| 72 |
+
"darija": "{name} متاعي {flag_lower} ({value} {unit}). شنوّة نجم يكون السبب، ولازم متابعة؟",
|
| 73 |
+
}
|
| 74 |
+
|
| 75 |
+
|
| 76 |
+
def t(table: dict, lang: str) -> str:
|
| 77 |
+
return table.get(lang) or table["en"]
|
lablens/models.py
ADDED
|
@@ -0,0 +1,44 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
from __future__ import annotations
|
| 2 |
+
|
| 3 |
+
from enum import Enum
|
| 4 |
+
|
| 5 |
+
from pydantic import BaseModel, Field
|
| 6 |
+
|
| 7 |
+
|
| 8 |
+
class Flag(str, Enum):
|
| 9 |
+
NORMAL = "normal"
|
| 10 |
+
LOW = "low"
|
| 11 |
+
HIGH = "high"
|
| 12 |
+
CRITICAL_LOW = "critical_low"
|
| 13 |
+
CRITICAL_HIGH = "critical_high"
|
| 14 |
+
UNKNOWN = "unknown" # could not be judged (no range, unknown unit)
|
| 15 |
+
CHECK = "check" # value is physically implausible: probably misread, ask the user
|
| 16 |
+
|
| 17 |
+
|
| 18 |
+
class Result(BaseModel):
|
| 19 |
+
"""One test line read from a report."""
|
| 20 |
+
|
| 21 |
+
test_key: str | None = None
|
| 22 |
+
raw_name: str
|
| 23 |
+
value: float
|
| 24 |
+
unit: str | None = None
|
| 25 |
+
ref_low: float | None = None
|
| 26 |
+
ref_high: float | None = None
|
| 27 |
+
ref_source: str = "report" # "report" | "catalog" | "none"
|
| 28 |
+
match_score: float = 0.0
|
| 29 |
+
source_line: str = ""
|
| 30 |
+
report_marker: str | None = None # "H" / "L" / "*" printed next to the value by the lab
|
| 31 |
+
|
| 32 |
+
|
| 33 |
+
class FlaggedResult(Result):
|
| 34 |
+
flag: Flag = Flag.UNKNOWN
|
| 35 |
+
canonical_value: float | None = None
|
| 36 |
+
reason: str = ""
|
| 37 |
+
|
| 38 |
+
|
| 39 |
+
class Analysis(BaseModel):
|
| 40 |
+
language: str = "en"
|
| 41 |
+
results: list[FlaggedResult] = Field(default_factory=list)
|
| 42 |
+
unmatched_lines: list[str] = Field(default_factory=list)
|
| 43 |
+
urgent: bool = False
|
| 44 |
+
extraction_method: str = ""
|
lablens/synth/__init__.py
ADDED
|
File without changes
|
lablens/synth/generator.py
ADDED
|
@@ -0,0 +1,320 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""Synthetic lab-report generator.
|
| 2 |
+
|
| 3 |
+
Produces realistic-looking lab reports (PDF, plus degraded "phone photo" PNGs) together
|
| 4 |
+
with ground-truth JSON. Everything is fictional: lab names, patients and values.
|
| 5 |
+
|
| 6 |
+
Variation that matters for robustness:
|
| 7 |
+
* locale: English (US-style units) and French (Tunisia/France-style units: g/L glucose,
|
| 8 |
+
G/L white cells, decimal commas)
|
| 9 |
+
* three layouts (clean table, dotted lines, compact two-column header)
|
| 10 |
+
* range formats: "12.0 - 16.0", "(12,0-16,0)", "< 2,00", "> 0,40", "[70 – 99]"
|
| 11 |
+
* optional H/L markers next to abnormal values
|
| 12 |
+
* value distribution: mostly normal, some low/high, a few critical
|
| 13 |
+
"""
|
| 14 |
+
from __future__ import annotations
|
| 15 |
+
|
| 16 |
+
import json
|
| 17 |
+
import random
|
| 18 |
+
from dataclasses import asdict, dataclass
|
| 19 |
+
from datetime import date, timedelta
|
| 20 |
+
from pathlib import Path
|
| 21 |
+
|
| 22 |
+
from reportlab.lib import colors
|
| 23 |
+
from reportlab.lib.pagesizes import A4
|
| 24 |
+
from reportlab.lib.units import mm
|
| 25 |
+
from reportlab.pdfbase import pdfmetrics
|
| 26 |
+
from reportlab.pdfbase.ttfonts import TTFont
|
| 27 |
+
from reportlab.pdfgen import canvas
|
| 28 |
+
|
| 29 |
+
from ..catalog import load_catalog
|
| 30 |
+
from ..flagger import flag_result
|
| 31 |
+
from ..models import Result
|
| 32 |
+
|
| 33 |
+
FONT_DIR = Path("/usr/share/fonts/truetype/dejavu")
|
| 34 |
+
pdfmetrics.registerFont(TTFont("DejaVu", str(FONT_DIR / "DejaVuSans.ttf")))
|
| 35 |
+
pdfmetrics.registerFont(TTFont("DejaVu-Bold", str(FONT_DIR / "DejaVuSans-Bold.ttf")))
|
| 36 |
+
|
| 37 |
+
# Units as printed on reports, per locale.
|
| 38 |
+
LOCALE_UNITS = {
|
| 39 |
+
"en": {
|
| 40 |
+
"hemoglobin": "g/dL", "hematocrit": "%", "rbc": "10^6/µL", "mcv": "fL", "wbc": "10^3/µL",
|
| 41 |
+
"platelets": "10^3/µL", "glucose": "mg/dL", "hba1c": "%", "creatinine": "mg/dL",
|
| 42 |
+
"urea": "mg/dL", "sodium": "mmol/L", "potassium": "mmol/L", "calcium": "mg/dL",
|
| 43 |
+
"cholesterol_total": "mg/dL", "ldl": "mg/dL", "hdl": "mg/dL", "triglycerides": "mg/dL",
|
| 44 |
+
"alt": "U/L", "ast": "U/L", "tsh": "mIU/L", "ferritin": "ng/mL", "crp": "mg/L",
|
| 45 |
+
"vitamin_d": "ng/mL",
|
| 46 |
+
},
|
| 47 |
+
"fr": {
|
| 48 |
+
"hemoglobin": "g/dL", "hematocrit": "%", "rbc": "T/L", "mcv": "fL", "wbc": "G/L",
|
| 49 |
+
"platelets": "G/L", "glucose": "g/L", "hba1c": "%", "creatinine": "mg/L",
|
| 50 |
+
"urea": "g/L", "sodium": "mmol/L", "potassium": "mmol/L", "calcium": "mmol/L",
|
| 51 |
+
"cholesterol_total": "g/L", "ldl": "g/L", "hdl": "g/L", "triglycerides": "g/L",
|
| 52 |
+
"alt": "UI/L", "ast": "UI/L", "tsh": "µUI/mL", "ferritin": "ng/mL", "crp": "mg/L",
|
| 53 |
+
"vitamin_d": "ng/mL",
|
| 54 |
+
},
|
| 55 |
+
}
|
| 56 |
+
|
| 57 |
+
SECTIONS = {
|
| 58 |
+
"blood_count": {"en": "COMPLETE BLOOD COUNT", "fr": "HÉMOGRAMME (NFS)"},
|
| 59 |
+
"metabolic": {"en": "GLUCOSE METABOLISM", "fr": "MÉTABOLISME GLUCIDIQUE"},
|
| 60 |
+
"kidney": {"en": "KIDNEY FUNCTION", "fr": "FONCTION RÉNALE"},
|
| 61 |
+
"electrolytes": {"en": "ELECTROLYTES", "fr": "IONOGRAMME"},
|
| 62 |
+
"lipids": {"en": "LIPID PANEL", "fr": "BILAN LIPIDIQUE"},
|
| 63 |
+
"liver": {"en": "LIVER ENZYMES", "fr": "BILAN HÉPATIQUE"},
|
| 64 |
+
"thyroid": {"en": "THYROID", "fr": "THYROÏDE"},
|
| 65 |
+
"iron": {"en": "IRON STUDIES", "fr": "BILAN MARTIAL"},
|
| 66 |
+
"inflammation": {"en": "INFLAMMATION", "fr": "INFLAMMATION"},
|
| 67 |
+
"vitamins": {"en": "VITAMINS", "fr": "VITAMINES"},
|
| 68 |
+
}
|
| 69 |
+
|
| 70 |
+
LABS = ["Laboratoire Les Oliviers", "Medina Biolab", "Laboratoire du Sahel",
|
| 71 |
+
"Cap Bon Diagnostics", "Laboratoire El Amen", "Nour Medical Lab"]
|
| 72 |
+
FIRST = ["Amira", "Youssef", "Salma", "Karim", "Ines", "Mehdi", "Rania", "Omar", "Leila", "Sami"]
|
| 73 |
+
LAST = ["Ben Salah", "Trabelsi", "Jaziri", "Haddad", "Mansour", "Gharbi", "Chaabane", "Ayari"]
|
| 74 |
+
|
| 75 |
+
HEADERS = {
|
| 76 |
+
"en": {"title": "LABORATORY REPORT", "patient": "Patient", "date": "Collection date",
|
| 77 |
+
"test": "Test", "result": "Result", "unit": "Unit", "ref": "Reference range",
|
| 78 |
+
"footer": "Results to be interpreted by your physician."},
|
| 79 |
+
"fr": {"title": "COMPTE RENDU D'ANALYSES", "patient": "Patient(e)", "date": "Date de prélèvement",
|
| 80 |
+
"test": "Analyse", "result": "Résultat", "unit": "Unité", "ref": "Valeurs de référence",
|
| 81 |
+
"footer": "Résultats à interpréter par votre médecin traitant."},
|
| 82 |
+
}
|
| 83 |
+
|
| 84 |
+
|
| 85 |
+
@dataclass
|
| 86 |
+
class TruthRow:
|
| 87 |
+
test_key: str
|
| 88 |
+
printed_name: str
|
| 89 |
+
value: float
|
| 90 |
+
value_str: str
|
| 91 |
+
unit: str
|
| 92 |
+
ref_low: float | None
|
| 93 |
+
ref_high: float | None
|
| 94 |
+
ref_str: str
|
| 95 |
+
expected_flag: str
|
| 96 |
+
|
| 97 |
+
|
| 98 |
+
def _decimals(high: float) -> int:
|
| 99 |
+
if high < 10:
|
| 100 |
+
return 2
|
| 101 |
+
if high < 100:
|
| 102 |
+
return 1
|
| 103 |
+
return 0
|
| 104 |
+
|
| 105 |
+
|
| 106 |
+
def _fmt(x: float, d: int, locale: str) -> str:
|
| 107 |
+
s = f"{x:.{d}f}"
|
| 108 |
+
return s.replace(".", ",") if locale == "fr" else s
|
| 109 |
+
|
| 110 |
+
|
| 111 |
+
def _sample_value(rng: random.Random, td, status: str) -> float:
|
| 112 |
+
lo, hi = td.typical_range
|
| 113 |
+
span = hi - lo if hi > lo else hi
|
| 114 |
+
if status == "normal":
|
| 115 |
+
return rng.uniform(lo + 0.05 * span, hi - 0.05 * span) if lo > 0 else rng.uniform(0.3 * hi, 0.95 * hi)
|
| 116 |
+
if status == "high":
|
| 117 |
+
cap = td.critical.get("high", hi * 2)
|
| 118 |
+
return rng.uniform(hi * 1.05, min(hi * 1.6, cap * 0.97))
|
| 119 |
+
if status == "low":
|
| 120 |
+
floor = td.critical.get("low", 0.0)
|
| 121 |
+
return rng.uniform(max(lo * 0.55, floor * 1.05), lo * 0.95)
|
| 122 |
+
if status == "critical_high":
|
| 123 |
+
return td.critical["high"] * rng.uniform(1.03, 1.15)
|
| 124 |
+
if status == "critical_low":
|
| 125 |
+
return td.critical["low"] * rng.uniform(0.85, 0.97)
|
| 126 |
+
raise ValueError(status)
|
| 127 |
+
|
| 128 |
+
|
| 129 |
+
def _choose_status(rng: random.Random, td) -> str:
|
| 130 |
+
one_sided_upper = td.typical_range[0] == 0
|
| 131 |
+
options = [("normal", 0.68), ("high", 0.16)]
|
| 132 |
+
if not one_sided_upper:
|
| 133 |
+
options.append(("low", 0.12))
|
| 134 |
+
if "high" in td.critical:
|
| 135 |
+
options.append(("critical_high", 0.03))
|
| 136 |
+
if "low" in td.critical:
|
| 137 |
+
options.append(("critical_low", 0.03))
|
| 138 |
+
r, acc = rng.random() * sum(w for _, w in options), 0.0
|
| 139 |
+
for name, w in options:
|
| 140 |
+
acc += w
|
| 141 |
+
if r <= acc:
|
| 142 |
+
return name
|
| 143 |
+
return "normal"
|
| 144 |
+
|
| 145 |
+
|
| 146 |
+
def make_report(seed: int, locale: str | None = None) -> dict:
|
| 147 |
+
rng = random.Random(seed)
|
| 148 |
+
catalog = load_catalog()
|
| 149 |
+
locale = locale or rng.choice(["en", "fr"])
|
| 150 |
+
layout = rng.choice(["table", "dotted", "compact"])
|
| 151 |
+
range_style = rng.choice(["dash", "paren", "bracket"])
|
| 152 |
+
show_hl = rng.random() < 0.5
|
| 153 |
+
|
| 154 |
+
keys = rng.sample(list(catalog), rng.randint(6, 14))
|
| 155 |
+
order = list(SECTIONS)
|
| 156 |
+
keys.sort(key=lambda k: (order.index(catalog[k].category), k))
|
| 157 |
+
|
| 158 |
+
rows: list[TruthRow] = []
|
| 159 |
+
for k in keys:
|
| 160 |
+
td = catalog[k]
|
| 161 |
+
unit = LOCALE_UNITS[locale][k]
|
| 162 |
+
factor = td.to_canonical(1.0, unit)
|
| 163 |
+
assert factor, (k, unit)
|
| 164 |
+
lo_c, hi_c = td.typical_range
|
| 165 |
+
lo_r, hi_r = lo_c / factor, hi_c / factor
|
| 166 |
+
d = _decimals(hi_r)
|
| 167 |
+
value = round(_sample_value(rng, td, _choose_status(rng, td)) / factor, d)
|
| 168 |
+
|
| 169 |
+
if lo_c == 0: # upper bound only
|
| 170 |
+
ref_low, ref_high = None, round(hi_r, d)
|
| 171 |
+
ref_str = f"< {_fmt(ref_high, d, locale)}"
|
| 172 |
+
elif td.key == "hdl": # lower bound only, as printed on real reports
|
| 173 |
+
ref_low, ref_high = round(lo_r, d), None
|
| 174 |
+
ref_str = f"> {_fmt(ref_low, d, locale)}"
|
| 175 |
+
else:
|
| 176 |
+
ref_low, ref_high = round(lo_r, d), round(hi_r, d)
|
| 177 |
+
a, b = _fmt(ref_low, d, locale), _fmt(ref_high, d, locale)
|
| 178 |
+
ref_str = {"dash": f"{a} - {b}", "paren": f"({a}-{b})", "bracket": f"[{a} – {b}]"}[range_style]
|
| 179 |
+
|
| 180 |
+
name = rng.choice(td.aliases[locale][: max(1, min(3, len(td.aliases[locale])))])
|
| 181 |
+
if name.isascii() and len(name.replace("-", "")) <= 6 and " " not in name:
|
| 182 |
+
name = name.upper() # abbreviations: MCV, HDL-C, SGOT
|
| 183 |
+
elif rng.random() < 0.3:
|
| 184 |
+
name = name.upper()
|
| 185 |
+
else:
|
| 186 |
+
name = name[:1].upper() + name[1:]
|
| 187 |
+
|
| 188 |
+
truth = Result(test_key=k, raw_name=name, value=value, unit=unit, ref_low=ref_low, ref_high=ref_high)
|
| 189 |
+
flag = flag_result(truth).flag.value
|
| 190 |
+
value_str = _fmt(value, d, locale)
|
| 191 |
+
rows.append(TruthRow(k, name, value, value_str, unit, ref_low, ref_high, ref_str, flag))
|
| 192 |
+
|
| 193 |
+
patient = f"{rng.choice(FIRST)} {rng.choice(LAST)}"
|
| 194 |
+
day = date(2026, 1, 1) + timedelta(days=rng.randint(0, 290))
|
| 195 |
+
return {
|
| 196 |
+
"id": f"rpt_{seed:05d}", "seed": seed, "locale": locale, "layout": layout,
|
| 197 |
+
"show_hl": show_hl, "lab": rng.choice(LABS), "patient": patient,
|
| 198 |
+
"date": day.isoformat(), "rows": [asdict(r) for r in rows],
|
| 199 |
+
}
|
| 200 |
+
|
| 201 |
+
|
| 202 |
+
def render_pdf(report: dict, path: Path) -> None:
|
| 203 |
+
catalog = load_catalog()
|
| 204 |
+
loc = report["locale"]
|
| 205 |
+
h = HEADERS[loc]
|
| 206 |
+
c = canvas.Canvas(str(path), pagesize=A4)
|
| 207 |
+
W, H = A4
|
| 208 |
+
x0, y = 18 * mm, H - 20 * mm
|
| 209 |
+
|
| 210 |
+
c.setFont("DejaVu-Bold", 14)
|
| 211 |
+
c.drawString(x0, y, report["lab"])
|
| 212 |
+
c.setFont("DejaVu", 8)
|
| 213 |
+
c.drawString(x0, y - 11, "Fictional laboratory - synthetic report generated by LabLens")
|
| 214 |
+
y -= 30
|
| 215 |
+
c.setFont("DejaVu-Bold", 12)
|
| 216 |
+
c.drawString(x0, y, h["title"])
|
| 217 |
+
y -= 18
|
| 218 |
+
c.setFont("DejaVu", 10)
|
| 219 |
+
if report["layout"] == "compact":
|
| 220 |
+
c.drawString(x0, y, f"{h['patient']}: {report['patient']}")
|
| 221 |
+
c.drawString(x0 + 95 * mm, y, f"{h['date']}: {report['date']}")
|
| 222 |
+
y -= 20
|
| 223 |
+
else:
|
| 224 |
+
c.drawString(x0, y, f"{h['patient']}: {report['patient']}")
|
| 225 |
+
c.drawString(x0, y - 14, f"{h['date']}: {report['date']}")
|
| 226 |
+
y -= 34
|
| 227 |
+
|
| 228 |
+
cols = [x0, x0 + 72 * mm, x0 + 102 * mm, x0 + 130 * mm]
|
| 229 |
+
c.setFont("DejaVu-Bold", 9.5)
|
| 230 |
+
for cx, label in zip(cols, [h["test"], h["result"], h["unit"], h["ref"]]):
|
| 231 |
+
c.drawString(cx, y, label)
|
| 232 |
+
c.setLineWidth(0.6)
|
| 233 |
+
c.line(x0, y - 4, W - 18 * mm, y - 4)
|
| 234 |
+
y -= 18
|
| 235 |
+
|
| 236 |
+
current = None
|
| 237 |
+
for r in report["rows"]:
|
| 238 |
+
cat = catalog[r["test_key"]].category
|
| 239 |
+
if cat != current:
|
| 240 |
+
current = cat
|
| 241 |
+
y -= 4
|
| 242 |
+
c.setFont("DejaVu-Bold", 9)
|
| 243 |
+
c.setFillColor(colors.HexColor("#1f4e79"))
|
| 244 |
+
c.drawString(x0, y, SECTIONS[cat][loc])
|
| 245 |
+
c.setFillColor(colors.black)
|
| 246 |
+
y -= 14
|
| 247 |
+
c.setFont("DejaVu", 9.5)
|
| 248 |
+
c.drawString(cols[0], y, r["printed_name"])
|
| 249 |
+
abnormal = r["expected_flag"] not in ("normal", "unknown")
|
| 250 |
+
val = r["value_str"]
|
| 251 |
+
if report["show_hl"] and abnormal:
|
| 252 |
+
val += " H" if "high" in r["expected_flag"] else " L"
|
| 253 |
+
c.setFont("DejaVu-Bold", 9.5)
|
| 254 |
+
c.drawString(cols[1], y, val)
|
| 255 |
+
c.setFont("DejaVu", 9.5)
|
| 256 |
+
c.drawString(cols[2], y, r["unit"])
|
| 257 |
+
c.drawString(cols[3], y, r["ref_str"])
|
| 258 |
+
if report["layout"] == "dotted":
|
| 259 |
+
c.setDash(1, 2)
|
| 260 |
+
c.setLineWidth(0.3)
|
| 261 |
+
c.line(x0, y - 3.5, W - 18 * mm, y - 3.5)
|
| 262 |
+
c.setDash()
|
| 263 |
+
y -= 15
|
| 264 |
+
if y < 30 * mm:
|
| 265 |
+
c.showPage()
|
| 266 |
+
y = H - 25 * mm
|
| 267 |
+
|
| 268 |
+
c.setFont("DejaVu", 8)
|
| 269 |
+
c.drawString(x0, 15 * mm, h["footer"])
|
| 270 |
+
c.save()
|
| 271 |
+
|
| 272 |
+
|
| 273 |
+
def degrade_to_photo(pdf_path: Path, png_path: Path, rng: random.Random) -> None:
|
| 274 |
+
"""Rasterize page 1 and add mild photo-like noise: rotation, blur, contrast, grain."""
|
| 275 |
+
import subprocess
|
| 276 |
+
|
| 277 |
+
from PIL import Image, ImageEnhance, ImageFilter
|
| 278 |
+
|
| 279 |
+
stem = png_path.with_suffix("")
|
| 280 |
+
subprocess.run(["pdftoppm", "-png", "-r", "150", "-f", "1", "-l", "1", "-singlefile",
|
| 281 |
+
str(pdf_path), str(stem)], check=True)
|
| 282 |
+
img = Image.open(f"{stem}.png").convert("L")
|
| 283 |
+
img = img.rotate(rng.uniform(-1.5, 1.5), expand=True, fillcolor=245)
|
| 284 |
+
img = img.filter(ImageFilter.GaussianBlur(rng.uniform(0.2, 0.7)))
|
| 285 |
+
img = ImageEnhance.Contrast(img).enhance(rng.uniform(0.75, 1.0))
|
| 286 |
+
img = ImageEnhance.Brightness(img).enhance(rng.uniform(0.9, 1.05))
|
| 287 |
+
noise = Image.effect_noise(img.size, rng.uniform(6, 14))
|
| 288 |
+
img = Image.blend(img, noise, 0.06)
|
| 289 |
+
img.save(png_path)
|
| 290 |
+
|
| 291 |
+
|
| 292 |
+
def generate(out_dir: Path, n: int, start_seed: int = 0, photos: bool = True) -> list[dict]:
|
| 293 |
+
out_dir.mkdir(parents=True, exist_ok=True)
|
| 294 |
+
reports = []
|
| 295 |
+
for i in range(n):
|
| 296 |
+
seed = start_seed + i
|
| 297 |
+
rep = make_report(seed)
|
| 298 |
+
pdf = out_dir / f"{rep['id']}.pdf"
|
| 299 |
+
render_pdf(rep, pdf)
|
| 300 |
+
rep["pdf"] = pdf.name
|
| 301 |
+
if photos:
|
| 302 |
+
png = out_dir / f"{rep['id']}.png"
|
| 303 |
+
degrade_to_photo(pdf, png, random.Random(seed + 10_000))
|
| 304 |
+
rep["image"] = png.name
|
| 305 |
+
(out_dir / f"{rep['id']}.json").write_text(json.dumps(rep, ensure_ascii=False, indent=2), encoding="utf-8")
|
| 306 |
+
reports.append(rep)
|
| 307 |
+
return reports
|
| 308 |
+
|
| 309 |
+
|
| 310 |
+
if __name__ == "__main__":
|
| 311 |
+
import argparse
|
| 312 |
+
|
| 313 |
+
ap = argparse.ArgumentParser(description="Generate synthetic lab reports with ground truth.")
|
| 314 |
+
ap.add_argument("--out", default="benchmark/reports")
|
| 315 |
+
ap.add_argument("-n", type=int, default=100)
|
| 316 |
+
ap.add_argument("--seed", type=int, default=0)
|
| 317 |
+
ap.add_argument("--no-photos", action="store_true")
|
| 318 |
+
a = ap.parse_args()
|
| 319 |
+
reps = generate(Path(a.out), a.n, a.seed, photos=not a.no_photos)
|
| 320 |
+
print(f"wrote {len(reps)} reports to {a.out}")
|
samples/sample_en_report.pdf
ADDED
|
Binary file (44.9 kB). View file
|
|
|
samples/sample_fr_photo.png
ADDED
|
Git LFS Details
|
samples/sample_fr_urgent.png
ADDED
|
Git LFS Details
|
tests/__init__.py
ADDED
|
File without changes
|
tests/test_core.py
ADDED
|
@@ -0,0 +1,196 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""Tests for the safety-critical, deterministic parts of LabLens."""
|
| 2 |
+
import pytest
|
| 3 |
+
|
| 4 |
+
from lablens.catalog import load_catalog, match_test_name
|
| 5 |
+
from lablens.explain import explain, render_markdown, violates_guardrails
|
| 6 |
+
from lablens.extract import extract, parse_line, parse_range
|
| 7 |
+
from lablens.flagger import flag_all, flag_result
|
| 8 |
+
from lablens.models import Flag, Result
|
| 9 |
+
|
| 10 |
+
|
| 11 |
+
def R(key, value, unit, lo=None, hi=None):
|
| 12 |
+
return Result(test_key=key, raw_name=key, value=value, unit=unit, ref_low=lo, ref_high=hi)
|
| 13 |
+
|
| 14 |
+
|
| 15 |
+
# ---------- flagging ----------
|
| 16 |
+
|
| 17 |
+
def test_report_range_decides_low_high():
|
| 18 |
+
assert flag_result(R("hemoglobin", 11.0, "g/dL", 12, 16)).flag == Flag.LOW
|
| 19 |
+
assert flag_result(R("hemoglobin", 16.5, "g/dL", 12, 16)).flag == Flag.HIGH
|
| 20 |
+
assert flag_result(R("hemoglobin", 14.0, "g/dL", 12, 16)).flag == Flag.NORMAL
|
| 21 |
+
|
| 22 |
+
|
| 23 |
+
def test_critical_beats_range_and_converts_units():
|
| 24 |
+
# 6.8 mmol/L potassium is critical regardless of the printed range
|
| 25 |
+
assert flag_result(R("potassium", 6.8, "mmol/L", 3.5, 5.1)).flag == Flag.CRITICAL_HIGH
|
| 26 |
+
# French g/L glucose: 0.45 g/L = 45 mg/dL, below the 50 mg/dL critical threshold
|
| 27 |
+
assert flag_result(R("glucose", 0.45, "g/L", 0.70, 1.10)).flag == Flag.CRITICAL_LOW
|
| 28 |
+
# 4.5 g/L glucose = 450 mg/dL -> critical high
|
| 29 |
+
assert flag_result(R("glucose", 4.5, "g/L", 0.70, 1.10)).flag == Flag.CRITICAL_HIGH
|
| 30 |
+
|
| 31 |
+
|
| 32 |
+
def test_cell_count_units_giga_per_litre():
|
| 33 |
+
# 40 G/L platelets = 40 x10^3/µL -> critical low
|
| 34 |
+
assert flag_result(R("platelets", 40, "G/L", 150, 400)).flag == Flag.CRITICAL_LOW
|
| 35 |
+
|
| 36 |
+
|
| 37 |
+
def test_implausible_value_is_never_normal_or_urgent():
|
| 38 |
+
# 436% HbA1c is a dropped decimal point, not a medical emergency or a normal value
|
| 39 |
+
r = flag_result(R("hba1c", 436, "%", 4.0, 5.6))
|
| 40 |
+
assert r.flag == Flag.CHECK
|
| 41 |
+
a = flag_all([R("hemoglobin", 151, "g/dL", 12, 17)])
|
| 42 |
+
assert a.results[0].flag == Flag.CHECK and not a.urgent
|
| 43 |
+
|
| 44 |
+
|
| 45 |
+
def test_catalog_range_used_when_report_has_none():
|
| 46 |
+
r = flag_result(R("sodium", 150, "mmol/L"))
|
| 47 |
+
assert r.flag == Flag.HIGH and r.ref_source == "catalog"
|
| 48 |
+
|
| 49 |
+
|
| 50 |
+
def test_unknown_unit_skips_critical_but_uses_printed_range():
|
| 51 |
+
r = flag_result(R("potassium", 7.0, "zz/qq", 3.5, 5.1))
|
| 52 |
+
assert r.flag == Flag.HIGH # not escalated: we can't be sure of the unit
|
| 53 |
+
|
| 54 |
+
|
| 55 |
+
def test_unknown_unit_and_no_range_is_unknown():
|
| 56 |
+
assert flag_result(R("potassium", 4.0, "zz/qq")).flag == Flag.UNKNOWN
|
| 57 |
+
|
| 58 |
+
|
| 59 |
+
def test_urgent_flag_set():
|
| 60 |
+
a = flag_all([R("hemoglobin", 14, "g/dL", 12, 16), R("sodium", 118, "mmol/L", 135, 145)])
|
| 61 |
+
assert a.urgent
|
| 62 |
+
|
| 63 |
+
|
| 64 |
+
def test_every_critical_threshold_is_plausible():
|
| 65 |
+
for td in load_catalog().values():
|
| 66 |
+
for v in td.critical.values():
|
| 67 |
+
assert td.plausible[0] < v < td.plausible[1], td.key
|
| 68 |
+
|
| 69 |
+
|
| 70 |
+
# ---------- name matching ----------
|
| 71 |
+
|
| 72 |
+
@pytest.mark.parametrize("name,key", [
|
| 73 |
+
("Glycémie à jeun", "glucose"), ("GLYCEMIE A JEUN", "glucose"), ("Kaliémie", "potassium"),
|
| 74 |
+
("Hemiogiouine", "hemoglobin"), ("HBAIC", "hba1c"), ("Ldi-choiestéroi", "ldl"),
|
| 75 |
+
("Hdl-chatesteroi", "hdl"), ("K", "potassium"), ("NA", "sodium"), ("الكرياتينين", "creatinine"),
|
| 76 |
+
])
|
| 77 |
+
def test_name_matching(name, key):
|
| 78 |
+
assert match_test_name(name)[0] == key
|
| 79 |
+
|
| 80 |
+
|
| 81 |
+
@pytest.mark.parametrize("name", ["Patient", "Collection date", "No", "Dossier", "Page"])
|
| 82 |
+
def test_non_tests_do_not_match(name):
|
| 83 |
+
assert match_test_name(name)[0] is None
|
| 84 |
+
|
| 85 |
+
|
| 86 |
+
# ---------- line and range parsing ----------
|
| 87 |
+
|
| 88 |
+
@pytest.mark.parametrize("text,expected", [
|
| 89 |
+
("12.0 - 16.0", (12.0, 16.0)), ("(0,70-1,10)", (0.70, 1.10)), ("[4,00 – 5,60]", (4.0, 5.6)),
|
| 90 |
+
("< 2,00", (None, 2.0)), ("> 0,40", (0.40, None)), ("[4.20 - 5.901", (4.20, 5.90)), ("", (None, None)),
|
| 91 |
+
])
|
| 92 |
+
def test_parse_range(text, expected):
|
| 93 |
+
assert parse_range(text) == expected
|
| 94 |
+
|
| 95 |
+
|
| 96 |
+
def test_parse_line_french_with_marker():
|
| 97 |
+
r = parse_line("Kaliémie 5,36H mmol/L 3,50 - 5,10")
|
| 98 |
+
assert (r.test_key, r.value, r.unit, r.ref_low, r.ref_high) == ("potassium", 5.36, "mmol/L", 3.5, 5.1)
|
| 99 |
+
|
| 100 |
+
|
| 101 |
+
def test_parse_line_skips_ocr_noise_and_stray_digit():
|
| 102 |
+
assert parse_line("TSH ae 3.11 mIU/L [0.40 - 4.00]").value == 3.11
|
| 103 |
+
assert parse_line("K 7 3.79 mmol/L [3.50 - 5.10]").value == 3.79
|
| 104 |
+
|
| 105 |
+
|
| 106 |
+
def test_parse_line_ignores_headers():
|
| 107 |
+
assert parse_line("Date de prélèvement: 2026-04-05") is None
|
| 108 |
+
assert parse_line("Patient: Karim Mansour") is None
|
| 109 |
+
|
| 110 |
+
|
| 111 |
+
# ---------- end to end on samples ----------
|
| 112 |
+
|
| 113 |
+
def test_sample_pdf_end_to_end():
|
| 114 |
+
results, _, lang, method = extract("samples/sample_en_report.pdf")
|
| 115 |
+
assert method == "pdf-text" and lang == "en" and len(results) == 13
|
| 116 |
+
|
| 117 |
+
|
| 118 |
+
def test_sample_urgent_photo_end_to_end():
|
| 119 |
+
results, _, lang, method = extract("samples/sample_fr_urgent.png")
|
| 120 |
+
a = flag_all(results)
|
| 121 |
+
assert method == "ocr" and lang == "fr" and a.urgent
|
| 122 |
+
|
| 123 |
+
|
| 124 |
+
# ---------- explanation guardrails ----------
|
| 125 |
+
|
| 126 |
+
@pytest.mark.parametrize("text", [
|
| 127 |
+
"You have diabetes.", "Vous avez une anémie.", "Take 1000 mg of iron daily.", "Stop taking your pills.",
|
| 128 |
+
])
|
| 129 |
+
def test_guardrails_block_diagnosis_and_dosing(text):
|
| 130 |
+
assert violates_guardrails(text)
|
| 131 |
+
|
| 132 |
+
|
| 133 |
+
def test_guardrails_allow_hedged_language():
|
| 134 |
+
assert not violates_guardrails("A high value can be linked to diabetes; your doctor can confirm.")
|
| 135 |
+
|
| 136 |
+
|
| 137 |
+
def test_template_explanation_without_api_key(monkeypatch):
|
| 138 |
+
monkeypatch.delenv("HF_TOKEN", raising=False)
|
| 139 |
+
monkeypatch.delenv("LLM_API_KEY", raising=False)
|
| 140 |
+
a = flag_all([R("ferritin", 8, "ng/mL", 15, 200), R("tsh", 2.0, "mIU/L", 0.4, 4.0)])
|
| 141 |
+
exp, source = explain(a, "fr")
|
| 142 |
+
assert source.startswith("template")
|
| 143 |
+
md = render_markdown(a, exp, "fr")
|
| 144 |
+
assert "Ferritine" in md and "Bas" in md and len(exp["questions"]) == 1
|
| 145 |
+
|
| 146 |
+
|
| 147 |
+
# ---------- LLM path (mocked: no network in tests) ----------
|
| 148 |
+
|
| 149 |
+
class _FakeClient:
|
| 150 |
+
def __init__(self, content):
|
| 151 |
+
msg = type("M", (), {"content": content})
|
| 152 |
+
choice = type("C", (), {"message": msg})
|
| 153 |
+
self.chat = type("Ch", (), {"completions": type("Co", (), {
|
| 154 |
+
"create": staticmethod(lambda **kw: type("R", (), {"choices": [choice]}))})})
|
| 155 |
+
|
| 156 |
+
|
| 157 |
+
def _patch_llm(monkeypatch, content):
|
| 158 |
+
import openai
|
| 159 |
+
monkeypatch.setenv("HF_TOKEN", "test")
|
| 160 |
+
monkeypatch.setattr(openai, "OpenAI", lambda **kw: _FakeClient(content))
|
| 161 |
+
|
| 162 |
+
|
| 163 |
+
def test_llm_output_used_when_safe(monkeypatch):
|
| 164 |
+
_patch_llm(monkeypatch, '{"summary": "Mostly fine.", "results": [{"key": "ferritin", "explanation": '
|
| 165 |
+
'"Low iron stores."}, {"key": "made_up", "explanation": "x"}], "questions": ["Why?"]}')
|
| 166 |
+
a = flag_all([R("ferritin", 8, "ng/mL", 15, 200)])
|
| 167 |
+
exp, source = explain(a, "en")
|
| 168 |
+
assert source == "llm"
|
| 169 |
+
assert [x["key"] for x in exp["results"]] == ["ferritin"] # hallucinated test dropped
|
| 170 |
+
|
| 171 |
+
|
| 172 |
+
def test_llm_diagnosis_falls_back_to_template(monkeypatch):
|
| 173 |
+
_patch_llm(monkeypatch, '{"summary": "You have anemia.", "results": [], "questions": []}')
|
| 174 |
+
a = flag_all([R("hemoglobin", 9, "g/dL", 12, 16)])
|
| 175 |
+
exp, source = explain(a, "en")
|
| 176 |
+
assert source.startswith("template") and "guardrail" in source
|
| 177 |
+
|
| 178 |
+
|
| 179 |
+
def test_llm_garbage_falls_back_to_template(monkeypatch):
|
| 180 |
+
_patch_llm(monkeypatch, "Sorry, I can't help with that.")
|
| 181 |
+
exp, source = explain(flag_all([R("tsh", 9, "mIU/L", 0.4, 4.0)]), "ar")
|
| 182 |
+
assert source.startswith("template")
|
| 183 |
+
|
| 184 |
+
|
| 185 |
+
def test_lab_marker_contradiction_becomes_check():
|
| 186 |
+
r = R("ast", 6.5, "UI/L", 3.0, 48.0) # range misread (8,0 -> 3,0) but lab printed "L"
|
| 187 |
+
r.report_marker = "L"
|
| 188 |
+
assert flag_result(r).flag == Flag.CHECK
|
| 189 |
+
ok = R("ast", 6.5, "UI/L", 8.0, 48.0)
|
| 190 |
+
ok.report_marker = "L"
|
| 191 |
+
assert flag_result(ok).flag == Flag.LOW
|
| 192 |
+
|
| 193 |
+
|
| 194 |
+
def test_glued_marker_with_trailing_punctuation():
|
| 195 |
+
r = parse_line("Vitamine d 22L. ng/mL 30 - 100")
|
| 196 |
+
assert (r.value, r.report_marker, r.ref_low) == (22.0, "L", 30.0)
|