Imene-ee commited on
Commit
dab7db2
·
verified ·
1 Parent(s): a1f3d11

Upload 19 files

Browse files
.gitattributes CHANGED
@@ -33,3 +33,5 @@ saved_model/**/* filter=lfs diff=lfs merge=lfs -text
33
  *.zip filter=lfs diff=lfs merge=lfs -text
34
  *.zst filter=lfs diff=lfs merge=lfs -text
35
  *tfevents* filter=lfs diff=lfs merge=lfs -text
 
 
 
33
  *.zip filter=lfs diff=lfs merge=lfs -text
34
  *.zst filter=lfs diff=lfs merge=lfs -text
35
  *tfevents* filter=lfs diff=lfs merge=lfs -text
36
+ samples/sample_fr_photo.png filter=lfs diff=lfs merge=lfs -text
37
+ samples/sample_fr_urgent.png filter=lfs diff=lfs merge=lfs -text
.github/workflows/ci.yml ADDED
@@ -0,0 +1,24 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ name: tests
2
+
3
+ on:
4
+ push:
5
+ pull_request:
6
+
7
+ jobs:
8
+ test:
9
+ runs-on: ubuntu-latest
10
+ steps:
11
+ - uses: actions/checkout@v4
12
+ - name: System packages (OCR + PDF tools)
13
+ run: |
14
+ sudo apt-get update
15
+ sudo apt-get install -y tesseract-ocr tesseract-ocr-eng tesseract-ocr-fra tesseract-ocr-ara poppler-utils fonts-dejavu-core
16
+ - uses: actions/setup-python@v5
17
+ with:
18
+ python-version: "3.12"
19
+ - run: pip install -r requirements.txt pytest
20
+ - run: pytest -q
21
+ - name: Mini benchmark (20 reports)
22
+ run: |
23
+ python -m lablens.synth.generator --out /tmp/bench -n 20 --seed 1000
24
+ python -m lablens.evaluate --bench /tmp/bench --out /tmp/results.json
benchmark/results.json ADDED
@@ -0,0 +1,86 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "pdf/all": {
3
+ "reports": 200,
4
+ "rows": 1989,
5
+ "recall": 1.0,
6
+ "precision": 1.0,
7
+ "value_acc": 1.0,
8
+ "unit_acc": 1.0,
9
+ "range_acc": 1.0,
10
+ "flag_acc": 1.0,
11
+ "critical_recall": 1.0,
12
+ "critical_rows": 59,
13
+ "dangerous_misses": 0,
14
+ "abnormal_rows": 584
15
+ },
16
+ "pdf/en": {
17
+ "reports": 101,
18
+ "rows": 1009,
19
+ "recall": 1.0,
20
+ "precision": 1.0,
21
+ "value_acc": 1.0,
22
+ "unit_acc": 1.0,
23
+ "range_acc": 1.0,
24
+ "flag_acc": 1.0,
25
+ "critical_recall": 1.0,
26
+ "critical_rows": 25,
27
+ "dangerous_misses": 0,
28
+ "abnormal_rows": 289
29
+ },
30
+ "pdf/fr": {
31
+ "reports": 99,
32
+ "rows": 980,
33
+ "recall": 1.0,
34
+ "precision": 1.0,
35
+ "value_acc": 1.0,
36
+ "unit_acc": 1.0,
37
+ "range_acc": 1.0,
38
+ "flag_acc": 1.0,
39
+ "critical_recall": 1.0,
40
+ "critical_rows": 34,
41
+ "dangerous_misses": 0,
42
+ "abnormal_rows": 295
43
+ },
44
+ "photo/all": {
45
+ "reports": 200,
46
+ "rows": 1989,
47
+ "recall": 0.9316,
48
+ "precision": 1.0,
49
+ "value_acc": 0.9595,
50
+ "unit_acc": 0.9709,
51
+ "range_acc": 0.9687,
52
+ "flag_acc": 0.8924,
53
+ "critical_recall": 0.8475,
54
+ "critical_rows": 59,
55
+ "dangerous_misses": 2,
56
+ "abnormal_rows": 584
57
+ },
58
+ "photo/en": {
59
+ "reports": 101,
60
+ "rows": 1009,
61
+ "recall": 0.9217,
62
+ "precision": 1.0,
63
+ "value_acc": 0.9624,
64
+ "unit_acc": 0.9796,
65
+ "range_acc": 0.9753,
66
+ "flag_acc": 0.885,
67
+ "critical_recall": 0.88,
68
+ "critical_rows": 25,
69
+ "dangerous_misses": 1,
70
+ "abnormal_rows": 289
71
+ },
72
+ "photo/fr": {
73
+ "reports": 99,
74
+ "rows": 980,
75
+ "recall": 0.9418,
76
+ "precision": 1.0,
77
+ "value_acc": 0.9567,
78
+ "unit_acc": 0.9621,
79
+ "range_acc": 0.9621,
80
+ "flag_acc": 0.9,
81
+ "critical_recall": 0.8235,
82
+ "critical_rows": 34,
83
+ "dangerous_misses": 1,
84
+ "abnormal_rows": 295
85
+ }
86
+ }
benchmark/results.md ADDED
@@ -0,0 +1,8 @@
 
 
 
 
 
 
 
 
 
1
+ | split | reports | rows | recall | precision | value_acc | unit_acc | range_acc | flag_acc | critical_recall | dangerous_misses |
2
+ |---|---|---|---|---|---|---|---|---|---|---|
3
+ | pdf/all | 200 | 1989 | 100.0% | 100.0% | 100.0% | 100.0% | 100.0% | 100.0% | 100.0% | 0 |
4
+ | pdf/en | 101 | 1009 | 100.0% | 100.0% | 100.0% | 100.0% | 100.0% | 100.0% | 100.0% | 0 |
5
+ | pdf/fr | 99 | 980 | 100.0% | 100.0% | 100.0% | 100.0% | 100.0% | 100.0% | 100.0% | 0 |
6
+ | photo/all | 200 | 1989 | 93.2% | 100.0% | 96.0% | 97.1% | 96.9% | 89.2% | 84.8% | 2 |
7
+ | photo/en | 101 | 1009 | 92.2% | 100.0% | 96.2% | 98.0% | 97.5% | 88.5% | 88.0% | 1 |
8
+ | photo/fr | 99 | 980 | 94.2% | 100.0% | 95.7% | 96.2% | 96.2% | 90.0% | 82.3% | 1 |
lablens/__init__.py ADDED
File without changes
lablens/catalog.py ADDED
@@ -0,0 +1,144 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ """Load the test catalog and resolve names and units against it."""
2
+ from __future__ import annotations
3
+
4
+ import re
5
+ import unicodedata
6
+ from dataclasses import dataclass, field
7
+ from functools import lru_cache
8
+ from pathlib import Path
9
+
10
+ import yaml
11
+ from rapidfuzz import fuzz, process
12
+
13
+ CATALOG_PATH = Path(__file__).parent / "data" / "catalog.yaml"
14
+
15
+
16
+ def normalize_text(s: str) -> str:
17
+ """Lowercase, strip accents (Latin only), collapse whitespace."""
18
+ s = s.strip().lower()
19
+ # Strip Latin diacritics but keep Arabic letters intact.
20
+ decomposed = unicodedata.normalize("NFKD", s)
21
+ s = "".join(c for c in decomposed if not (unicodedata.combining(c) and ord(c) < 0x0600))
22
+ s = s.replace("μ", "µ")
23
+ return re.sub(r"\s+", " ", s)
24
+
25
+
26
+ def normalize_unit(u: str) -> str:
27
+ u = normalize_text(u or "").replace(" ", "")
28
+ u = u.replace("μ", "µ").replace("×", "").replace("x10", "10")
29
+ u = u.replace("10e", "10^").replace("10*", "10^")
30
+ u = re.sub(r"10\^?(\d)", r"10^\1", u)
31
+ return u
32
+
33
+
34
+ @dataclass
35
+ class TestDef:
36
+ key: str
37
+ display: dict[str, str]
38
+ category: str
39
+ canonical_unit: str
40
+ units: dict[str, float]
41
+ typical_range: tuple[float, float]
42
+ critical: dict[str, float]
43
+ plausible: tuple[float, float]
44
+ aliases: dict[str, list[str]]
45
+ kb: dict[str, str]
46
+ all_aliases: list[str] = field(default_factory=list)
47
+
48
+ def to_canonical(self, value: float, unit: str | None) -> float | None:
49
+ """Convert a value to the canonical unit. Returns None if the unit is unknown."""
50
+ if not unit:
51
+ return value # assume canonical when no unit printed
52
+ nu = normalize_unit(unit)
53
+ for spelled, factor in self.units.items():
54
+ if normalize_unit(spelled) == nu:
55
+ return value * factor
56
+ if nu == normalize_unit(self.canonical_unit):
57
+ return value
58
+ return None
59
+
60
+
61
+ @lru_cache(maxsize=1)
62
+ def load_catalog(path: str | None = None) -> dict[str, TestDef]:
63
+ raw = yaml.safe_load(Path(path or CATALOG_PATH).read_text(encoding="utf-8"))
64
+ out: dict[str, TestDef] = {}
65
+ for key, t in raw["tests"].items():
66
+ td = TestDef(
67
+ key=key,
68
+ display=t["display"],
69
+ category=t["category"],
70
+ canonical_unit=t["canonical_unit"],
71
+ units=t["units"],
72
+ typical_range=tuple(t["typical_range"]),
73
+ critical=t.get("critical") or {},
74
+ plausible=tuple(t["plausible"]),
75
+ aliases=t["aliases"],
76
+ kb=t["kb"],
77
+ )
78
+ td.all_aliases = sorted(
79
+ {normalize_text(a) for lang in td.aliases.values() for a in lang}, key=len, reverse=True
80
+ )
81
+ out[key] = td
82
+ return out
83
+
84
+
85
+ @lru_cache(maxsize=1)
86
+ def _alias_index() -> dict[str, str]:
87
+ idx: dict[str, str] = {}
88
+ for key, td in load_catalog().items():
89
+ for a in td.all_aliases:
90
+ idx.setdefault(a, key)
91
+ return idx
92
+
93
+
94
+ # Characters Tesseract commonly confuses, folded to one representative before fuzzy
95
+ # matching: "Hemiogiouine" and "hemoglobine" end up much closer.
96
+ _OCR_FOLD = str.maketrans({"l": "i", "1": "i", "|": "i", "!": "i", "j": "i", "t": "i",
97
+ "0": "o", "a": "o", "e": "o", "c": "o", "u": "o", "b": "o", "g": "o"})
98
+
99
+
100
+ def _fold(s: str) -> str:
101
+ s = s.replace("rn", "m").replace("-", " ").replace("_", " ")
102
+ return re.sub(r"\s+", " ", s.translate(_OCR_FOLD)).strip()
103
+
104
+
105
+ @lru_cache(maxsize=1)
106
+ def _folded_index() -> dict[str, set[str]]:
107
+ out: dict[str, set[str]] = {}
108
+ for a, k in _alias_index().items():
109
+ if len(a) >= 4:
110
+ out.setdefault(_fold(a), set()).add(k)
111
+ return out
112
+
113
+
114
+ def match_test_name(name: str, min_score: int = 80, margin: float = 6.0) -> tuple[str | None, float]:
115
+ """Map a printed test name to a catalog key.
116
+
117
+ Exact alias first. Otherwise fuzzy matching on OCR-folded strings, accepted only when
118
+ the best test beats the runner-up *test* by a margin. HDL vs LDL differ by one letter
119
+ and have opposite meanings, so an ambiguous match is rejected rather than guessed.
120
+ """
121
+ n = normalize_text(name)
122
+ n = re.sub(r"[:.\-–_]+$", "", n).strip()
123
+ idx = _alias_index()
124
+ if n in idx:
125
+ return idx[n], 100.0
126
+ if len(n) < 4:
127
+ # Short aliases (hb, k, na, a1c) only match exactly after OCR folding, and only
128
+ # when the folded form is unambiguous ("Alc" -> a1c).
129
+ if len(n) < 3:
130
+ return None, 0.0 # never guess 1-2 letter tokens ("No", "N°" sample numbers)
131
+ keys = {k for a, k in idx.items() if len(a) < 4 and _fold(a) == _fold(n)}
132
+ return (keys.pop(), 90.0) if len(keys) == 1 else (None, 0.0)
133
+ folded = _folded_index()
134
+ hits = process.extract(_fold(n), list(folded), scorer=fuzz.ratio, limit=12)
135
+ best_per_key: dict[str, float] = {}
136
+ for alias, score, _ in hits:
137
+ for k in folded[alias]:
138
+ best_per_key[k] = max(best_per_key.get(k, 0.0), score)
139
+ ranked = sorted(best_per_key.items(), key=lambda kv: kv[1], reverse=True)
140
+ if not ranked or ranked[0][1] < min_score:
141
+ return None, ranked[0][1] if ranked else 0.0
142
+ if len(ranked) > 1 and ranked[0][1] - ranked[1][1] < margin:
143
+ return None, ranked[0][1]
144
+ return ranked[0][0], float(ranked[0][1])
lablens/data/catalog.yaml ADDED
@@ -0,0 +1,407 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ # LabLens test catalog.
2
+ #
3
+ # Each test has:
4
+ # canonical_unit unit used internally for critical thresholds
5
+ # units accepted unit spellings -> multiply factor to canonical unit
6
+ # typical_range adult reference interval in canonical unit (used ONLY when the
7
+ # report prints no range of its own)
8
+ # critical life-threatening thresholds in canonical unit (deterministic escalation)
9
+ # plausible physiologically possible bounds; values outside are treated as a
10
+ # likely misread (e.g. a dropped decimal point) and sent for review
11
+ # aliases names as printed on reports (en / fr / ar), matched case-insensitively
12
+ # kb plain-language knowledge used to ground explanations
13
+ #
14
+ # Critical thresholds are conservative values drawn from commonly published hospital
15
+ # critical-value lists. They are a safety net, not a diagnosis, and every lab sets its own.
16
+
17
+ tests:
18
+ hemoglobin:
19
+ display: {en: Hemoglobin, fr: Hémoglobine, ar: الهيموغلوبين, darija: "الهيموغلوبين"}
20
+ category: blood_count
21
+ canonical_unit: g/dL
22
+ units: {"g/dl": 1, "g/l": 0.1, "mmol/l": 1.611}
23
+ typical_range: [12.0, 17.0]
24
+ critical: {low: 7.0, high: 20.0}
25
+ plausible: [2, 25]
26
+ aliases:
27
+ en: [hemoglobin, haemoglobin, hgb, hb]
28
+ fr: [hémoglobine, hemoglobine]
29
+ ar: [الهيموغلوبين, هيموغلوبين, خضاب الدم]
30
+ kb:
31
+ what: The protein in red blood cells that carries oxygen from the lungs to the rest of the body.
32
+ low: Often seen with anemia, which can come from iron deficiency, blood loss, low vitamin B12 or folate, or chronic illness. People may feel tired or short of breath.
33
+ high: Can be seen with dehydration, smoking, living at high altitude, or conditions where the body makes too many red cells.
34
+
35
+ hematocrit:
36
+ display: {en: Hematocrit, fr: Hématocrite, ar: الهيماتوكريت, darija: "الهيماتوكريت"}
37
+ category: blood_count
38
+ canonical_unit: "%"
39
+ units: {"%": 1, "l/l": 100}
40
+ typical_range: [36.0, 50.0]
41
+ critical: {low: 20.0, high: 60.0}
42
+ plausible: [8, 75]
43
+ aliases:
44
+ en: [hematocrit, haematocrit, hct]
45
+ fr: [hématocrite, hematocrite]
46
+ ar: [الهيماتوكريت, هيماتوكريت]
47
+ kb:
48
+ what: The share of your blood volume made up of red blood cells.
49
+ low: Usually moves together with hemoglobin and can point to anemia or blood loss.
50
+ high: Can reflect dehydration or an excess of red blood cells.
51
+
52
+ rbc:
53
+ display: {en: Red blood cells, fr: Globules rouges, ar: كريات الدم الحمراء, darija: "الكريات الحمرا"}
54
+ category: blood_count
55
+ canonical_unit: "10^6/µL"
56
+ units: {"10^6/µl": 1, "10^6/ul": 1, "10^6/mm3": 1, "10^12/l": 1, "t/l": 1, "m/µl": 1, "/mm3": 0.000001}
57
+ typical_range: [4.2, 5.9]
58
+ critical: {}
59
+ plausible: [1, 9]
60
+ aliases:
61
+ en: [red blood cells, red blood cell count, rbc, erythrocytes]
62
+ fr: [globules rouges, hématies, hematies, érythrocytes]
63
+ ar: [كريات الدم الحمراء, الكريات الحمراء]
64
+ kb:
65
+ what: The number of red blood cells, which carry oxygen.
66
+ low: Can be part of anemia.
67
+ high: Can be seen with dehydration or conditions that raise red cell production.
68
+
69
+ mcv:
70
+ display: {en: Mean corpuscular volume, fr: VGM, ar: متوسط حجم الكرية, darija: "حجم الكريات (VGM)"}
71
+ category: blood_count
72
+ canonical_unit: fL
73
+ units: {"fl": 1, "µm3": 1, "um3": 1}
74
+ typical_range: [80.0, 100.0]
75
+ critical: {}
76
+ plausible: [50, 140]
77
+ aliases:
78
+ en: [mcv, mean corpuscular volume, mean cell volume]
79
+ fr: [vgm, volume globulaire moyen]
80
+ ar: [متوسط حجم الكرية, حجم الكرية الوسطي]
81
+ kb:
82
+ what: The average size of your red blood cells.
83
+ low: Small red cells are often linked to iron deficiency or inherited conditions such as thalassemia trait.
84
+ high: Large red cells can be linked to low vitamin B12 or folate, alcohol use, some medicines, or thyroid and liver issues.
85
+
86
+ wbc:
87
+ display: {en: White blood cells, fr: Globules blancs, ar: كريات الدم البيضاء, darija: "الكريات البيض"}
88
+ category: blood_count
89
+ canonical_unit: "10^3/µL"
90
+ units: {"10^3/µl": 1, "10^3/ul": 1, "10^3/mm3": 1, "10^9/l": 1, "g/l": 1, "k/µl": 1, "/mm3": 0.001, "/µl": 0.001}
91
+ typical_range: [4.0, 11.0]
92
+ critical: {low: 2.0, high: 30.0}
93
+ plausible: [0.1, 300]
94
+ aliases:
95
+ en: [white blood cells, white blood cell count, wbc, leukocytes, leucocytes]
96
+ fr: [globules blancs, leucocytes]
97
+ ar: [كريات الدم البيضاء, الكريات البيضاء]
98
+ kb:
99
+ what: The cells of your immune system that fight infection.
100
+ low: Can follow viral infections, some medicines, or problems with bone-marrow production; very low counts raise infection risk.
101
+ high: Often a response to infection, inflammation, stress or steroids; less often a blood disorder.
102
+
103
+ platelets:
104
+ display: {en: Platelets, fr: Plaquettes, ar: الصفائح الدموية, darija: "الصفائح (plaquettes)"}
105
+ category: blood_count
106
+ canonical_unit: "10^3/µL"
107
+ units: {"10^3/µl": 1, "10^3/ul": 1, "10^3/mm3": 1, "10^9/l": 1, "g/l": 1, "k/µl": 1, "/mm3": 0.001, "/µl": 0.001}
108
+ typical_range: [150.0, 400.0]
109
+ critical: {low: 50.0, high: 1000.0}
110
+ plausible: [2, 2500]
111
+ aliases:
112
+ en: [platelets, platelet count, plt]
113
+ fr: [plaquettes, numération plaquettaire]
114
+ ar: [الصفائح الدموية, الصفائح]
115
+ kb:
116
+ what: Small cell fragments that help your blood clot.
117
+ low: Can follow infections, some medicines, or immune or bone-marrow conditions; very low counts raise bleeding risk.
118
+ high: Often a reaction to inflammation, infection or iron deficiency.
119
+
120
+ glucose:
121
+ display: {en: Fasting glucose, fr: Glycémie à jeun, ar: سكر الدم الصائم, darija: "السكر فالدم (على الريق)"}
122
+ category: metabolic
123
+ canonical_unit: mg/dL
124
+ units: {"mg/dl": 1, "g/l": 100, "mmol/l": 18.016}
125
+ typical_range: [70.0, 99.0]
126
+ critical: {low: 50.0, high: 400.0}
127
+ plausible: [10, 1500]
128
+ aliases:
129
+ en: [fasting glucose, glucose, fasting blood sugar, blood glucose]
130
+ fr: [glycémie à jeun, glycemie a jeun, glycémie, glycemie, glucose]
131
+ ar: [سكر الدم الصائم, سكر الدم, الغلوكوز]
132
+ kb:
133
+ what: The amount of sugar in your blood after not eating, usually for 8 hours or more.
134
+ low: Can happen with some diabetes medicines, long fasting or, rarely, hormone problems.
135
+ high: Can point to prediabetes or diabetes, especially if repeated; stress, illness and some medicines can also raise it.
136
+
137
+ hba1c:
138
+ display: {en: HbA1c, fr: Hémoglobine glyquée, ar: الهيموغلوبين السكري, darija: "السكر التراكمي (HbA1c)"}
139
+ category: metabolic
140
+ canonical_unit: "%"
141
+ units: {"%": 1}
142
+ typical_range: [4.0, 5.6]
143
+ critical: {}
144
+ plausible: [3, 20]
145
+ aliases:
146
+ en: [hba1c, a1c, glycated hemoglobin, glycated haemoglobin]
147
+ fr: [hémoglobine glyquée, hemoglobine glyquee, hba1c]
148
+ ar: [الهيموغلوبين السكري, السكر التراكمي]
149
+ kb:
150
+ what: Your average blood sugar over the past two to three months.
151
+ low: Rarely a concern on its own; can be affected by anemia or recent blood loss.
152
+ high: 5.7 to 6.4% is commonly called the prediabetes range and 6.5% or above is a threshold used for diagnosing diabetes, which a doctor confirms.
153
+
154
+ creatinine:
155
+ display: {en: Creatinine, fr: Créatinine, ar: الكرياتينين, darija: "الكرياتينين"}
156
+ category: kidney
157
+ canonical_unit: mg/dL
158
+ units: {"mg/dl": 1, "µmol/l": 0.01131, "umol/l": 0.01131, "mg/l": 0.1}
159
+ typical_range: [0.6, 1.2]
160
+ critical: {}
161
+ plausible: [0.1, 25]
162
+ aliases:
163
+ en: [creatinine, serum creatinine]
164
+ fr: [créatinine, creatinine, créatininémie]
165
+ ar: [الكرياتينين, كرياتينين]
166
+ kb:
167
+ what: A waste product from muscles that the kidneys filter out; it is used to estimate kidney function.
168
+ low: Usually not a concern; often seen with low muscle mass.
169
+ high: Can mean the kidneys are filtering less well; dehydration, high muscle mass and some medicines can also raise it.
170
+
171
+ urea:
172
+ display: {en: Urea, fr: Urée, ar: اليوريا, darija: "اليوريا (urée)"}
173
+ category: kidney
174
+ canonical_unit: mg/dL
175
+ units: {"mg/dl": 1, "g/l": 100, "mmol/l": 6.006}
176
+ typical_range: [15.0, 45.0]
177
+ critical: {}
178
+ plausible: [2, 400]
179
+ aliases:
180
+ en: [urea, blood urea]
181
+ fr: [urée, uree, urémie]
182
+ ar: [اليوريا, البولة]
183
+ kb:
184
+ what: A waste product from protein breakdown that the kidneys remove.
185
+ low: Can be seen with low protein intake or liver conditions.
186
+ high: Can reflect dehydration, a high-protein diet, or reduced kidney function.
187
+
188
+ sodium:
189
+ display: {en: Sodium, fr: Sodium, ar: الصوديوم, darija: "الصوديوم"}
190
+ category: electrolytes
191
+ canonical_unit: mmol/L
192
+ units: {"mmol/l": 1, "meq/l": 1}
193
+ typical_range: [135.0, 145.0]
194
+ critical: {low: 120.0, high: 160.0}
195
+ plausible: [95, 190]
196
+ aliases:
197
+ en: [sodium, na]
198
+ fr: [sodium, natrémie, natremie, na+]
199
+ ar: [الصوديوم]
200
+ kb:
201
+ what: A salt that keeps your body's water balance and nerves working.
202
+ low: Can come from drinking a lot of water, some diuretics, vomiting or hormone and heart conditions.
203
+ high: Usually reflects dehydration or not drinking enough.
204
+
205
+ potassium:
206
+ display: {en: Potassium, fr: Potassium, ar: البوتاسيوم, darija: "البوتاسيوم"}
207
+ category: electrolytes
208
+ canonical_unit: mmol/L
209
+ units: {"mmol/l": 1, "meq/l": 1}
210
+ typical_range: [3.5, 5.1]
211
+ critical: {low: 2.8, high: 6.2}
212
+ plausible: [1.2, 10]
213
+ aliases:
214
+ en: [potassium, k]
215
+ fr: [potassium, kaliémie, kaliemie, k+]
216
+ ar: [البوتاسيوم]
217
+ kb:
218
+ what: A mineral essential for your heart rhythm and muscles.
219
+ low: Can come from vomiting, diarrhea or some diuretics.
220
+ high: Can come from kidney problems or some medicines; a sample that was handled roughly can also read falsely high.
221
+
222
+ calcium:
223
+ display: {en: Calcium, fr: Calcium, ar: الكالسيوم, darija: "الكالسيوم"}
224
+ category: electrolytes
225
+ canonical_unit: mg/dL
226
+ units: {"mg/dl": 1, "mmol/l": 4.008, "mg/l": 0.1}
227
+ typical_range: [8.5, 10.5]
228
+ critical: {low: 6.5, high: 13.0}
229
+ plausible: [3, 20]
230
+ aliases:
231
+ en: [calcium, total calcium, ca]
232
+ fr: [calcium, calcémie, calcemie]
233
+ ar: [الكالسيوم]
234
+ kb:
235
+ what: A mineral for bones, muscles, nerves and the heart.
236
+ low: Can be linked to low vitamin D, low albumin or parathyroid conditions.
237
+ high: Can be linked to parathyroid conditions, too much vitamin D or calcium supplements, or other conditions a doctor should look at.
238
+
239
+ cholesterol_total:
240
+ display: {en: Total cholesterol, fr: Cholestérol total, ar: الكوليسترول الكلي, darija: "الكوليسترول الكل"}
241
+ category: lipids
242
+ canonical_unit: mg/dL
243
+ units: {"mg/dl": 1, "g/l": 100, "mmol/l": 38.67}
244
+ typical_range: [0.0, 200.0]
245
+ critical: {}
246
+ plausible: [40, 1000]
247
+ aliases:
248
+ en: [total cholesterol, cholesterol]
249
+ fr: [cholestérol total, cholesterol total, cholestérol]
250
+ ar: [الكوليسترول الكلي, الكوليسترول]
251
+ kb:
252
+ what: All the cholesterol carried in your blood.
253
+ low: Rarely a concern.
254
+ high: Raises long-term heart and blood-vessel risk, especially together with high LDL; diet, activity, genetics and medicines all play a role.
255
+
256
+ ldl:
257
+ display: {en: LDL cholesterol, fr: LDL-cholestérol, ar: الكوليسترول الضار, darija: "الكوليسترول الخايب (LDL)"}
258
+ category: lipids
259
+ canonical_unit: mg/dL
260
+ units: {"mg/dl": 1, "g/l": 100, "mmol/l": 38.67}
261
+ typical_range: [0.0, 130.0]
262
+ critical: {}
263
+ plausible: [10, 600]
264
+ aliases:
265
+ en: [ldl cholesterol, ldl, ldl-c]
266
+ fr: [ldl-cholestérol, ldl cholestérol, ldl-cholesterol, cholestérol ldl, ldl]
267
+ ar: [الكوليسترول الضار, كوليسترول ldl]
268
+ kb:
269
+ what: Often called "bad" cholesterol because it can build up in artery walls.
270
+ low: Generally fine.
271
+ high: Raises heart and stroke risk over time; the right target depends on your overall risk, which a doctor assesses.
272
+
273
+ hdl:
274
+ display: {en: HDL cholesterol, fr: HDL-cholestérol, ar: الكوليسترول النافع, darija: "الكوليسترول الباهي (HDL)"}
275
+ category: lipids
276
+ canonical_unit: mg/dL
277
+ units: {"mg/dl": 1, "g/l": 100, "mmol/l": 38.67}
278
+ typical_range: [40.0, 100.0]
279
+ critical: {}
280
+ plausible: [5, 150]
281
+ aliases:
282
+ en: [hdl cholesterol, hdl, hdl-c]
283
+ fr: [hdl-cholestérol, hdl cholestérol, hdl-cholesterol, cholestérol hdl, hdl]
284
+ ar: [الكوليسترول النافع, كوليسترول hdl]
285
+ kb:
286
+ what: Often called "good" cholesterol because it helps carry cholesterol away from arteries.
287
+ low: Associated with higher heart risk; exercise and not smoking tend to raise it.
288
+ high: Generally considered favorable.
289
+
290
+ triglycerides:
291
+ display: {en: Triglycerides, fr: Triglycérides, ar: الدهون الثلاثية, darija: "التريغليسيريد"}
292
+ category: lipids
293
+ canonical_unit: mg/dL
294
+ units: {"mg/dl": 1, "g/l": 100, "mmol/l": 88.57}
295
+ typical_range: [0.0, 150.0]
296
+ critical: {high: 1000.0}
297
+ plausible: [10, 10000]
298
+ aliases:
299
+ en: [triglycerides, tg]
300
+ fr: [triglycérides, triglycerides]
301
+ ar: [الدهون الثلاثية, الشحوم الثلاثية]
302
+ kb:
303
+ what: A type of fat in the blood that stores energy from food.
304
+ low: Rarely a concern.
305
+ high: Can be linked to diet, alcohol, weight, diabetes or genetics; very high levels raise the risk of pancreatitis.
306
+
307
+ alt:
308
+ display: {en: ALT, fr: ALAT (TGP), ar: إنزيم ALT, darija: "ALAT (الكبدة)"}
309
+ category: liver
310
+ canonical_unit: U/L
311
+ units: {"u/l": 1, "ui/l": 1, "iu/l": 1}
312
+ typical_range: [7.0, 55.0]
313
+ critical: {}
314
+ plausible: [1, 5000]
315
+ aliases:
316
+ en: [alt, alanine aminotransferase, sgpt]
317
+ fr: [alat, tgp, alanine aminotransférase, alat (tgp)]
318
+ ar: [إنزيم alt, ناقلة أمين الألانين]
319
+ kb:
320
+ what: A liver enzyme; it leaks into the blood when liver cells are irritated or damaged.
321
+ low: Not a concern.
322
+ high: Can be linked to fatty liver, alcohol, some medicines, hepatitis or recent intense exercise.
323
+
324
+ ast:
325
+ display: {en: AST, fr: ASAT (TGO), ar: إنزيم AST, darija: "ASAT (الكبدة)"}
326
+ category: liver
327
+ canonical_unit: U/L
328
+ units: {"u/l": 1, "ui/l": 1, "iu/l": 1}
329
+ typical_range: [8.0, 48.0]
330
+ critical: {}
331
+ plausible: [1, 5000]
332
+ aliases:
333
+ en: [ast, aspartate aminotransferase, sgot]
334
+ fr: [asat, tgo, aspartate aminotransférase, asat (tgo)]
335
+ ar: [إنزيم ast, ناقلة أمين الأسبارتات]
336
+ kb:
337
+ what: An enzyme found in the liver, heart and muscles.
338
+ low: Not a concern.
339
+ high: Can be linked to liver irritation, muscle injury or strenuous exercise; it is read together with ALT.
340
+
341
+ tsh:
342
+ display: {en: TSH, fr: TSH, ar: الهرمون المنبه للدرقية, darija: "TSH (الغدة الدرقية)"}
343
+ category: thyroid
344
+ canonical_unit: mIU/L
345
+ units: {"miu/l": 1, "µui/ml": 1, "uui/ml": 1, "µiu/ml": 1, "uiu/ml": 1, "mui/l": 1}
346
+ typical_range: [0.4, 4.0]
347
+ critical: {}
348
+ plausible: [0.001, 200]
349
+ aliases:
350
+ en: [tsh, thyroid stimulating hormone, thyrotropin]
351
+ fr: [tsh, thyréostimuline, tsh ultrasensible, tshus]
352
+ ar: [الهرمون المنبه للدرقية, هرمون tsh]
353
+ kb:
354
+ what: A hormone from the brain that tells the thyroid how hard to work.
355
+ low: Can point to an overactive thyroid or too high a dose of thyroid medicine.
356
+ high: Can point to an underactive thyroid.
357
+
358
+ ferritin:
359
+ display: {en: Ferritin, fr: Ferritine, ar: الفيريتين, darija: "الفيريتين (مخزون الحديد)"}
360
+ category: iron
361
+ canonical_unit: ng/mL
362
+ units: {"ng/ml": 1, "µg/l": 1, "ug/l": 1}
363
+ typical_range: [15.0, 200.0]
364
+ critical: {}
365
+ plausible: [1, 10000]
366
+ aliases:
367
+ en: [ferritin, serum ferritin]
368
+ fr: [ferritine, ferritinémie]
369
+ ar: [الفيريتين, مخزون الحديد]
370
+ kb:
371
+ what: Reflects how much iron your body has in storage.
372
+ low: The most common sign of iron deficiency, even before anemia appears.
373
+ high: Can be raised by inflammation, infection, liver conditions or iron overload.
374
+
375
+ crp:
376
+ display: {en: C-reactive protein, fr: CRP, ar: البروتين المتفاعل C, darija: "CRP (الالتهاب)"}
377
+ category: inflammation
378
+ canonical_unit: mg/L
379
+ units: {"mg/l": 1, "mg/dl": 10}
380
+ typical_range: [0.0, 5.0]
381
+ critical: {}
382
+ plausible: [0, 600]
383
+ aliases:
384
+ en: [c-reactive protein, crp, c reactive protein]
385
+ fr: [crp, protéine c-réactive, protéine c réactive]
386
+ ar: [البروتين المتفاعل c, البروتين التفاعلي c]
387
+ kb:
388
+ what: A protein the liver makes when there is inflammation in the body.
389
+ low: Normal.
390
+ high: Signals inflammation or infection somewhere in the body; it does not say where or why.
391
+
392
+ vitamin_d:
393
+ display: {en: Vitamin D, fr: Vitamine D, ar: فيتامين د, darija: "فيتامين D"}
394
+ category: vitamins
395
+ canonical_unit: ng/mL
396
+ units: {"ng/ml": 1, "nmol/l": 0.4006, "µg/l": 1, "ug/l": 1}
397
+ typical_range: [30.0, 100.0]
398
+ critical: {}
399
+ plausible: [2, 250]
400
+ aliases:
401
+ en: [vitamin d, 25-oh vitamin d, 25(oh)d, 25-hydroxyvitamin d]
402
+ fr: [vitamine d, 25-oh vitamine d, 25(oh)d, vitamine d totale]
403
+ ar: [فيتامين د, فيتامين d]
404
+ kb:
405
+ what: A vitamin your skin makes in sunlight; it helps your body use calcium for bones.
406
+ low: Very common, including in sunny countries; can affect bones and muscles.
407
+ high: Usually from taking too many supplements.
lablens/evaluate.py ADDED
@@ -0,0 +1,122 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ """Benchmark the extraction + flagging pipeline against synthetic ground truth.
2
+
3
+ Metrics (per input type: PDF text layer vs degraded photo, and per locale):
4
+ recall share of true test rows that were found
5
+ precision share of extracted rows that are real rows of the report
6
+ value_acc exact value match among found rows
7
+ unit_acc unit understood as the same unit (same conversion factor)
8
+ range_acc reference range read exactly
9
+ flag_acc end-to-end flag equals the ground-truth flag (missing rows count as wrong)
10
+ critical_recall critical rows that came out as critical (the safety metric)
11
+ dangerous_misses abnormal rows reported as NORMAL (worse than missing: false reassurance)
12
+ """
13
+ from __future__ import annotations
14
+
15
+ import argparse
16
+ import json
17
+ from collections import defaultdict
18
+ from pathlib import Path
19
+
20
+ from .catalog import load_catalog
21
+ from .extract import extract
22
+ from .flagger import flag_all
23
+
24
+
25
+ def _same_unit(key: str, a: str | None, b: str | None) -> bool:
26
+ td = load_catalog()[key]
27
+ fa, fb = td.to_canonical(1.0, a), td.to_canonical(1.0, b)
28
+ return fa is not None and fb is not None and abs(fa - fb) < 1e-9
29
+
30
+
31
+ def score_report(truth: dict, path: Path) -> dict:
32
+ results, _, _, _ = extract(path)
33
+ analysis = flag_all(results)
34
+ got = {r.test_key: r for r in analysis.results}
35
+ true_keys = {row["test_key"] for row in truth["rows"]}
36
+
37
+ c = defaultdict(int)
38
+ c["rows"] = len(truth["rows"])
39
+ c["extracted"] = len(got)
40
+ c["true_positive_keys"] = len(true_keys & set(got))
41
+ for row in truth["rows"]:
42
+ k, exp = row["test_key"], row["expected_flag"]
43
+ crit = exp.startswith("critical")
44
+ c["critical_rows"] += crit
45
+ c["abnormal_rows"] += exp != "normal"
46
+ g = got.get(k)
47
+ if g is None:
48
+ continue
49
+ c["found"] += 1
50
+ c["value_ok"] += abs(g.value - row["value"]) < 1e-9
51
+ c["unit_ok"] += _same_unit(k, g.unit, row["unit"])
52
+ c["range_ok"] += (g.ref_low, g.ref_high) == (row["ref_low"], row["ref_high"]) or (
53
+ g.ref_source == "catalog" and row["ref_low"] is None and row["ref_high"] is None)
54
+ c["flag_ok"] += g.flag.value == exp
55
+ c["critical_caught"] += crit and g.flag.value.startswith("critical")
56
+ c["dangerous_miss"] += exp != "normal" and g.flag.value == "normal"
57
+ return dict(c)
58
+
59
+
60
+ def summarize(counts: list[dict]) -> dict:
61
+ t = defaultdict(int)
62
+ for c in counts:
63
+ for k, v in c.items():
64
+ t[k] += v
65
+
66
+ def ratio(a, b):
67
+ return round(t[a] / t[b], 4) if t[b] else None
68
+
69
+ return {
70
+ "reports": len(counts), "rows": t["rows"],
71
+ "recall": ratio("found", "rows"),
72
+ "precision": ratio("true_positive_keys", "extracted"),
73
+ "value_acc": ratio("value_ok", "found"),
74
+ "unit_acc": ratio("unit_ok", "found"),
75
+ "range_acc": ratio("range_ok", "found"),
76
+ "flag_acc": ratio("flag_ok", "rows"),
77
+ "critical_recall": ratio("critical_caught", "critical_rows"),
78
+ "critical_rows": t["critical_rows"],
79
+ "dangerous_misses": t["dangerous_miss"],
80
+ "abnormal_rows": t["abnormal_rows"],
81
+ }
82
+
83
+
84
+ def run(bench_dir: Path) -> dict:
85
+ groups: dict[str, list[dict]] = defaultdict(list)
86
+ for jf in sorted(bench_dir.glob("rpt_*.json")):
87
+ truth = json.loads(jf.read_text(encoding="utf-8"))
88
+ for kind, field in (("pdf", "pdf"), ("photo", "image")):
89
+ if field not in truth:
90
+ continue
91
+ c = score_report(truth, bench_dir / truth[field])
92
+ groups[f"{kind}/all"].append(c)
93
+ groups[f"{kind}/{truth['locale']}"].append(c)
94
+ return {g: summarize(v) for g, v in sorted(groups.items())}
95
+
96
+
97
+ def to_markdown(summary: dict) -> str:
98
+ cols = ["reports", "rows", "recall", "precision", "value_acc", "unit_acc", "range_acc",
99
+ "flag_acc", "critical_recall", "dangerous_misses"]
100
+ out = ["| split | " + " | ".join(cols) + " |", "|" + "---|" * (len(cols) + 1)]
101
+ for g, s in summary.items():
102
+ cells = []
103
+ for k in cols:
104
+ v = s[k]
105
+ if isinstance(v, float) and k not in ("reports", "rows", "dangerous_misses"):
106
+ cells.append(f"{v:.1%}")
107
+ elif k == "critical_recall" and v is None:
108
+ cells.append("n/a")
109
+ else:
110
+ cells.append(str(v))
111
+ out.append(f"| {g} | " + " | ".join(cells) + " |")
112
+ return "\n".join(out)
113
+
114
+
115
+ if __name__ == "__main__":
116
+ ap = argparse.ArgumentParser()
117
+ ap.add_argument("--bench", default="benchmark/reports")
118
+ ap.add_argument("--out", default="benchmark/results.json")
119
+ a = ap.parse_args()
120
+ summary = run(Path(a.bench))
121
+ Path(a.out).write_text(json.dumps(summary, indent=2), encoding="utf-8")
122
+ print(to_markdown(summary))
lablens/explain.py ADDED
@@ -0,0 +1,186 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ """Plain-language explanations: hosted open LLM, grounded and guarded, with a template fallback.
2
+
3
+ Privacy by construction: the model only ever receives test names, values, units, ranges and
4
+ the deterministic flags. The report image, patient name, lab, dates and any other text on
5
+ the page never leave the extraction step.
6
+
7
+ Safety by construction: the model does not decide what is abnormal or urgent. Flags and the
8
+ urgent banner are computed by `flagger.py` and rendered from fixed strings in `i18n.py`.
9
+ The model's output is checked for diagnosis/treatment claims and replaced by the template
10
+ if it fails.
11
+ """
12
+ from __future__ import annotations
13
+
14
+ import json
15
+ import os
16
+ import re
17
+
18
+ from .catalog import load_catalog
19
+ from .i18n import FLAG_LABEL, HEADINGS, LANGS, TEMPLATE_QUESTION, t
20
+ from .models import Analysis, Flag
21
+
22
+ DEFAULT_BASE_URL = "https://router.huggingface.co/v1"
23
+ DEFAULT_MODEL = "Qwen/Qwen2.5-72B-Instruct"
24
+
25
+ LANG_INSTRUCTIONS = {
26
+ "en": "Write in simple English, at the level of a 12-year-old reader.",
27
+ "fr": "Écris en français simple, compréhensible par quelqu'un sans formation médicale.",
28
+ "ar": "اكتب بالعربية الفصحى المبسّطة التي يفهمها أي شخص.",
29
+ "darija": ("Write in Tunisian Arabic (Darija/Tounsi) in Arabic script, the way people speak in Tunis or "
30
+ "Sousse. Keep common French medical words as Tunisians say them (analyse, cholestérol). "
31
+ "Do not write Modern Standard Arabic."),
32
+ }
33
+
34
+ SYSTEM_PROMPT = """You are LabLens, a careful assistant that explains blood test results to patients in plain language.
35
+
36
+ Hard rules:
37
+ 1. You do NOT diagnose. Never say the person "has" a disease. Use "can be linked to", "is sometimes seen with".
38
+ 2. You do NOT recommend medicines, doses, supplements or stopping any treatment.
39
+ 3. The status of each result (within range, low, high, very low/high, check) is FINAL and given to you. Never contradict it or re-judge a value yourself.
40
+ 4. Only talk about the tests listed in the input. Base explanations on the provided knowledge notes; do not add other medical claims.
41
+ 5. Be calm and kind. Abnormal results are common and often minor, but do not dismiss them.
42
+ 6. Keep it short: one or two sentences per result.
43
+
44
+ Return ONLY valid JSON with this shape:
45
+ {"summary": "2-3 sentences", "results": [{"key": "<test key>", "explanation": "..."}], "questions": ["3 to 5 short questions to ask a doctor"]}
46
+ """
47
+
48
+ # Patterns that indicate diagnosis or treatment advice; any hit triggers the template fallback.
49
+ BANNED = [
50
+ r"\byou have (?:a |an )?(?:diabetes|anemia|anaemia|cancer|leukemia|kidney disease|hypothyroidism|hyperthyroidism|infection)\b",
51
+ r"\byou are (?:diabetic|anemic|anaemic)\b",
52
+ r"\bvous (?:avez|êtes) (?:le |un |une |du )?(?:diabète|diabétique|anémie|anémique|cancer|leucémie|infection)\b",
53
+ r"\b(?:take|prenez|prends)\s+\d+\s*(?:mg|g|µg|ui|iu)\b",
54
+ r"\b(?:stop|arrêtez|arrête)\s+(?:taking|your|votre|ton|de prendre)\b",
55
+ r"\bdiagnos(?:is|ed)\s+(?:of|with)\b",
56
+ ]
57
+
58
+
59
+ def _payload(analysis: Analysis, lang: str) -> list[dict]:
60
+ catalog = load_catalog()
61
+ items = []
62
+ for r in analysis.results:
63
+ td = catalog.get(r.test_key or "")
64
+ if not td:
65
+ continue
66
+ items.append({
67
+ "key": td.key,
68
+ "name": td.display.get(lang) or td.display["en"],
69
+ "value": r.value, "unit": r.unit,
70
+ "reference_range": [r.ref_low, r.ref_high],
71
+ "range_is_generic": r.ref_source == "catalog",
72
+ "status": r.flag.value,
73
+ "knowledge": {"what": td.kb["what"],
74
+ "if_low": td.kb["low"] if r.flag in (Flag.LOW, Flag.CRITICAL_LOW) else None,
75
+ "if_high": td.kb["high"] if r.flag in (Flag.HIGH, Flag.CRITICAL_HIGH) else None},
76
+ })
77
+ return items
78
+
79
+
80
+ def violates_guardrails(text: str) -> str | None:
81
+ low = text.lower()
82
+ for pat in BANNED:
83
+ if re.search(pat, low):
84
+ return pat
85
+ return None
86
+
87
+
88
+ def llm_available() -> bool:
89
+ return bool(os.getenv("LLM_API_KEY") or os.getenv("HF_TOKEN"))
90
+
91
+
92
+ def explain_with_llm(analysis: Analysis, lang: str, timeout: float = 45.0) -> dict:
93
+ from openai import OpenAI
94
+
95
+ client = OpenAI(
96
+ base_url=os.getenv("LLM_BASE_URL", DEFAULT_BASE_URL),
97
+ api_key=os.getenv("LLM_API_KEY") or os.getenv("HF_TOKEN"),
98
+ timeout=timeout,
99
+ )
100
+ items = _payload(analysis, lang)
101
+ user = (f"Language: {LANGS.get(lang, 'English')}. {LANG_INSTRUCTIONS.get(lang, '')}\n\n"
102
+ f"Results (JSON):\n{json.dumps(items, ensure_ascii=False)}")
103
+ resp = client.chat.completions.create(
104
+ model=os.getenv("LLM_MODEL", DEFAULT_MODEL),
105
+ messages=[{"role": "system", "content": SYSTEM_PROMPT}, {"role": "user", "content": user}],
106
+ temperature=0.2,
107
+ max_tokens=1400,
108
+ )
109
+ text = resp.choices[0].message.content or ""
110
+ m = re.search(r"\{.*\}", text, re.S)
111
+ if not m:
112
+ raise ValueError("model did not return JSON")
113
+ data = json.loads(m.group(0))
114
+ valid_keys = {i["key"] for i in items}
115
+ data["results"] = [x for x in data.get("results", []) if x.get("key") in valid_keys]
116
+ flat = " ".join([data.get("summary", "")] + [x.get("explanation", "") for x in data["results"]]
117
+ + list(data.get("questions", [])))
118
+ hit = violates_guardrails(flat)
119
+ if hit:
120
+ raise ValueError(f"guardrail: {hit}")
121
+ return data
122
+
123
+
124
+ def explain_with_template(analysis: Analysis, lang: str) -> dict:
125
+ """Deterministic fallback. Explanations come straight from the knowledge base (English)."""
126
+ catalog = load_catalog()
127
+ abnormal = [r for r in analysis.results if r.flag not in (Flag.NORMAL, Flag.UNKNOWN)]
128
+ results, questions = [], []
129
+ for r in analysis.results:
130
+ td = catalog.get(r.test_key or "")
131
+ if not td:
132
+ continue
133
+ parts = [td.kb["what"]]
134
+ if r.flag in (Flag.LOW, Flag.CRITICAL_LOW):
135
+ parts.append(td.kb["low"])
136
+ elif r.flag in (Flag.HIGH, Flag.CRITICAL_HIGH):
137
+ parts.append(td.kb["high"])
138
+ results.append({"key": td.key, "explanation": " ".join(parts)})
139
+ if r in abnormal and r.flag != Flag.CHECK and len(questions) < 5:
140
+ questions.append(t(TEMPLATE_QUESTION, lang).format(
141
+ name=td.display.get(lang) or td.display["en"],
142
+ flag_lower=t(FLAG_LABEL[r.flag.value], lang).lower(),
143
+ value=f"{r.value:g}", unit=r.unit or ""))
144
+ n_ab = len([r for r in abnormal if r.flag != Flag.CHECK])
145
+ summary = {
146
+ "en": f"{len(results)} results read, {n_ab} outside the reference range.",
147
+ "fr": f"{len(results)} résultats lus, dont {n_ab} en dehors des valeurs de référence.",
148
+ "ar": f"تمت قراءة {len(results)} نتيجة، منها {n_ab} خارج المعدل المرجعي.",
149
+ "darija": f"قريت {len(results)} نتيجة، منهم {n_ab} برّا المعدل.",
150
+ }
151
+ return {"summary": t(summary, lang), "results": results, "questions": questions}
152
+
153
+
154
+ def explain(analysis: Analysis, lang: str) -> tuple[dict, str]:
155
+ """Returns (explanation dict, source) where source is 'llm' or 'template:<reason>'."""
156
+ if not llm_available():
157
+ return explain_with_template(analysis, lang), "template:no-api-key"
158
+ try:
159
+ return explain_with_llm(analysis, lang), "llm"
160
+ except Exception as e: # network, JSON, or guardrail failure -> safe fallback
161
+ return explain_with_template(analysis, lang), f"template:{type(e).__name__}: {str(e)[:120]}"
162
+
163
+
164
+ def render_markdown(analysis: Analysis, exp: dict, lang: str) -> str:
165
+ catalog = load_catalog()
166
+ by_key = {x["key"]: x["explanation"] for x in exp.get("results", [])}
167
+ lines = [f"### {t(HEADINGS['summary'], lang)}", exp.get("summary", ""), "",
168
+ f"### {t(HEADINGS['details'], lang)}"]
169
+ order = {Flag.CRITICAL_HIGH: 0, Flag.CRITICAL_LOW: 0, Flag.CHECK: 1, Flag.HIGH: 2, Flag.LOW: 2,
170
+ Flag.UNKNOWN: 3, Flag.NORMAL: 4}
171
+ normal_names = []
172
+ for r in sorted(analysis.results, key=lambda r: order[r.flag]):
173
+ td = catalog.get(r.test_key or "")
174
+ if not td:
175
+ continue
176
+ name = td.display.get(lang) or td.display["en"]
177
+ if r.flag == Flag.NORMAL:
178
+ normal_names.append(name)
179
+ continue
180
+ label = t(FLAG_LABEL[r.flag.value], lang)
181
+ lines.append(f"- **{name}: {r.value:g} {r.unit or ''}** — _{label}_. {by_key.get(td.key, '')}")
182
+ if normal_names:
183
+ lines += ["", f"**{t(HEADINGS['normal_list'], lang)}:** " + ", ".join(normal_names)]
184
+ if exp.get("questions"):
185
+ lines += ["", f"### {t(HEADINGS['questions'], lang)}"] + [f"{i}. {q}" for i, q in enumerate(exp["questions"], 1)]
186
+ return "\n".join(lines)
lablens/extract.py ADDED
@@ -0,0 +1,237 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ """Turn a lab report (PDF or photo) into structured results.
2
+
3
+ Pipeline:
4
+ 1. Get text lines: the PDF text layer when there is one, otherwise OCR (Tesseract).
5
+ 2. For every line, find "name value [H/L] unit range".
6
+ 3. Map the printed name to a catalog test (exact alias, then fuzzy).
7
+ 4. Snap OCR-mangled units to the closest unit the catalog knows for that test.
8
+
9
+ No LLM is involved here: extraction errors should be measurable and reproducible.
10
+ """
11
+ from __future__ import annotations
12
+
13
+ import re
14
+ from pathlib import Path
15
+
16
+ from rapidfuzz import fuzz, process
17
+
18
+ from .catalog import load_catalog, match_test_name, normalize_unit
19
+ from .models import Result
20
+
21
+ NUM = r"[<>≤≥]?\s?\d+(?:[.,]\d+)?"
22
+ NUM_RE = re.compile(rf"^{NUM}$")
23
+ MARKERS = {"h", "l", "*", "hh", "ll", "↑", "↓", "+", "-"}
24
+
25
+
26
+ def _to_float(s: str) -> float:
27
+ s = s.strip().lstrip("<>≤≥").strip().replace(",", ".")
28
+ return float(s)
29
+
30
+
31
+ def _clean_ocr_line(line: str) -> str:
32
+ line = line.replace("|", " ").replace("—", "-").replace("–", "-")
33
+ # Common OCR confusions inside numbers: O->0 and l->1 when surrounded by digits.
34
+ line = re.sub(r"(?<=\d)[Oo](?=[\d,.])|(?<=[\d,.])[Oo](?=\d|\b)", "0", line)
35
+ line = re.sub(r"(?<=\d)\s+([.,])\s*(?=\d)", r"\1", line) # "0 ,70" -> "0,70"
36
+ return re.sub(r"\s+", " ", line).strip()
37
+
38
+
39
+ def _looks_like_unit(tok: str) -> bool:
40
+ if re.match(r"^[\[(<>≤≥]", tok):
41
+ return False
42
+ if tok[0].isdigit():
43
+ return "/" in tok or "^" in tok # 10^3/µL, 10^9/L
44
+ return bool(re.search(r"[A-Za-zµ%]", tok))
45
+
46
+
47
+ def parse_range(text: str) -> tuple[float | None, float | None]:
48
+ raw = text.strip()
49
+ # OCR often reads a closing "]" as "1": "[4.20 - 5.901" -> "[4.20 - 5.90"
50
+ m = re.match(r"^[\[(].*?(\d+[.,](\d+))\s*-\s*(\d+[.,](\d+))$", raw)
51
+ if m and m.group(3).endswith("1") and len(m.group(4)) > len(m.group(2)):
52
+ raw = raw[:-1]
53
+ t = raw.strip("()[]{} ").replace(",", ".")
54
+ t = t.replace("–", "-").replace("—", "-").replace(" à ", "-").replace(" to ", "-")
55
+ m = re.match(r"^(?:<|≤|<=|inf\.?\s*[aà]?)\s*(\d+(?:\.\d+)?)", t, re.I)
56
+ if m:
57
+ return None, float(m.group(1))
58
+ m = re.match(r"^(?:>|≥|>=|sup\.?\s*[aà]?)\s*(\d+(?:\.\d+)?)", t, re.I)
59
+ if m:
60
+ return float(m.group(1)), None
61
+ m = re.search(r"(\d+(?:\.\d+)?)\s*-\s*(\d+(?:\.\d+)?)", t)
62
+ if m:
63
+ lo, hi = float(m.group(1)), float(m.group(2))
64
+ if lo <= hi:
65
+ return lo, hi
66
+ return None, None
67
+
68
+
69
+ # How units are usually printed; used to display snapped units nicely.
70
+ _PRETTY = {normalize_unit(u): u for u in [
71
+ "g/dL", "g/L", "mg/dL", "mg/L", "mmol/L", "µmol/L", "mEq/L", "%", "fL", "U/L", "UI/L", "IU/L",
72
+ "mIU/L", "µUI/mL", "µIU/mL", "ng/mL", "µg/L", "nmol/L", "10^3/µL", "10^6/µL",
73
+ "10^9/L", "10^12/L", "/mm3", "L/L"]}
74
+ _PRETTY.update({"10^3/ul": "10^3/µL", "10^6/ul": "10^6/µL", "uui/ml": "µUI/mL", "umol/l": "µmol/L"})
75
+
76
+
77
+ def _ocr_fold_unit(u: str) -> str:
78
+ u = normalize_unit(u)
79
+ u = re.sub(r"^p(?=[uµ])", "µ", u) # "pUI/mL" -> "µUI/mL"
80
+ u = re.sub(r"10[%*°](?=\d)", "10^", u) # "10%3/pL" -> "10^3/pL"
81
+ u = re.sub(r"/[pu](?=l$)", "/µ", u) # "/pL", "/uL" -> "/µL"
82
+ return u.replace("w", "/").replace("|", "l").replace("1", "l")
83
+
84
+
85
+ def _snap_unit(test_key: str | None, unit: str | None) -> str | None:
86
+ """Return the unit as a known spelling for this test, repairing small OCR errors."""
87
+ if not unit or not test_key:
88
+ return unit
89
+ td = load_catalog()[test_key]
90
+ known = list(td.units) + [td.canonical_unit]
91
+ norm_known = [normalize_unit(k) for k in known]
92
+ nu = normalize_unit(unit)
93
+ if nu not in norm_known:
94
+ best = process.extractOne(_ocr_fold_unit(unit), norm_known, scorer=fuzz.ratio)
95
+ if not best or best[1] < 60:
96
+ return unit # leave it; flags will rely on the printed range only
97
+ nu = best[0]
98
+ if test_key in ("wbc", "platelets", "rbc") and nu in ("g/l", "t/l"):
99
+ return nu.upper() # giga/tera per litre cell counts, as French labs print them
100
+ return _PRETTY.get(nu, known[norm_known.index(nu)])
101
+
102
+
103
+ def parse_line(line: str) -> Result | None:
104
+ """Parse one report line. Returns None when the line has no test-like structure."""
105
+ # Split markers glued to values: "5,36H" -> "5,36 H"
106
+ line = re.sub(r"(?<=\d)([HhLl*])(?=[\s.,;:_]|$)", r" \1", line)
107
+ line = line.replace("«", "<").replace("»", ">")
108
+ tokens = [t.strip("_.,;:'\"`~") or t for t in line.split(" ")]
109
+ tokens = [t for t in tokens if t and not re.fullmatch(r"[_.,;:'\"`~=]+", t)]
110
+ for i in range(1, len(tokens)):
111
+ tok = tokens[i]
112
+ if not NUM_RE.match(tok) or tok[0] in "<>≤≥":
113
+ continue
114
+ key, score, name = None, 0.0, ""
115
+ # The name is a prefix of the tokens before the value. OCR noise ("ae", "oo", "L")
116
+ # can sit between name and value, so also try dropping short trailing tokens.
117
+ for k in range(i, 0, -1):
118
+ if k < i and len(tokens[k]) > 3:
119
+ break
120
+ cand = " ".join(tokens[:k]).strip(" :.")
121
+ if not re.search(r"[A-Za-zÀ-ÿ؀-ۿ]", cand):
122
+ continue
123
+ ck, cs = match_test_name(cand)
124
+ if ck and cs > score:
125
+ key, score, name = ck, cs, cand
126
+ if cs == 100.0:
127
+ break
128
+ if key is None:
129
+ continue
130
+ # OCR sometimes emits a stray short number before the real value:
131
+ # "K 7 3.79 mmol/L" -> the value is the number directly followed by a unit.
132
+ if (i + 2 < len(tokens) and NUM_RE.match(tokens[i + 1]) and "." not in tok and "," not in tok
133
+ and len(tok) <= 2 and _looks_like_unit(tokens[i + 2])):
134
+ i += 1
135
+ tok = tokens[i]
136
+ j = i + 1
137
+ marker = None
138
+ if j < len(tokens) and tokens[j].lower() in MARKERS:
139
+ m = tokens[j].lower()
140
+ marker = "H" if m in ("h", "hh", "↑") else "L" if m in ("l", "ll", "↓") else "*" if m == "*" else None
141
+ j += 1
142
+ unit = None
143
+ if j < len(tokens) and _looks_like_unit(tokens[j]):
144
+ unit = tokens[j]
145
+ j += 1
146
+ ref_low, ref_high = parse_range(" ".join(tokens[j:]))
147
+ return Result(
148
+ test_key=key, raw_name=name, value=_to_float(tok), unit=_snap_unit(key, unit),
149
+ ref_low=ref_low, ref_high=ref_high,
150
+ ref_source="report" if (ref_low is not None or ref_high is not None) else "none",
151
+ match_score=score, source_line=line, report_marker=marker,
152
+ )
153
+ return None
154
+
155
+
156
+ def lines_from_pdf(path: Path) -> list[str]:
157
+ import pdfplumber
158
+
159
+ lines: list[str] = []
160
+ with pdfplumber.open(str(path)) as pdf:
161
+ for page in pdf.pages:
162
+ lines.extend((page.extract_text() or "").splitlines())
163
+ return lines
164
+
165
+
166
+ def preprocess_image(path: Path):
167
+ """Grayscale, upscale, adaptive-threshold and strip table rules (incl. dotted lines).
168
+
169
+ Dotted row separators are the main thing that breaks Tesseract on lab reports:
170
+ without this step OCR on dotted layouts drops almost every row.
171
+ """
172
+ import cv2
173
+ import numpy as np
174
+ from PIL import Image, ImageOps
175
+
176
+ pil = ImageOps.exif_transpose(Image.open(path)).convert("L")
177
+ img = np.array(pil)
178
+ if img.shape[1] < 2400:
179
+ f = 2400 / img.shape[1]
180
+ img = cv2.resize(img, None, fx=f, fy=f, interpolation=cv2.INTER_CUBIC)
181
+ img = cv2.GaussianBlur(img, (3, 3), 0)
182
+ bw = cv2.adaptiveThreshold(img, 255, cv2.ADAPTIVE_THRESH_GAUSSIAN_C, cv2.THRESH_BINARY, 31, 15)
183
+ inv = 255 - bw
184
+ closed = cv2.morphologyEx(inv, cv2.MORPH_CLOSE, cv2.getStructuringElement(cv2.MORPH_RECT, (9, 1)))
185
+ rules = cv2.morphologyEx(closed, cv2.MORPH_OPEN, cv2.getStructuringElement(cv2.MORPH_RECT, (150, 1)))
186
+ rules = cv2.dilate(rules, cv2.getStructuringElement(cv2.MORPH_RECT, (1, 3)))
187
+ return 255 - cv2.bitwise_and(inv, 255 - rules)
188
+
189
+
190
+ def lines_from_image(path: Path, langs: str = "eng+fra") -> list[str]:
191
+ import pytesseract
192
+
193
+ text = pytesseract.image_to_string(preprocess_image(path), lang=langs, config="--psm 6")
194
+ return text.splitlines()
195
+
196
+
197
+ def detect_language(lines: list[str]) -> str:
198
+ text = " ".join(lines).lower()
199
+ fr = sum(w in text for w in ["analyse", "résultat", "resultat", "prélèvement", "référence", "glycémie", "valeurs"])
200
+ en = sum(w in text for w in ["result", "reference", "collection", "patient:", "test", "range"])
201
+ return "fr" if fr > en else "en"
202
+
203
+
204
+ def extract(path: str | Path) -> tuple[list[Result], list[str], str, str]:
205
+ """Returns (results, unmatched numeric lines, detected language, method)."""
206
+ path = Path(path)
207
+ method = "pdf-text"
208
+ if path.suffix.lower() == ".pdf":
209
+ lines = lines_from_pdf(path)
210
+ if sum(len(line) for line in lines) < 40: # scanned PDF without a text layer
211
+ import subprocess
212
+ import tempfile
213
+
214
+ with tempfile.TemporaryDirectory() as td:
215
+ subprocess.run(["pdftoppm", "-png", "-r", "300", str(path), f"{td}/p"], check=True)
216
+ lines = []
217
+ for png in sorted(Path(td).glob("p*.png")):
218
+ lines.extend(lines_from_image(png))
219
+ method = "ocr"
220
+ else:
221
+ lines = lines_from_image(path)
222
+ method = "ocr"
223
+
224
+ results: list[Result] = []
225
+ unmatched: list[str] = []
226
+ seen: set[str] = set()
227
+ for raw in lines:
228
+ line = _clean_ocr_line(raw)
229
+ if not line:
230
+ continue
231
+ r = parse_line(line)
232
+ if r and r.test_key not in seen:
233
+ seen.add(r.test_key)
234
+ results.append(r)
235
+ elif r is None and re.search(r"\d+[.,]\d+", line) and not re.search(r"\d{4}-\d{2}-\d{2}", line):
236
+ unmatched.append(line)
237
+ return results, unmatched, detect_language(lines), method
lablens/flagger.py ADDED
@@ -0,0 +1,84 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ """Deterministic flagging. No LLM is involved in deciding what is abnormal or urgent.
2
+
3
+ Rules, in order:
4
+ 0. Values outside physiologically possible bounds are marked CHECK (likely misread).
5
+ 1. Critical thresholds from the catalog (converted to canonical units) always win.
6
+ They escalate to an "urgent" banner.
7
+ 2. Otherwise the report's own printed reference range decides low / high / normal.
8
+ 3. If the report printed no range, the catalog's typical adult range is used and the
9
+ result is marked as such, so the UI can say the range is generic.
10
+ 4. If the unit can't be understood, critical checks are skipped and the flag uses
11
+ only the printed range; if there is none, the flag is UNKNOWN.
12
+ 5. If the lab printed an H/L marker that disagrees with the computed flag, the result
13
+ becomes CHECK: a misread value must never come out as reassuring.
14
+ """
15
+ from __future__ import annotations
16
+
17
+ from .catalog import load_catalog
18
+ from .models import Analysis, Flag, FlaggedResult, Result
19
+
20
+
21
+ def flag_result(r: Result) -> FlaggedResult:
22
+ catalog = load_catalog()
23
+ fr = FlaggedResult(**r.model_dump())
24
+ td = catalog.get(r.test_key) if r.test_key else None
25
+
26
+ canonical = td.to_canonical(r.value, r.unit) if td else None
27
+ fr.canonical_value = round(canonical, 4) if canonical is not None else None
28
+
29
+ # 0. Implausible values are almost always reading errors (dropped decimal, stray digit).
30
+ # Never call them normal and never escalate on them: ask the user to check.
31
+ if td and canonical is not None and not (td.plausible[0] <= canonical <= td.plausible[1]):
32
+ fr.flag = Flag.CHECK
33
+ fr.reason = "value outside what is physically possible; please check it against your report"
34
+ return fr
35
+
36
+ # 1. Critical values (only when we are sure of the unit).
37
+ if td and canonical is not None:
38
+ crit_low = td.critical.get("low")
39
+ crit_high = td.critical.get("high")
40
+ if crit_low is not None and canonical < crit_low:
41
+ fr.flag = Flag.CRITICAL_LOW
42
+ fr.reason = f"below critical threshold {crit_low} {td.canonical_unit}"
43
+ return fr
44
+ if crit_high is not None and canonical > crit_high:
45
+ fr.flag = Flag.CRITICAL_HIGH
46
+ fr.reason = f"above critical threshold {crit_high} {td.canonical_unit}"
47
+ return fr
48
+
49
+ # 2./3. Reference range: report first, catalog fallback.
50
+ low, high, value = r.ref_low, r.ref_high, r.value
51
+ if low is None and high is None:
52
+ if td and canonical is not None:
53
+ low, high = td.typical_range
54
+ value = canonical
55
+ fr.ref_source = "catalog"
56
+ fr.ref_low, fr.ref_high = low, high
57
+ else:
58
+ fr.flag = Flag.UNKNOWN
59
+ fr.ref_source = "none"
60
+ fr.reason = "no reference range on the report and unit not recognized"
61
+ return fr
62
+
63
+ if low is not None and value < low:
64
+ fr.flag, fr.reason = Flag.LOW, f"{value:g} is below {low:g}"
65
+ elif high is not None and value > high:
66
+ fr.flag, fr.reason = Flag.HIGH, f"{value:g} is above {high:g}"
67
+ else:
68
+ fr.flag, fr.reason = Flag.NORMAL, "within range"
69
+
70
+ # 5. Cross-check with the lab's own H/L marker. If the lab marked the value abnormal
71
+ # but the numbers we read say otherwise, something was misread: never reassure.
72
+ m = r.report_marker
73
+ if m and ((fr.flag == Flag.NORMAL) or (m == "H" and fr.flag == Flag.LOW) or (m == "L" and fr.flag == Flag.HIGH)):
74
+ fr.flag = Flag.CHECK
75
+ fr.reason = f"the lab marked this value '{m}' but the numbers read disagree; please check"
76
+ return fr
77
+
78
+
79
+ def flag_all(results: list[Result], language: str = "en", unmatched: list[str] | None = None,
80
+ method: str = "") -> Analysis:
81
+ flagged = [flag_result(r) for r in results]
82
+ urgent = any(f.flag in (Flag.CRITICAL_LOW, Flag.CRITICAL_HIGH) for f in flagged)
83
+ return Analysis(language=language, results=flagged, unmatched_lines=unmatched or [],
84
+ urgent=urgent, extraction_method=method)
lablens/i18n.py ADDED
@@ -0,0 +1,77 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ """UI strings and deterministic text in the four supported languages.
2
+
3
+ Everything safety-relevant (flags, the urgent banner, the disclaimer) comes from here,
4
+ never from the LLM, so it is identical on every run and can be reviewed once.
5
+ """
6
+ from __future__ import annotations
7
+
8
+ LANGS = {"en": "English", "fr": "Français", "ar": "العربية", "darija": "Tounsi (Darija)"}
9
+ RTL = {"ar", "darija"}
10
+
11
+ FLAG_LABEL = {
12
+ "normal": {"en": "Within range", "fr": "Dans les normes", "ar": "ضمن المعدل", "darija": "ف النورمال"},
13
+ "low": {"en": "Low", "fr": "Bas", "ar": "منخفض", "darija": "ناقص"},
14
+ "high": {"en": "High", "fr": "Élevé", "ar": "مرتفع", "darija": "طالع"},
15
+ "critical_low": {"en": "Very low – urgent", "fr": "Très bas – urgent", "ar": "منخفض جدًا – عاجل",
16
+ "darija": "ناقص برشا – عاجل"},
17
+ "critical_high": {"en": "Very high – urgent", "fr": "Très élevé – urgent", "ar": "مرتفع جدًا – عاجل",
18
+ "darija": "طالع برشا – عاجل"},
19
+ "unknown": {"en": "Can't judge", "fr": "Non évaluable", "ar": "لا يمكن التقييم", "darija": "ما نجموش نقيّموه"},
20
+ "check": {"en": "Please check value", "fr": "Vérifiez la valeur", "ar": "تحقق من القيمة",
21
+ "darija": "ثبّت فالرقم"},
22
+ }
23
+
24
+ URGENT = {
25
+ "en": "**One or more results are in a range that can be dangerous.** Please contact a doctor today, "
26
+ "or go to an emergency service if you feel unwell. Lab values can be wrong, so a doctor needs to "
27
+ "confirm them, but don't wait.",
28
+ "fr": "**Un ou plusieurs résultats sont dans une zone qui peut être dangereuse.** Contactez un médecin "
29
+ "aujourd'hui, ou rendez-vous aux urgences si vous ne vous sentez pas bien. Un résultat peut être "
30
+ "erroné et doit être confirmé par un médecin, mais n'attendez pas.",
31
+ "ar": "**نتيجة أو أكثر في نطاق قد يكون خطيرًا.** يرجى التواصل مع طبيب اليوم، أو التوجه إلى الطوارئ إذا "
32
+ "شعرت بتوعك. قد تكون النتائج خاطئة ويجب أن يؤكدها الطبيب، لكن لا تنتظر.",
33
+ "darija": "**فما نتيجة ولا أكثر في مستوى ينجم يكون خطير.** كلّم طبيب اليوم، ولا امشي للاستعجالي كان "
34
+ "ماكش لاباس. النتيجة تنجم تكون غالطة والطبيب لازم يثبّتها، أما ما تستناش.",
35
+ }
36
+
37
+ DISCLAIMER = {
38
+ "en": "LabLens explains lab results in plain language. It does not diagnose and does not replace a doctor.",
39
+ "fr": "LabLens explique les résultats d'analyses en langage simple. Il ne pose pas de diagnostic et ne "
40
+ "remplace pas un médecin.",
41
+ "ar": "يشرح LabLens نتائج التحاليل بلغة بسيطة. لا يقدّم تشخيصًا ولا يغني عن الطبيب.",
42
+ "darija": "LabLens يفسّرلك نتايج التحاليل بكلام ساهل. ما يعطيكش تشخيص وما يعوّضش الطبيب.",
43
+ }
44
+
45
+ CHECK_NOTE = {
46
+ "en": "Some values look misread. Compare them with your paper report and fix them in the table, then explain again.",
47
+ "fr": "Certaines valeurs semblent mal lues. Comparez-les avec votre compte rendu, corrigez-les dans le tableau, puis relancez.",
48
+ "ar": "يبدو أن بعض القيم قُرئت بشكل خاطئ. قارنها مع التقرير الورقي وصحّحها في الجدول ثم أعد الشرح.",
49
+ "darija": "فما أرقام يظهر تقراو غالط. قارنهم مع الورقة متاعك، صلّحهم فالجدول، وعاود.",
50
+ }
51
+
52
+ GENERIC_RANGE = {
53
+ "en": "typical adult range (your report printed none)",
54
+ "fr": "valeurs usuelles adulte (aucune sur votre compte rendu)",
55
+ "ar": "المعدل المعتاد للبالغين (لا يوجد في تقريرك)",
56
+ "darija": "المعدل العادي للكبار (ما فماش فالتقرير)",
57
+ }
58
+
59
+ HEADINGS = {
60
+ "summary": {"en": "Summary", "fr": "En résumé", "ar": "الخلاصة", "darija": "باختصار"},
61
+ "details": {"en": "Your results explained", "fr": "Vos résultats expliqués", "ar": "شرح نتائجك",
62
+ "darija": "نتايجك مفسّرة"},
63
+ "questions": {"en": "Questions to ask your doctor", "fr": "Questions à poser à votre médecin",
64
+ "ar": "أسئلة لطبيبك", "darija": "أسئلة تسألهم لطبيبك"},
65
+ "normal_list": {"en": "Within range", "fr": "Dans les normes", "ar": "ضمن المعدل", "darija": "ف النورمال"},
66
+ }
67
+
68
+ TEMPLATE_QUESTION = {
69
+ "en": "My {name} is {flag_lower} ({value} {unit}). What could explain it, and does it need follow-up?",
70
+ "fr": "Mon résultat de {name} est {flag_lower} ({value} {unit}). Qu'est-ce qui peut l'expliquer, et faut-il un suivi ?",
71
+ "ar": "نتيجة {name} لدي {flag_lower} ({value} {unit}). ما الذي قد يفسر ذلك، وهل تحتاج إلى متابعة؟",
72
+ "darija": "{name} متاعي {flag_lower} ({value} {unit}). شنوّة نجم يكون السبب، ولازم متابعة؟",
73
+ }
74
+
75
+
76
+ def t(table: dict, lang: str) -> str:
77
+ return table.get(lang) or table["en"]
lablens/models.py ADDED
@@ -0,0 +1,44 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ from __future__ import annotations
2
+
3
+ from enum import Enum
4
+
5
+ from pydantic import BaseModel, Field
6
+
7
+
8
+ class Flag(str, Enum):
9
+ NORMAL = "normal"
10
+ LOW = "low"
11
+ HIGH = "high"
12
+ CRITICAL_LOW = "critical_low"
13
+ CRITICAL_HIGH = "critical_high"
14
+ UNKNOWN = "unknown" # could not be judged (no range, unknown unit)
15
+ CHECK = "check" # value is physically implausible: probably misread, ask the user
16
+
17
+
18
+ class Result(BaseModel):
19
+ """One test line read from a report."""
20
+
21
+ test_key: str | None = None
22
+ raw_name: str
23
+ value: float
24
+ unit: str | None = None
25
+ ref_low: float | None = None
26
+ ref_high: float | None = None
27
+ ref_source: str = "report" # "report" | "catalog" | "none"
28
+ match_score: float = 0.0
29
+ source_line: str = ""
30
+ report_marker: str | None = None # "H" / "L" / "*" printed next to the value by the lab
31
+
32
+
33
+ class FlaggedResult(Result):
34
+ flag: Flag = Flag.UNKNOWN
35
+ canonical_value: float | None = None
36
+ reason: str = ""
37
+
38
+
39
+ class Analysis(BaseModel):
40
+ language: str = "en"
41
+ results: list[FlaggedResult] = Field(default_factory=list)
42
+ unmatched_lines: list[str] = Field(default_factory=list)
43
+ urgent: bool = False
44
+ extraction_method: str = ""
lablens/synth/__init__.py ADDED
File without changes
lablens/synth/generator.py ADDED
@@ -0,0 +1,320 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ """Synthetic lab-report generator.
2
+
3
+ Produces realistic-looking lab reports (PDF, plus degraded "phone photo" PNGs) together
4
+ with ground-truth JSON. Everything is fictional: lab names, patients and values.
5
+
6
+ Variation that matters for robustness:
7
+ * locale: English (US-style units) and French (Tunisia/France-style units: g/L glucose,
8
+ G/L white cells, decimal commas)
9
+ * three layouts (clean table, dotted lines, compact two-column header)
10
+ * range formats: "12.0 - 16.0", "(12,0-16,0)", "< 2,00", "> 0,40", "[70 – 99]"
11
+ * optional H/L markers next to abnormal values
12
+ * value distribution: mostly normal, some low/high, a few critical
13
+ """
14
+ from __future__ import annotations
15
+
16
+ import json
17
+ import random
18
+ from dataclasses import asdict, dataclass
19
+ from datetime import date, timedelta
20
+ from pathlib import Path
21
+
22
+ from reportlab.lib import colors
23
+ from reportlab.lib.pagesizes import A4
24
+ from reportlab.lib.units import mm
25
+ from reportlab.pdfbase import pdfmetrics
26
+ from reportlab.pdfbase.ttfonts import TTFont
27
+ from reportlab.pdfgen import canvas
28
+
29
+ from ..catalog import load_catalog
30
+ from ..flagger import flag_result
31
+ from ..models import Result
32
+
33
+ FONT_DIR = Path("/usr/share/fonts/truetype/dejavu")
34
+ pdfmetrics.registerFont(TTFont("DejaVu", str(FONT_DIR / "DejaVuSans.ttf")))
35
+ pdfmetrics.registerFont(TTFont("DejaVu-Bold", str(FONT_DIR / "DejaVuSans-Bold.ttf")))
36
+
37
+ # Units as printed on reports, per locale.
38
+ LOCALE_UNITS = {
39
+ "en": {
40
+ "hemoglobin": "g/dL", "hematocrit": "%", "rbc": "10^6/µL", "mcv": "fL", "wbc": "10^3/µL",
41
+ "platelets": "10^3/µL", "glucose": "mg/dL", "hba1c": "%", "creatinine": "mg/dL",
42
+ "urea": "mg/dL", "sodium": "mmol/L", "potassium": "mmol/L", "calcium": "mg/dL",
43
+ "cholesterol_total": "mg/dL", "ldl": "mg/dL", "hdl": "mg/dL", "triglycerides": "mg/dL",
44
+ "alt": "U/L", "ast": "U/L", "tsh": "mIU/L", "ferritin": "ng/mL", "crp": "mg/L",
45
+ "vitamin_d": "ng/mL",
46
+ },
47
+ "fr": {
48
+ "hemoglobin": "g/dL", "hematocrit": "%", "rbc": "T/L", "mcv": "fL", "wbc": "G/L",
49
+ "platelets": "G/L", "glucose": "g/L", "hba1c": "%", "creatinine": "mg/L",
50
+ "urea": "g/L", "sodium": "mmol/L", "potassium": "mmol/L", "calcium": "mmol/L",
51
+ "cholesterol_total": "g/L", "ldl": "g/L", "hdl": "g/L", "triglycerides": "g/L",
52
+ "alt": "UI/L", "ast": "UI/L", "tsh": "µUI/mL", "ferritin": "ng/mL", "crp": "mg/L",
53
+ "vitamin_d": "ng/mL",
54
+ },
55
+ }
56
+
57
+ SECTIONS = {
58
+ "blood_count": {"en": "COMPLETE BLOOD COUNT", "fr": "HÉMOGRAMME (NFS)"},
59
+ "metabolic": {"en": "GLUCOSE METABOLISM", "fr": "MÉTABOLISME GLUCIDIQUE"},
60
+ "kidney": {"en": "KIDNEY FUNCTION", "fr": "FONCTION RÉNALE"},
61
+ "electrolytes": {"en": "ELECTROLYTES", "fr": "IONOGRAMME"},
62
+ "lipids": {"en": "LIPID PANEL", "fr": "BILAN LIPIDIQUE"},
63
+ "liver": {"en": "LIVER ENZYMES", "fr": "BILAN HÉPATIQUE"},
64
+ "thyroid": {"en": "THYROID", "fr": "THYROÏDE"},
65
+ "iron": {"en": "IRON STUDIES", "fr": "BILAN MARTIAL"},
66
+ "inflammation": {"en": "INFLAMMATION", "fr": "INFLAMMATION"},
67
+ "vitamins": {"en": "VITAMINS", "fr": "VITAMINES"},
68
+ }
69
+
70
+ LABS = ["Laboratoire Les Oliviers", "Medina Biolab", "Laboratoire du Sahel",
71
+ "Cap Bon Diagnostics", "Laboratoire El Amen", "Nour Medical Lab"]
72
+ FIRST = ["Amira", "Youssef", "Salma", "Karim", "Ines", "Mehdi", "Rania", "Omar", "Leila", "Sami"]
73
+ LAST = ["Ben Salah", "Trabelsi", "Jaziri", "Haddad", "Mansour", "Gharbi", "Chaabane", "Ayari"]
74
+
75
+ HEADERS = {
76
+ "en": {"title": "LABORATORY REPORT", "patient": "Patient", "date": "Collection date",
77
+ "test": "Test", "result": "Result", "unit": "Unit", "ref": "Reference range",
78
+ "footer": "Results to be interpreted by your physician."},
79
+ "fr": {"title": "COMPTE RENDU D'ANALYSES", "patient": "Patient(e)", "date": "Date de prélèvement",
80
+ "test": "Analyse", "result": "Résultat", "unit": "Unité", "ref": "Valeurs de référence",
81
+ "footer": "Résultats à interpréter par votre médecin traitant."},
82
+ }
83
+
84
+
85
+ @dataclass
86
+ class TruthRow:
87
+ test_key: str
88
+ printed_name: str
89
+ value: float
90
+ value_str: str
91
+ unit: str
92
+ ref_low: float | None
93
+ ref_high: float | None
94
+ ref_str: str
95
+ expected_flag: str
96
+
97
+
98
+ def _decimals(high: float) -> int:
99
+ if high < 10:
100
+ return 2
101
+ if high < 100:
102
+ return 1
103
+ return 0
104
+
105
+
106
+ def _fmt(x: float, d: int, locale: str) -> str:
107
+ s = f"{x:.{d}f}"
108
+ return s.replace(".", ",") if locale == "fr" else s
109
+
110
+
111
+ def _sample_value(rng: random.Random, td, status: str) -> float:
112
+ lo, hi = td.typical_range
113
+ span = hi - lo if hi > lo else hi
114
+ if status == "normal":
115
+ return rng.uniform(lo + 0.05 * span, hi - 0.05 * span) if lo > 0 else rng.uniform(0.3 * hi, 0.95 * hi)
116
+ if status == "high":
117
+ cap = td.critical.get("high", hi * 2)
118
+ return rng.uniform(hi * 1.05, min(hi * 1.6, cap * 0.97))
119
+ if status == "low":
120
+ floor = td.critical.get("low", 0.0)
121
+ return rng.uniform(max(lo * 0.55, floor * 1.05), lo * 0.95)
122
+ if status == "critical_high":
123
+ return td.critical["high"] * rng.uniform(1.03, 1.15)
124
+ if status == "critical_low":
125
+ return td.critical["low"] * rng.uniform(0.85, 0.97)
126
+ raise ValueError(status)
127
+
128
+
129
+ def _choose_status(rng: random.Random, td) -> str:
130
+ one_sided_upper = td.typical_range[0] == 0
131
+ options = [("normal", 0.68), ("high", 0.16)]
132
+ if not one_sided_upper:
133
+ options.append(("low", 0.12))
134
+ if "high" in td.critical:
135
+ options.append(("critical_high", 0.03))
136
+ if "low" in td.critical:
137
+ options.append(("critical_low", 0.03))
138
+ r, acc = rng.random() * sum(w for _, w in options), 0.0
139
+ for name, w in options:
140
+ acc += w
141
+ if r <= acc:
142
+ return name
143
+ return "normal"
144
+
145
+
146
+ def make_report(seed: int, locale: str | None = None) -> dict:
147
+ rng = random.Random(seed)
148
+ catalog = load_catalog()
149
+ locale = locale or rng.choice(["en", "fr"])
150
+ layout = rng.choice(["table", "dotted", "compact"])
151
+ range_style = rng.choice(["dash", "paren", "bracket"])
152
+ show_hl = rng.random() < 0.5
153
+
154
+ keys = rng.sample(list(catalog), rng.randint(6, 14))
155
+ order = list(SECTIONS)
156
+ keys.sort(key=lambda k: (order.index(catalog[k].category), k))
157
+
158
+ rows: list[TruthRow] = []
159
+ for k in keys:
160
+ td = catalog[k]
161
+ unit = LOCALE_UNITS[locale][k]
162
+ factor = td.to_canonical(1.0, unit)
163
+ assert factor, (k, unit)
164
+ lo_c, hi_c = td.typical_range
165
+ lo_r, hi_r = lo_c / factor, hi_c / factor
166
+ d = _decimals(hi_r)
167
+ value = round(_sample_value(rng, td, _choose_status(rng, td)) / factor, d)
168
+
169
+ if lo_c == 0: # upper bound only
170
+ ref_low, ref_high = None, round(hi_r, d)
171
+ ref_str = f"< {_fmt(ref_high, d, locale)}"
172
+ elif td.key == "hdl": # lower bound only, as printed on real reports
173
+ ref_low, ref_high = round(lo_r, d), None
174
+ ref_str = f"> {_fmt(ref_low, d, locale)}"
175
+ else:
176
+ ref_low, ref_high = round(lo_r, d), round(hi_r, d)
177
+ a, b = _fmt(ref_low, d, locale), _fmt(ref_high, d, locale)
178
+ ref_str = {"dash": f"{a} - {b}", "paren": f"({a}-{b})", "bracket": f"[{a} – {b}]"}[range_style]
179
+
180
+ name = rng.choice(td.aliases[locale][: max(1, min(3, len(td.aliases[locale])))])
181
+ if name.isascii() and len(name.replace("-", "")) <= 6 and " " not in name:
182
+ name = name.upper() # abbreviations: MCV, HDL-C, SGOT
183
+ elif rng.random() < 0.3:
184
+ name = name.upper()
185
+ else:
186
+ name = name[:1].upper() + name[1:]
187
+
188
+ truth = Result(test_key=k, raw_name=name, value=value, unit=unit, ref_low=ref_low, ref_high=ref_high)
189
+ flag = flag_result(truth).flag.value
190
+ value_str = _fmt(value, d, locale)
191
+ rows.append(TruthRow(k, name, value, value_str, unit, ref_low, ref_high, ref_str, flag))
192
+
193
+ patient = f"{rng.choice(FIRST)} {rng.choice(LAST)}"
194
+ day = date(2026, 1, 1) + timedelta(days=rng.randint(0, 290))
195
+ return {
196
+ "id": f"rpt_{seed:05d}", "seed": seed, "locale": locale, "layout": layout,
197
+ "show_hl": show_hl, "lab": rng.choice(LABS), "patient": patient,
198
+ "date": day.isoformat(), "rows": [asdict(r) for r in rows],
199
+ }
200
+
201
+
202
+ def render_pdf(report: dict, path: Path) -> None:
203
+ catalog = load_catalog()
204
+ loc = report["locale"]
205
+ h = HEADERS[loc]
206
+ c = canvas.Canvas(str(path), pagesize=A4)
207
+ W, H = A4
208
+ x0, y = 18 * mm, H - 20 * mm
209
+
210
+ c.setFont("DejaVu-Bold", 14)
211
+ c.drawString(x0, y, report["lab"])
212
+ c.setFont("DejaVu", 8)
213
+ c.drawString(x0, y - 11, "Fictional laboratory - synthetic report generated by LabLens")
214
+ y -= 30
215
+ c.setFont("DejaVu-Bold", 12)
216
+ c.drawString(x0, y, h["title"])
217
+ y -= 18
218
+ c.setFont("DejaVu", 10)
219
+ if report["layout"] == "compact":
220
+ c.drawString(x0, y, f"{h['patient']}: {report['patient']}")
221
+ c.drawString(x0 + 95 * mm, y, f"{h['date']}: {report['date']}")
222
+ y -= 20
223
+ else:
224
+ c.drawString(x0, y, f"{h['patient']}: {report['patient']}")
225
+ c.drawString(x0, y - 14, f"{h['date']}: {report['date']}")
226
+ y -= 34
227
+
228
+ cols = [x0, x0 + 72 * mm, x0 + 102 * mm, x0 + 130 * mm]
229
+ c.setFont("DejaVu-Bold", 9.5)
230
+ for cx, label in zip(cols, [h["test"], h["result"], h["unit"], h["ref"]]):
231
+ c.drawString(cx, y, label)
232
+ c.setLineWidth(0.6)
233
+ c.line(x0, y - 4, W - 18 * mm, y - 4)
234
+ y -= 18
235
+
236
+ current = None
237
+ for r in report["rows"]:
238
+ cat = catalog[r["test_key"]].category
239
+ if cat != current:
240
+ current = cat
241
+ y -= 4
242
+ c.setFont("DejaVu-Bold", 9)
243
+ c.setFillColor(colors.HexColor("#1f4e79"))
244
+ c.drawString(x0, y, SECTIONS[cat][loc])
245
+ c.setFillColor(colors.black)
246
+ y -= 14
247
+ c.setFont("DejaVu", 9.5)
248
+ c.drawString(cols[0], y, r["printed_name"])
249
+ abnormal = r["expected_flag"] not in ("normal", "unknown")
250
+ val = r["value_str"]
251
+ if report["show_hl"] and abnormal:
252
+ val += " H" if "high" in r["expected_flag"] else " L"
253
+ c.setFont("DejaVu-Bold", 9.5)
254
+ c.drawString(cols[1], y, val)
255
+ c.setFont("DejaVu", 9.5)
256
+ c.drawString(cols[2], y, r["unit"])
257
+ c.drawString(cols[3], y, r["ref_str"])
258
+ if report["layout"] == "dotted":
259
+ c.setDash(1, 2)
260
+ c.setLineWidth(0.3)
261
+ c.line(x0, y - 3.5, W - 18 * mm, y - 3.5)
262
+ c.setDash()
263
+ y -= 15
264
+ if y < 30 * mm:
265
+ c.showPage()
266
+ y = H - 25 * mm
267
+
268
+ c.setFont("DejaVu", 8)
269
+ c.drawString(x0, 15 * mm, h["footer"])
270
+ c.save()
271
+
272
+
273
+ def degrade_to_photo(pdf_path: Path, png_path: Path, rng: random.Random) -> None:
274
+ """Rasterize page 1 and add mild photo-like noise: rotation, blur, contrast, grain."""
275
+ import subprocess
276
+
277
+ from PIL import Image, ImageEnhance, ImageFilter
278
+
279
+ stem = png_path.with_suffix("")
280
+ subprocess.run(["pdftoppm", "-png", "-r", "150", "-f", "1", "-l", "1", "-singlefile",
281
+ str(pdf_path), str(stem)], check=True)
282
+ img = Image.open(f"{stem}.png").convert("L")
283
+ img = img.rotate(rng.uniform(-1.5, 1.5), expand=True, fillcolor=245)
284
+ img = img.filter(ImageFilter.GaussianBlur(rng.uniform(0.2, 0.7)))
285
+ img = ImageEnhance.Contrast(img).enhance(rng.uniform(0.75, 1.0))
286
+ img = ImageEnhance.Brightness(img).enhance(rng.uniform(0.9, 1.05))
287
+ noise = Image.effect_noise(img.size, rng.uniform(6, 14))
288
+ img = Image.blend(img, noise, 0.06)
289
+ img.save(png_path)
290
+
291
+
292
+ def generate(out_dir: Path, n: int, start_seed: int = 0, photos: bool = True) -> list[dict]:
293
+ out_dir.mkdir(parents=True, exist_ok=True)
294
+ reports = []
295
+ for i in range(n):
296
+ seed = start_seed + i
297
+ rep = make_report(seed)
298
+ pdf = out_dir / f"{rep['id']}.pdf"
299
+ render_pdf(rep, pdf)
300
+ rep["pdf"] = pdf.name
301
+ if photos:
302
+ png = out_dir / f"{rep['id']}.png"
303
+ degrade_to_photo(pdf, png, random.Random(seed + 10_000))
304
+ rep["image"] = png.name
305
+ (out_dir / f"{rep['id']}.json").write_text(json.dumps(rep, ensure_ascii=False, indent=2), encoding="utf-8")
306
+ reports.append(rep)
307
+ return reports
308
+
309
+
310
+ if __name__ == "__main__":
311
+ import argparse
312
+
313
+ ap = argparse.ArgumentParser(description="Generate synthetic lab reports with ground truth.")
314
+ ap.add_argument("--out", default="benchmark/reports")
315
+ ap.add_argument("-n", type=int, default=100)
316
+ ap.add_argument("--seed", type=int, default=0)
317
+ ap.add_argument("--no-photos", action="store_true")
318
+ a = ap.parse_args()
319
+ reps = generate(Path(a.out), a.n, a.seed, photos=not a.no_photos)
320
+ print(f"wrote {len(reps)} reports to {a.out}")
samples/sample_en_report.pdf ADDED
Binary file (44.9 kB). View file
 
samples/sample_fr_photo.png ADDED

Git LFS Details

  • SHA256: c920d2b13b6a3013d8fd3911958ccbc19d8d6ff426075d628404e1bc0c22de17
  • Pointer size: 131 Bytes
  • Size of remote file: 767 kB
samples/sample_fr_urgent.png ADDED

Git LFS Details

  • SHA256: 2919cbe228e64ebb3d5d5a066c48760e4689878cb1251375d5e7a58eb5663f2c
  • Pointer size: 131 Bytes
  • Size of remote file: 705 kB
tests/__init__.py ADDED
File without changes
tests/test_core.py ADDED
@@ -0,0 +1,196 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ """Tests for the safety-critical, deterministic parts of LabLens."""
2
+ import pytest
3
+
4
+ from lablens.catalog import load_catalog, match_test_name
5
+ from lablens.explain import explain, render_markdown, violates_guardrails
6
+ from lablens.extract import extract, parse_line, parse_range
7
+ from lablens.flagger import flag_all, flag_result
8
+ from lablens.models import Flag, Result
9
+
10
+
11
+ def R(key, value, unit, lo=None, hi=None):
12
+ return Result(test_key=key, raw_name=key, value=value, unit=unit, ref_low=lo, ref_high=hi)
13
+
14
+
15
+ # ---------- flagging ----------
16
+
17
+ def test_report_range_decides_low_high():
18
+ assert flag_result(R("hemoglobin", 11.0, "g/dL", 12, 16)).flag == Flag.LOW
19
+ assert flag_result(R("hemoglobin", 16.5, "g/dL", 12, 16)).flag == Flag.HIGH
20
+ assert flag_result(R("hemoglobin", 14.0, "g/dL", 12, 16)).flag == Flag.NORMAL
21
+
22
+
23
+ def test_critical_beats_range_and_converts_units():
24
+ # 6.8 mmol/L potassium is critical regardless of the printed range
25
+ assert flag_result(R("potassium", 6.8, "mmol/L", 3.5, 5.1)).flag == Flag.CRITICAL_HIGH
26
+ # French g/L glucose: 0.45 g/L = 45 mg/dL, below the 50 mg/dL critical threshold
27
+ assert flag_result(R("glucose", 0.45, "g/L", 0.70, 1.10)).flag == Flag.CRITICAL_LOW
28
+ # 4.5 g/L glucose = 450 mg/dL -> critical high
29
+ assert flag_result(R("glucose", 4.5, "g/L", 0.70, 1.10)).flag == Flag.CRITICAL_HIGH
30
+
31
+
32
+ def test_cell_count_units_giga_per_litre():
33
+ # 40 G/L platelets = 40 x10^3/µL -> critical low
34
+ assert flag_result(R("platelets", 40, "G/L", 150, 400)).flag == Flag.CRITICAL_LOW
35
+
36
+
37
+ def test_implausible_value_is_never_normal_or_urgent():
38
+ # 436% HbA1c is a dropped decimal point, not a medical emergency or a normal value
39
+ r = flag_result(R("hba1c", 436, "%", 4.0, 5.6))
40
+ assert r.flag == Flag.CHECK
41
+ a = flag_all([R("hemoglobin", 151, "g/dL", 12, 17)])
42
+ assert a.results[0].flag == Flag.CHECK and not a.urgent
43
+
44
+
45
+ def test_catalog_range_used_when_report_has_none():
46
+ r = flag_result(R("sodium", 150, "mmol/L"))
47
+ assert r.flag == Flag.HIGH and r.ref_source == "catalog"
48
+
49
+
50
+ def test_unknown_unit_skips_critical_but_uses_printed_range():
51
+ r = flag_result(R("potassium", 7.0, "zz/qq", 3.5, 5.1))
52
+ assert r.flag == Flag.HIGH # not escalated: we can't be sure of the unit
53
+
54
+
55
+ def test_unknown_unit_and_no_range_is_unknown():
56
+ assert flag_result(R("potassium", 4.0, "zz/qq")).flag == Flag.UNKNOWN
57
+
58
+
59
+ def test_urgent_flag_set():
60
+ a = flag_all([R("hemoglobin", 14, "g/dL", 12, 16), R("sodium", 118, "mmol/L", 135, 145)])
61
+ assert a.urgent
62
+
63
+
64
+ def test_every_critical_threshold_is_plausible():
65
+ for td in load_catalog().values():
66
+ for v in td.critical.values():
67
+ assert td.plausible[0] < v < td.plausible[1], td.key
68
+
69
+
70
+ # ---------- name matching ----------
71
+
72
+ @pytest.mark.parametrize("name,key", [
73
+ ("Glycémie à jeun", "glucose"), ("GLYCEMIE A JEUN", "glucose"), ("Kaliémie", "potassium"),
74
+ ("Hemiogiouine", "hemoglobin"), ("HBAIC", "hba1c"), ("Ldi-choiestéroi", "ldl"),
75
+ ("Hdl-chatesteroi", "hdl"), ("K", "potassium"), ("NA", "sodium"), ("الكرياتينين", "creatinine"),
76
+ ])
77
+ def test_name_matching(name, key):
78
+ assert match_test_name(name)[0] == key
79
+
80
+
81
+ @pytest.mark.parametrize("name", ["Patient", "Collection date", "No", "Dossier", "Page"])
82
+ def test_non_tests_do_not_match(name):
83
+ assert match_test_name(name)[0] is None
84
+
85
+
86
+ # ---------- line and range parsing ----------
87
+
88
+ @pytest.mark.parametrize("text,expected", [
89
+ ("12.0 - 16.0", (12.0, 16.0)), ("(0,70-1,10)", (0.70, 1.10)), ("[4,00 – 5,60]", (4.0, 5.6)),
90
+ ("< 2,00", (None, 2.0)), ("> 0,40", (0.40, None)), ("[4.20 - 5.901", (4.20, 5.90)), ("", (None, None)),
91
+ ])
92
+ def test_parse_range(text, expected):
93
+ assert parse_range(text) == expected
94
+
95
+
96
+ def test_parse_line_french_with_marker():
97
+ r = parse_line("Kaliémie 5,36H mmol/L 3,50 - 5,10")
98
+ assert (r.test_key, r.value, r.unit, r.ref_low, r.ref_high) == ("potassium", 5.36, "mmol/L", 3.5, 5.1)
99
+
100
+
101
+ def test_parse_line_skips_ocr_noise_and_stray_digit():
102
+ assert parse_line("TSH ae 3.11 mIU/L [0.40 - 4.00]").value == 3.11
103
+ assert parse_line("K 7 3.79 mmol/L [3.50 - 5.10]").value == 3.79
104
+
105
+
106
+ def test_parse_line_ignores_headers():
107
+ assert parse_line("Date de prélèvement: 2026-04-05") is None
108
+ assert parse_line("Patient: Karim Mansour") is None
109
+
110
+
111
+ # ---------- end to end on samples ----------
112
+
113
+ def test_sample_pdf_end_to_end():
114
+ results, _, lang, method = extract("samples/sample_en_report.pdf")
115
+ assert method == "pdf-text" and lang == "en" and len(results) == 13
116
+
117
+
118
+ def test_sample_urgent_photo_end_to_end():
119
+ results, _, lang, method = extract("samples/sample_fr_urgent.png")
120
+ a = flag_all(results)
121
+ assert method == "ocr" and lang == "fr" and a.urgent
122
+
123
+
124
+ # ---------- explanation guardrails ----------
125
+
126
+ @pytest.mark.parametrize("text", [
127
+ "You have diabetes.", "Vous avez une anémie.", "Take 1000 mg of iron daily.", "Stop taking your pills.",
128
+ ])
129
+ def test_guardrails_block_diagnosis_and_dosing(text):
130
+ assert violates_guardrails(text)
131
+
132
+
133
+ def test_guardrails_allow_hedged_language():
134
+ assert not violates_guardrails("A high value can be linked to diabetes; your doctor can confirm.")
135
+
136
+
137
+ def test_template_explanation_without_api_key(monkeypatch):
138
+ monkeypatch.delenv("HF_TOKEN", raising=False)
139
+ monkeypatch.delenv("LLM_API_KEY", raising=False)
140
+ a = flag_all([R("ferritin", 8, "ng/mL", 15, 200), R("tsh", 2.0, "mIU/L", 0.4, 4.0)])
141
+ exp, source = explain(a, "fr")
142
+ assert source.startswith("template")
143
+ md = render_markdown(a, exp, "fr")
144
+ assert "Ferritine" in md and "Bas" in md and len(exp["questions"]) == 1
145
+
146
+
147
+ # ---------- LLM path (mocked: no network in tests) ----------
148
+
149
+ class _FakeClient:
150
+ def __init__(self, content):
151
+ msg = type("M", (), {"content": content})
152
+ choice = type("C", (), {"message": msg})
153
+ self.chat = type("Ch", (), {"completions": type("Co", (), {
154
+ "create": staticmethod(lambda **kw: type("R", (), {"choices": [choice]}))})})
155
+
156
+
157
+ def _patch_llm(monkeypatch, content):
158
+ import openai
159
+ monkeypatch.setenv("HF_TOKEN", "test")
160
+ monkeypatch.setattr(openai, "OpenAI", lambda **kw: _FakeClient(content))
161
+
162
+
163
+ def test_llm_output_used_when_safe(monkeypatch):
164
+ _patch_llm(monkeypatch, '{"summary": "Mostly fine.", "results": [{"key": "ferritin", "explanation": '
165
+ '"Low iron stores."}, {"key": "made_up", "explanation": "x"}], "questions": ["Why?"]}')
166
+ a = flag_all([R("ferritin", 8, "ng/mL", 15, 200)])
167
+ exp, source = explain(a, "en")
168
+ assert source == "llm"
169
+ assert [x["key"] for x in exp["results"]] == ["ferritin"] # hallucinated test dropped
170
+
171
+
172
+ def test_llm_diagnosis_falls_back_to_template(monkeypatch):
173
+ _patch_llm(monkeypatch, '{"summary": "You have anemia.", "results": [], "questions": []}')
174
+ a = flag_all([R("hemoglobin", 9, "g/dL", 12, 16)])
175
+ exp, source = explain(a, "en")
176
+ assert source.startswith("template") and "guardrail" in source
177
+
178
+
179
+ def test_llm_garbage_falls_back_to_template(monkeypatch):
180
+ _patch_llm(monkeypatch, "Sorry, I can't help with that.")
181
+ exp, source = explain(flag_all([R("tsh", 9, "mIU/L", 0.4, 4.0)]), "ar")
182
+ assert source.startswith("template")
183
+
184
+
185
+ def test_lab_marker_contradiction_becomes_check():
186
+ r = R("ast", 6.5, "UI/L", 3.0, 48.0) # range misread (8,0 -> 3,0) but lab printed "L"
187
+ r.report_marker = "L"
188
+ assert flag_result(r).flag == Flag.CHECK
189
+ ok = R("ast", 6.5, "UI/L", 8.0, 48.0)
190
+ ok.report_marker = "L"
191
+ assert flag_result(ok).flag == Flag.LOW
192
+
193
+
194
+ def test_glued_marker_with_trailing_punctuation():
195
+ r = parse_line("Vitamine d 22L. ng/mL 30 - 100")
196
+ assert (r.value, r.report_marker, r.ref_low) == (22.0, "L", 30.0)