LabLens / lablens /catalog.py
Imene-ee's picture
Upload 19 files
dab7db2 verified
Raw History Blame Contribute Delete
5.27 kB
"""Load the test catalog and resolve names and units against it."""
from __future__ import annotations
import re
import unicodedata
from dataclasses import dataclass, field
from functools import lru_cache
from pathlib import Path
import yaml
from rapidfuzz import fuzz, process
CATALOG_PATH = Path(__file__).parent / "data" / "catalog.yaml"
def normalize_text(s: str) -> str:
"""Lowercase, strip accents (Latin only), collapse whitespace."""
s = s.strip().lower()
# Strip Latin diacritics but keep Arabic letters intact.
decomposed = unicodedata.normalize("NFKD", s)
s = "".join(c for c in decomposed if not (unicodedata.combining(c) and ord(c) < 0x0600))
s = s.replace("μ", "µ")
return re.sub(r"\s+", " ", s)
def normalize_unit(u: str) -> str:
u = normalize_text(u or "").replace(" ", "")
u = u.replace("μ", "µ").replace("×", "").replace("x10", "10")
u = u.replace("10e", "10^").replace("10*", "10^")
u = re.sub(r"10\^?(\d)", r"10^\1", u)
return u
@dataclass
class TestDef:
key: str
display: dict[str, str]
category: str
canonical_unit: str
units: dict[str, float]
typical_range: tuple[float, float]
critical: dict[str, float]
plausible: tuple[float, float]
aliases: dict[str, list[str]]
kb: dict[str, str]
all_aliases: list[str] = field(default_factory=list)
def to_canonical(self, value: float, unit: str | None) -> float | None:
"""Convert a value to the canonical unit. Returns None if the unit is unknown."""
if not unit:
return value # assume canonical when no unit printed
nu = normalize_unit(unit)
for spelled, factor in self.units.items():
if normalize_unit(spelled) == nu:
return value * factor
if nu == normalize_unit(self.canonical_unit):
return value
return None
@lru_cache(maxsize=1)
def load_catalog(path: str | None = None) -> dict[str, TestDef]:
raw = yaml.safe_load(Path(path or CATALOG_PATH).read_text(encoding="utf-8"))
out: dict[str, TestDef] = {}
for key, t in raw["tests"].items():
td = TestDef(
key=key,
display=t["display"],
category=t["category"],
canonical_unit=t["canonical_unit"],
units=t["units"],
typical_range=tuple(t["typical_range"]),
critical=t.get("critical") or {},
plausible=tuple(t["plausible"]),
aliases=t["aliases"],
kb=t["kb"],
)
td.all_aliases = sorted(
{normalize_text(a) for lang in td.aliases.values() for a in lang}, key=len, reverse=True
)
out[key] = td
return out
@lru_cache(maxsize=1)
def _alias_index() -> dict[str, str]:
idx: dict[str, str] = {}
for key, td in load_catalog().items():
for a in td.all_aliases:
idx.setdefault(a, key)
return idx
# Characters Tesseract commonly confuses, folded to one representative before fuzzy
# matching: "Hemiogiouine" and "hemoglobine" end up much closer.
_OCR_FOLD = str.maketrans({"l": "i", "1": "i", "|": "i", "!": "i", "j": "i", "t": "i",
"0": "o", "a": "o", "e": "o", "c": "o", "u": "o", "b": "o", "g": "o"})
def _fold(s: str) -> str:
s = s.replace("rn", "m").replace("-", " ").replace("_", " ")
return re.sub(r"\s+", " ", s.translate(_OCR_FOLD)).strip()
@lru_cache(maxsize=1)
def _folded_index() -> dict[str, set[str]]:
out: dict[str, set[str]] = {}
for a, k in _alias_index().items():
if len(a) >= 4:
out.setdefault(_fold(a), set()).add(k)
return out
def match_test_name(name: str, min_score: int = 80, margin: float = 6.0) -> tuple[str | None, float]:
"""Map a printed test name to a catalog key.
Exact alias first. Otherwise fuzzy matching on OCR-folded strings, accepted only when
the best test beats the runner-up *test* by a margin. HDL vs LDL differ by one letter
and have opposite meanings, so an ambiguous match is rejected rather than guessed.
"""
n = normalize_text(name)
n = re.sub(r"[:.\-–_]+$", "", n).strip()
idx = _alias_index()
if n in idx:
return idx[n], 100.0
if len(n) < 4:
# Short aliases (hb, k, na, a1c) only match exactly after OCR folding, and only
# when the folded form is unambiguous ("Alc" -> a1c).
if len(n) < 3:
return None, 0.0 # never guess 1-2 letter tokens ("No", "N°" sample numbers)
keys = {k for a, k in idx.items() if len(a) < 4 and _fold(a) == _fold(n)}
return (keys.pop(), 90.0) if len(keys) == 1 else (None, 0.0)
folded = _folded_index()
hits = process.extract(_fold(n), list(folded), scorer=fuzz.ratio, limit=12)
best_per_key: dict[str, float] = {}
for alias, score, _ in hits:
for k in folded[alias]:
best_per_key[k] = max(best_per_key.get(k, 0.0), score)
ranked = sorted(best_per_key.items(), key=lambda kv: kv[1], reverse=True)
if not ranked or ranked[0][1] < min_score:
return None, ranked[0][1] if ranked else 0.0
if len(ranked) > 1 and ranked[0][1] - ranked[1][1] < margin:
return None, ranked[0][1]
return ranked[0][0], float(ranked[0][1])