Download lablens/evaluate.py from Imene-ee/LabLens: direct link, hf CLI and curl.
- Browser
- Download file 4.83 kB
-
https://huggingface.co/spaces/Imene-ee/LabLens/resolve/main/lablens/evaluate.py
- Command line
-
hf download hf://spaces/Imene-ee/LabLens/lablens/evaluate.py
-
curl -L -o evaluate.py https://huggingface.co/spaces/Imene-ee/LabLens/resolve/main/lablens/evaluate.py
4.83 kB
| """Benchmark the extraction + flagging pipeline against synthetic ground truth. | |
| Metrics (per input type: PDF text layer vs degraded photo, and per locale): | |
| recall share of true test rows that were found | |
| precision share of extracted rows that are real rows of the report | |
| value_acc exact value match among found rows | |
| unit_acc unit understood as the same unit (same conversion factor) | |
| range_acc reference range read exactly | |
| flag_acc end-to-end flag equals the ground-truth flag (missing rows count as wrong) | |
| critical_recall critical rows that came out as critical (the safety metric) | |
| dangerous_misses abnormal rows reported as NORMAL (worse than missing: false reassurance) | |
| """ | |
| from __future__ import annotations | |
| import argparse | |
| import json | |
| from collections import defaultdict | |
| from pathlib import Path | |
| from .catalog import load_catalog | |
| from .extract import extract | |
| from .flagger import flag_all | |
| def _same_unit(key: str, a: str | None, b: str | None) -> bool: | |
| td = load_catalog()[key] | |
| fa, fb = td.to_canonical(1.0, a), td.to_canonical(1.0, b) | |
| return fa is not None and fb is not None and abs(fa - fb) < 1e-9 | |
| def score_report(truth: dict, path: Path) -> dict: | |
| results, _, _, _ = extract(path) | |
| analysis = flag_all(results) | |
| got = {r.test_key: r for r in analysis.results} | |
| true_keys = {row["test_key"] for row in truth["rows"]} | |
| c = defaultdict(int) | |
| c["rows"] = len(truth["rows"]) | |
| c["extracted"] = len(got) | |
| c["true_positive_keys"] = len(true_keys & set(got)) | |
| for row in truth["rows"]: | |
| k, exp = row["test_key"], row["expected_flag"] | |
| crit = exp.startswith("critical") | |
| c["critical_rows"] += crit | |
| c["abnormal_rows"] += exp != "normal" | |
| g = got.get(k) | |
| if g is None: | |
| continue | |
| c["found"] += 1 | |
| c["value_ok"] += abs(g.value - row["value"]) < 1e-9 | |
| c["unit_ok"] += _same_unit(k, g.unit, row["unit"]) | |
| c["range_ok"] += (g.ref_low, g.ref_high) == (row["ref_low"], row["ref_high"]) or ( | |
| g.ref_source == "catalog" and row["ref_low"] is None and row["ref_high"] is None) | |
| c["flag_ok"] += g.flag.value == exp | |
| c["critical_caught"] += crit and g.flag.value.startswith("critical") | |
| c["dangerous_miss"] += exp != "normal" and g.flag.value == "normal" | |
| return dict(c) | |
| def summarize(counts: list[dict]) -> dict: | |
| t = defaultdict(int) | |
| for c in counts: | |
| for k, v in c.items(): | |
| t[k] += v | |
| def ratio(a, b): | |
| return round(t[a] / t[b], 4) if t[b] else None | |
| return { | |
| "reports": len(counts), "rows": t["rows"], | |
| "recall": ratio("found", "rows"), | |
| "precision": ratio("true_positive_keys", "extracted"), | |
| "value_acc": ratio("value_ok", "found"), | |
| "unit_acc": ratio("unit_ok", "found"), | |
| "range_acc": ratio("range_ok", "found"), | |
| "flag_acc": ratio("flag_ok", "rows"), | |
| "critical_recall": ratio("critical_caught", "critical_rows"), | |
| "critical_rows": t["critical_rows"], | |
| "dangerous_misses": t["dangerous_miss"], | |
| "abnormal_rows": t["abnormal_rows"], | |
| } | |
| def run(bench_dir: Path) -> dict: | |
| groups: dict[str, list[dict]] = defaultdict(list) | |
| for jf in sorted(bench_dir.glob("rpt_*.json")): | |
| truth = json.loads(jf.read_text(encoding="utf-8")) | |
| for kind, field in (("pdf", "pdf"), ("photo", "image")): | |
| if field not in truth: | |
| continue | |
| c = score_report(truth, bench_dir / truth[field]) | |
| groups[f"{kind}/all"].append(c) | |
| groups[f"{kind}/{truth['locale']}"].append(c) | |
| return {g: summarize(v) for g, v in sorted(groups.items())} | |
| def to_markdown(summary: dict) -> str: | |
| cols = ["reports", "rows", "recall", "precision", "value_acc", "unit_acc", "range_acc", | |
| "flag_acc", "critical_recall", "dangerous_misses"] | |
| out = ["| split | " + " | ".join(cols) + " |", "|" + "---|" * (len(cols) + 1)] | |
| for g, s in summary.items(): | |
| cells = [] | |
| for k in cols: | |
| v = s[k] | |
| if isinstance(v, float) and k not in ("reports", "rows", "dangerous_misses"): | |
| cells.append(f"{v:.1%}") | |
| elif k == "critical_recall" and v is None: | |
| cells.append("n/a") | |
| else: | |
| cells.append(str(v)) | |
| out.append(f"| {g} | " + " | ".join(cells) + " |") | |
| return "\n".join(out) | |
| if __name__ == "__main__": | |
| ap = argparse.ArgumentParser() | |
| ap.add_argument("--bench", default="benchmark/reports") | |
| ap.add_argument("--out", default="benchmark/results.json") | |
| a = ap.parse_args() | |
| summary = run(Path(a.bench)) | |
| Path(a.out).write_text(json.dumps(summary, indent=2), encoding="utf-8") | |
| print(to_markdown(summary)) | |