"""Benchmark the extraction + flagging pipeline against synthetic ground truth. Metrics (per input type: PDF text layer vs degraded photo, and per locale): recall share of true test rows that were found precision share of extracted rows that are real rows of the report value_acc exact value match among found rows unit_acc unit understood as the same unit (same conversion factor) range_acc reference range read exactly flag_acc end-to-end flag equals the ground-truth flag (missing rows count as wrong) critical_recall critical rows that came out as critical (the safety metric) dangerous_misses abnormal rows reported as NORMAL (worse than missing: false reassurance) """ from __future__ import annotations import argparse import json from collections import defaultdict from pathlib import Path from .catalog import load_catalog from .extract import extract from .flagger import flag_all def _same_unit(key: str, a: str | None, b: str | None) -> bool: td = load_catalog()[key] fa, fb = td.to_canonical(1.0, a), td.to_canonical(1.0, b) return fa is not None and fb is not None and abs(fa - fb) < 1e-9 def score_report(truth: dict, path: Path) -> dict: results, _, _, _ = extract(path) analysis = flag_all(results) got = {r.test_key: r for r in analysis.results} true_keys = {row["test_key"] for row in truth["rows"]} c = defaultdict(int) c["rows"] = len(truth["rows"]) c["extracted"] = len(got) c["true_positive_keys"] = len(true_keys & set(got)) for row in truth["rows"]: k, exp = row["test_key"], row["expected_flag"] crit = exp.startswith("critical") c["critical_rows"] += crit c["abnormal_rows"] += exp != "normal" g = got.get(k) if g is None: continue c["found"] += 1 c["value_ok"] += abs(g.value - row["value"]) < 1e-9 c["unit_ok"] += _same_unit(k, g.unit, row["unit"]) c["range_ok"] += (g.ref_low, g.ref_high) == (row["ref_low"], row["ref_high"]) or ( g.ref_source == "catalog" and row["ref_low"] is None and row["ref_high"] is None) c["flag_ok"] += g.flag.value == exp c["critical_caught"] += crit and g.flag.value.startswith("critical") c["dangerous_miss"] += exp != "normal" and g.flag.value == "normal" return dict(c) def summarize(counts: list[dict]) -> dict: t = defaultdict(int) for c in counts: for k, v in c.items(): t[k] += v def ratio(a, b): return round(t[a] / t[b], 4) if t[b] else None return { "reports": len(counts), "rows": t["rows"], "recall": ratio("found", "rows"), "precision": ratio("true_positive_keys", "extracted"), "value_acc": ratio("value_ok", "found"), "unit_acc": ratio("unit_ok", "found"), "range_acc": ratio("range_ok", "found"), "flag_acc": ratio("flag_ok", "rows"), "critical_recall": ratio("critical_caught", "critical_rows"), "critical_rows": t["critical_rows"], "dangerous_misses": t["dangerous_miss"], "abnormal_rows": t["abnormal_rows"], } def run(bench_dir: Path) -> dict: groups: dict[str, list[dict]] = defaultdict(list) for jf in sorted(bench_dir.glob("rpt_*.json")): truth = json.loads(jf.read_text(encoding="utf-8")) for kind, field in (("pdf", "pdf"), ("photo", "image")): if field not in truth: continue c = score_report(truth, bench_dir / truth[field]) groups[f"{kind}/all"].append(c) groups[f"{kind}/{truth['locale']}"].append(c) return {g: summarize(v) for g, v in sorted(groups.items())} def to_markdown(summary: dict) -> str: cols = ["reports", "rows", "recall", "precision", "value_acc", "unit_acc", "range_acc", "flag_acc", "critical_recall", "dangerous_misses"] out = ["| split | " + " | ".join(cols) + " |", "|" + "---|" * (len(cols) + 1)] for g, s in summary.items(): cells = [] for k in cols: v = s[k] if isinstance(v, float) and k not in ("reports", "rows", "dangerous_misses"): cells.append(f"{v:.1%}") elif k == "critical_recall" and v is None: cells.append("n/a") else: cells.append(str(v)) out.append(f"| {g} | " + " | ".join(cells) + " |") return "\n".join(out) if __name__ == "__main__": ap = argparse.ArgumentParser() ap.add_argument("--bench", default="benchmark/reports") ap.add_argument("--out", default="benchmark/results.json") a = ap.parse_args() summary = run(Path(a.bench)) Path(a.out).write_text(json.dumps(summary, indent=2), encoding="utf-8") print(to_markdown(summary))