File size: 4,832 Bytes
dab7db2
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
"""Benchmark the extraction + flagging pipeline against synthetic ground truth.

Metrics (per input type: PDF text layer vs degraded photo, and per locale):
  recall            share of true test rows that were found
  precision         share of extracted rows that are real rows of the report
  value_acc         exact value match among found rows
  unit_acc          unit understood as the same unit (same conversion factor)
  range_acc         reference range read exactly
  flag_acc          end-to-end flag equals the ground-truth flag (missing rows count as wrong)
  critical_recall   critical rows that came out as critical (the safety metric)
  dangerous_misses  abnormal rows reported as NORMAL (worse than missing: false reassurance)
"""
from __future__ import annotations

import argparse
import json
from collections import defaultdict
from pathlib import Path

from .catalog import load_catalog
from .extract import extract
from .flagger import flag_all


def _same_unit(key: str, a: str | None, b: str | None) -> bool:
    td = load_catalog()[key]
    fa, fb = td.to_canonical(1.0, a), td.to_canonical(1.0, b)
    return fa is not None and fb is not None and abs(fa - fb) < 1e-9


def score_report(truth: dict, path: Path) -> dict:
    results, _, _, _ = extract(path)
    analysis = flag_all(results)
    got = {r.test_key: r for r in analysis.results}
    true_keys = {row["test_key"] for row in truth["rows"]}

    c = defaultdict(int)
    c["rows"] = len(truth["rows"])
    c["extracted"] = len(got)
    c["true_positive_keys"] = len(true_keys & set(got))
    for row in truth["rows"]:
        k, exp = row["test_key"], row["expected_flag"]
        crit = exp.startswith("critical")
        c["critical_rows"] += crit
        c["abnormal_rows"] += exp != "normal"
        g = got.get(k)
        if g is None:
            continue
        c["found"] += 1
        c["value_ok"] += abs(g.value - row["value"]) < 1e-9
        c["unit_ok"] += _same_unit(k, g.unit, row["unit"])
        c["range_ok"] += (g.ref_low, g.ref_high) == (row["ref_low"], row["ref_high"]) or (
            g.ref_source == "catalog" and row["ref_low"] is None and row["ref_high"] is None)
        c["flag_ok"] += g.flag.value == exp
        c["critical_caught"] += crit and g.flag.value.startswith("critical")
        c["dangerous_miss"] += exp != "normal" and g.flag.value == "normal"
    return dict(c)


def summarize(counts: list[dict]) -> dict:
    t = defaultdict(int)
    for c in counts:
        for k, v in c.items():
            t[k] += v

    def ratio(a, b):
        return round(t[a] / t[b], 4) if t[b] else None

    return {
        "reports": len(counts), "rows": t["rows"],
        "recall": ratio("found", "rows"),
        "precision": ratio("true_positive_keys", "extracted"),
        "value_acc": ratio("value_ok", "found"),
        "unit_acc": ratio("unit_ok", "found"),
        "range_acc": ratio("range_ok", "found"),
        "flag_acc": ratio("flag_ok", "rows"),
        "critical_recall": ratio("critical_caught", "critical_rows"),
        "critical_rows": t["critical_rows"],
        "dangerous_misses": t["dangerous_miss"],
        "abnormal_rows": t["abnormal_rows"],
    }


def run(bench_dir: Path) -> dict:
    groups: dict[str, list[dict]] = defaultdict(list)
    for jf in sorted(bench_dir.glob("rpt_*.json")):
        truth = json.loads(jf.read_text(encoding="utf-8"))
        for kind, field in (("pdf", "pdf"), ("photo", "image")):
            if field not in truth:
                continue
            c = score_report(truth, bench_dir / truth[field])
            groups[f"{kind}/all"].append(c)
            groups[f"{kind}/{truth['locale']}"].append(c)
    return {g: summarize(v) for g, v in sorted(groups.items())}


def to_markdown(summary: dict) -> str:
    cols = ["reports", "rows", "recall", "precision", "value_acc", "unit_acc", "range_acc",
            "flag_acc", "critical_recall", "dangerous_misses"]
    out = ["| split | " + " | ".join(cols) + " |", "|" + "---|" * (len(cols) + 1)]
    for g, s in summary.items():
        cells = []
        for k in cols:
            v = s[k]
            if isinstance(v, float) and k not in ("reports", "rows", "dangerous_misses"):
                cells.append(f"{v:.1%}")
            elif k == "critical_recall" and v is None:
                cells.append("n/a")
            else:
                cells.append(str(v))
        out.append(f"| {g} | " + " | ".join(cells) + " |")
    return "\n".join(out)


if __name__ == "__main__":
    ap = argparse.ArgumentParser()
    ap.add_argument("--bench", default="benchmark/reports")
    ap.add_argument("--out", default="benchmark/results.json")
    a = ap.parse_args()
    summary = run(Path(a.bench))
    Path(a.out).write_text(json.dumps(summary, indent=2), encoding="utf-8")
    print(to_markdown(summary))