"""B3 — does the parser choice change how many gold terms survive to extraction? uv run --no-sync python -m eval.parsing.run_eval --artifact --label hybrid-high One question, one variable. The term filter, the gold set and the scorer are extraction's and are used **exactly as they ship** — the only thing that differs between runs here is which parser produced the artifact. Anything else that moved would make the comparison meaningless. Free stage: GLiNER is a local CPU encoder, so this costs no API calls. That is what makes the parser question answerable without a paid run. Why this lives in `eval/parsing/` and not `eval/knowledge/`: the yardstick is extraction's, but the experiment is parsing's. `eval/knowledge/run_eval.py` is a reserved slot for extraction's own runner (see its README) and is deliberately left free. The comparison baseline is PyMuPDF at recall 0.8537 (35/41), frozen in `eval/knowledge/results/baseline_prototype_2026-08-13_145132.json`. Calibration §10 measured MinerU at 0.8293 through a *scratch converter* and attributed the shortfall to chunking rather than content — that claim is what this runner exists to confirm or refute, now that the real module produces the artifact. """ from __future__ import annotations import argparse import json import sys from datetime import datetime from pathlib import Path ROOT = Path(__file__).resolve().parent.parent.parent GOLD = ROOT / "eval" / "knowledge" / "knowledge_gold.yaml" RESULTS = Path(__file__).resolve().parent / "results" # The committed reference artifact — see fixtures/README.md. Default so that # running this with no arguments scores the artifact both halves share, rather # than whichever run output happens to be newest on one person's disk. FIXTURE = (Path(__file__).resolve().parent / "fixtures" / "STD_2026_006_MNO.hybrid-high.json") def load_artifact(path: Path) -> tuple[list, str]: """Artifact on disk → extraction's `Chunk` list, through the real seam. Deliberately routed through `adapter.parsed_doc_from_artifact` rather than a local conversion: a bespoke converter here would be measuring a path that production never takes, which is the exact flaw calibration §10 disclosed in its own numbers. """ from src.knowledge_extraction.adapter import parsed_doc_from_artifact raw = json.loads(path.read_text(encoding="utf-8")) doc = parsed_doc_from_artifact(raw) return doc.chunks, raw def main() -> int: ap = argparse.ArgumentParser(description=__doc__) ap.add_argument("--artifact", type=Path, default=FIXTURE, help="chunks.json to score (default: the committed fixture)") ap.add_argument("--label", default="fixture", help="what produced it, e.g. hybrid-high (names the result file)") ap.add_argument("--threshold", type=float, default=None, help="override the filter's configured score floor (leave unset)") args = ap.parse_args() if not args.artifact.exists(): print(f"artifact not found: {args.artifact}", file=sys.stderr) return 1 sys.path.insert(0, str(ROOT)) from eval.knowledge.score import load_gold, score_term_filter from src.knowledge_extraction.filters.span_filter import extract_mentions from src.knowledge_parsing.normalize import normaliser_version chunks, raw = load_artifact(args.artifact) backend = raw.get("parser_backend") or "unknown" print(f"artifact : {args.artifact}") print(f"backend : {backend}") print(f"chunks : {len(chunks)}") # The check this eval exists to make possible. An artifact is a frozen # snapshot of the normalisation code at the moment it ran; when that code is # fixed, existing artifacts do not update themselves and nothing about their # metadata says so. Scoring a stale one produces a real number for input that # no longer represents the pipeline — which is how a parsing bug once read as # an extraction regression for a day. stamped = raw.get("normaliser_version") current = normaliser_version() if stamped is None: print(" !! no normaliser_version — artifact predates the stamp (schema < 0.3.1).") print(" Rebuild before trusting this score: py scripts/parse.py --backend hybrid") elif stamped != current: print(f" !! STALE ARTIFACT — built by normaliser {stamped}, current is {current}.") print(" The score below is for input the pipeline no longer produces.") print(" Rebuild (no GPU needed if the parse is cached):") print(" py scripts/parse.py --backend hybrid") if backend == "pipeline": # Not fatal, but it must never be quoted as MinerU's quality: this # backend transcribes the multiplication sign as a bare letter and # produces no image descriptions. Calibration §13. print(" !! pipeline output is the LOW-QUALITY backend — do not quote as a") print(" measurement of MinerU. Production is hybrid/high.") mentions = extract_mentions(chunks, threshold=args.threshold) surfaces = [m.surface for m in mentions] capped = sum(1 for m in mentions if m.hit_span_cap) print(f"mentions : {len(mentions)} ({len(set(surfaces))} distinct)" + (f", {capped} hit the span cap" if capped else "")) gold = load_gold(GOLD) score = score_term_filter(gold, surfaces) print("\n--- E1 term filter ---") print(f" recall {score.recall:.4f} ({score.true_positives}/{score.n_gold})") print(f" precision {score.precision:.4f} (reported, NOT E1's criterion)") print(f" f1 {score.f1:.4f}") print("\n baseline 0.8537 (35/41, PyMuPDF)") delta = score.recall - 0.8537 verdict = "BEATS" if delta > 0 else "below" if delta < 0 else "matches" print(f" delta {delta:+.4f} -> {verdict} the PyMuPDF baseline") if score.misses: print(f"\n missed ({len(score.misses)}): {', '.join(sorted(score.misses))}") RESULTS.mkdir(parents=True, exist_ok=True) stamp = datetime.now().strftime("%Y-%m-%d_%H%M%S") out = RESULTS / f"b3_{args.label}_{stamp}.json" payload = { "experiment": "B3 parser comparison", "label": args.label, "artifact": str(args.artifact), "parser_backend": backend, "normaliser_version": stamped, "normaliser_current": current, "artifact_stale": stamped != current, "chunks": len(chunks), "mentions": len(mentions), "distinct_surfaces": len(set(surfaces)), "span_cap_hits": capped, "threshold_override": args.threshold, "score": score.as_dict(), "gold": str(GOLD), "compared_against": "PyMuPDF 0.8537 (35/41), " "eval/knowledge/results/baseline_prototype_2026-08-13_145132.json", } out.write_text(json.dumps(payload, indent=2, default=str), encoding="utf-8") print(f"\nwritten : {out}") return 0 if __name__ == "__main__": raise SystemExit(main())