[NOTICKET] feat(knowledge-parsing): commit the reference artifact, and stop the stamp churning
a0e59e4 | """B3 — does the parser choice change how many gold terms survive to extraction? | |
| uv run --no-sync python -m eval.parsing.run_eval --artifact <chunks.json> --label hybrid-high | |
| One question, one variable. The term filter, the gold set and the scorer are | |
| extraction's and are used **exactly as they ship** — the only thing that differs | |
| between runs here is which parser produced the artifact. Anything else that | |
| moved would make the comparison meaningless. | |
| Free stage: GLiNER is a local CPU encoder, so this costs no API calls. That is | |
| what makes the parser question answerable without a paid run. | |
| Why this lives in `eval/parsing/` and not `eval/knowledge/`: the yardstick is | |
| extraction's, but the experiment is parsing's. `eval/knowledge/run_eval.py` is a | |
| reserved slot for extraction's own runner (see its README) and is deliberately | |
| left free. | |
| The comparison baseline is PyMuPDF at recall 0.8537 (35/41), frozen in | |
| `eval/knowledge/results/baseline_prototype_2026-08-13_145132.json`. Calibration | |
| §10 measured MinerU at 0.8293 through a *scratch converter* and attributed the | |
| shortfall to chunking rather than content — that claim is what this runner | |
| exists to confirm or refute, now that the real module produces the artifact. | |
| """ | |
| from __future__ import annotations | |
| import argparse | |
| import json | |
| import sys | |
| from datetime import datetime | |
| from pathlib import Path | |
| ROOT = Path(__file__).resolve().parent.parent.parent | |
| GOLD = ROOT / "eval" / "knowledge" / "knowledge_gold.yaml" | |
| RESULTS = Path(__file__).resolve().parent / "results" | |
| # The committed reference artifact — see fixtures/README.md. Default so that | |
| # running this with no arguments scores the artifact both halves share, rather | |
| # than whichever run output happens to be newest on one person's disk. | |
| FIXTURE = (Path(__file__).resolve().parent / "fixtures" | |
| / "STD_2026_006_MNO.hybrid-high.json") | |
| def load_artifact(path: Path) -> tuple[list, str]: | |
| """Artifact on disk → extraction's `Chunk` list, through the real seam. | |
| Deliberately routed through `adapter.parsed_doc_from_artifact` rather than a | |
| local conversion: a bespoke converter here would be measuring a path that | |
| production never takes, which is the exact flaw calibration §10 disclosed in | |
| its own numbers. | |
| """ | |
| from src.knowledge_extraction.adapter import parsed_doc_from_artifact | |
| raw = json.loads(path.read_text(encoding="utf-8")) | |
| doc = parsed_doc_from_artifact(raw) | |
| return doc.chunks, raw | |
| def main() -> int: | |
| ap = argparse.ArgumentParser(description=__doc__) | |
| ap.add_argument("--artifact", type=Path, default=FIXTURE, | |
| help="chunks.json to score (default: the committed fixture)") | |
| ap.add_argument("--label", default="fixture", | |
| help="what produced it, e.g. hybrid-high (names the result file)") | |
| ap.add_argument("--threshold", type=float, default=None, | |
| help="override the filter's configured score floor (leave unset)") | |
| args = ap.parse_args() | |
| if not args.artifact.exists(): | |
| print(f"artifact not found: {args.artifact}", file=sys.stderr) | |
| return 1 | |
| sys.path.insert(0, str(ROOT)) | |
| from eval.knowledge.score import load_gold, score_term_filter | |
| from src.knowledge_extraction.filters.span_filter import extract_mentions | |
| from src.knowledge_parsing.normalize import normaliser_version | |
| chunks, raw = load_artifact(args.artifact) | |
| backend = raw.get("parser_backend") or "unknown" | |
| print(f"artifact : {args.artifact}") | |
| print(f"backend : {backend}") | |
| print(f"chunks : {len(chunks)}") | |
| # The check this eval exists to make possible. An artifact is a frozen | |
| # snapshot of the normalisation code at the moment it ran; when that code is | |
| # fixed, existing artifacts do not update themselves and nothing about their | |
| # metadata says so. Scoring a stale one produces a real number for input that | |
| # no longer represents the pipeline — which is how a parsing bug once read as | |
| # an extraction regression for a day. | |
| stamped = raw.get("normaliser_version") | |
| current = normaliser_version() | |
| if stamped is None: | |
| print(" !! no normaliser_version — artifact predates the stamp (schema < 0.3.1).") | |
| print(" Rebuild before trusting this score: py scripts/parse.py --backend hybrid") | |
| elif stamped != current: | |
| print(f" !! STALE ARTIFACT — built by normaliser {stamped}, current is {current}.") | |
| print(" The score below is for input the pipeline no longer produces.") | |
| print(" Rebuild (no GPU needed if the parse is cached):") | |
| print(" py scripts/parse.py --backend hybrid") | |
| if backend == "pipeline": | |
| # Not fatal, but it must never be quoted as MinerU's quality: this | |
| # backend transcribes the multiplication sign as a bare letter and | |
| # produces no image descriptions. Calibration §13. | |
| print(" !! pipeline output is the LOW-QUALITY backend — do not quote as a") | |
| print(" measurement of MinerU. Production is hybrid/high.") | |
| mentions = extract_mentions(chunks, threshold=args.threshold) | |
| surfaces = [m.surface for m in mentions] | |
| capped = sum(1 for m in mentions if m.hit_span_cap) | |
| print(f"mentions : {len(mentions)} ({len(set(surfaces))} distinct)" | |
| + (f", {capped} hit the span cap" if capped else "")) | |
| gold = load_gold(GOLD) | |
| score = score_term_filter(gold, surfaces) | |
| print("\n--- E1 term filter ---") | |
| print(f" recall {score.recall:.4f} ({score.true_positives}/{score.n_gold})") | |
| print(f" precision {score.precision:.4f} (reported, NOT E1's criterion)") | |
| print(f" f1 {score.f1:.4f}") | |
| print("\n baseline 0.8537 (35/41, PyMuPDF)") | |
| delta = score.recall - 0.8537 | |
| verdict = "BEATS" if delta > 0 else "below" if delta < 0 else "matches" | |
| print(f" delta {delta:+.4f} -> {verdict} the PyMuPDF baseline") | |
| if score.misses: | |
| print(f"\n missed ({len(score.misses)}): {', '.join(sorted(score.misses))}") | |
| RESULTS.mkdir(parents=True, exist_ok=True) | |
| stamp = datetime.now().strftime("%Y-%m-%d_%H%M%S") | |
| out = RESULTS / f"b3_{args.label}_{stamp}.json" | |
| payload = { | |
| "experiment": "B3 parser comparison", | |
| "label": args.label, | |
| "artifact": str(args.artifact), | |
| "parser_backend": backend, | |
| "normaliser_version": stamped, | |
| "normaliser_current": current, | |
| "artifact_stale": stamped != current, | |
| "chunks": len(chunks), | |
| "mentions": len(mentions), | |
| "distinct_surfaces": len(set(surfaces)), | |
| "span_cap_hits": capped, | |
| "threshold_override": args.threshold, | |
| "score": score.as_dict(), | |
| "gold": str(GOLD), | |
| "compared_against": "PyMuPDF 0.8537 (35/41), " | |
| "eval/knowledge/results/baseline_prototype_2026-08-13_145132.json", | |
| } | |
| out.write_text(json.dumps(payload, indent=2, default=str), encoding="utf-8") | |
| print(f"\nwritten : {out}") | |
| return 0 | |
| if __name__ == "__main__": | |
| raise SystemExit(main()) | |