Spaces:
Running on Zero
Running on Zero
File size: 7,268 Bytes
e5c059f | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 151 152 153 154 155 156 157 158 159 160 161 162 163 164 165 166 167 168 | """Measure the confidence gate, and whether retrieval still finds what it used to.
python scripts/calibrate_gate.py --config configs/ml.yaml --tag v1
The gate (`answer.min_mean_score`) decides when ResearchOS says "nothing in
here answers that". Its threshold was set by eye from a handful of queries;
a corpus 35x larger changes what "confident" means - more papers means more
near-misses for any query, which pushes every score distribution up. So this
measures it, on three query groups:
specific 163 questions written from corpus papers (data/generated_eval.jsonl)
vague 250 half-remembered descriptions of corpus papers (the rescue eval's)
off-topic questions this tool should decline
and reports each group's mean-of-top-5 distribution, the AUC separating
answerable from off-topic, the threshold that maximises Youden's J, and
recall@10 on both gold-labelled groups - the regression check that adding
papers did not bury the ones users could already find.
Live sources and diagnosis are off: deterministic, and costs nothing.
"""
from __future__ import annotations
import argparse
import json
import sys
from pathlib import Path
sys.path.insert(0, str(Path(__file__).resolve().parents[1]))
if hasattr(sys.stdout, "reconfigure"):
sys.stdout.reconfigure(encoding="utf-8", errors="replace")
from src.config import load_config # noqa: E402
OFF_TOPIC = [
"how long should sourdough bread proof in the fridge",
"best exercises for lower back pain",
"what caused the fall of the roman empire",
"how to file taxes as a freelancer in the uk",
"symptoms of vitamin d deficiency in adults",
"how do tides work",
"recipe for authentic pad thai",
"how to grow tomatoes in containers",
"who won the 1966 football world cup",
"how does a mortgage refinance work",
"what is the plot of pride and prejudice",
"how to remove red wine stains from carpet",
"treatment for a sprained ankle",
"history of the byzantine empire",
"how to train a puppy to sit",
"what is the capital of mongolia",
"how do vaccines trigger immunity",
"best time to visit kyoto",
"how to change a car tyre",
"what is impressionism in painting",
"how does compound interest work",
"causes of the first world war",
"how to brew pour over coffee",
"symptoms of seasonal allergies",
"how do volcanoes form",
"what is a sonnet",
"how to negotiate a salary offer",
"how does photosynthesis work in plants",
"rules of cricket for beginners",
"how to knit a scarf",
"what is the keto diet",
"how do solar panels work at home",
"history of jazz music",
"how to care for succulents",
"what is inflation and why does it happen",
"how to write a cover letter",
"migration patterns of monarch butterflies",
"how to make cold brew coffee",
"what is stoic philosophy",
"how to repair a leaking tap",
]
def stats(xs: list[float]) -> dict[str, float]:
xs = sorted(xs)
pick = lambda q: xs[min(len(xs) - 1, int(q * (len(xs) - 1)))] # noqa: E731
return {"n": len(xs), "min": round(xs[0], 4), "p05": round(pick(0.05), 4),
"median": round(pick(0.5), 4), "p95": round(pick(0.95), 4), "max": round(xs[-1], 4)}
def auc(pos: list[float], neg: list[float]) -> float:
wins = sum((p > n) + 0.5 * (p == n) for p in pos for n in neg)
return wins / (len(pos) * len(neg))
def best_threshold(pos: list[float], neg: list[float]) -> tuple[float, float]:
"""Threshold maximising Youden's J = TPR - FPR (answerable = positive)."""
best = (0.0, -1.0)
for t in sorted(set(pos + neg)):
j = sum(p >= t for p in pos) / len(pos) - sum(n >= t for n in neg) / len(neg)
if j > best[1]:
best = (t, j)
return round(best[0], 4), round(best[1], 4)
def main() -> None:
ap = argparse.ArgumentParser(description=__doc__)
ap.add_argument("--config", default="configs/ml.yaml")
ap.add_argument("--tag", required=True)
ap.add_argument("--out-dir", default="results")
args = ap.parse_args()
cfg = load_config(args.config)
cfg["citation"]["allow_remote"] = False
cfg.setdefault("diagnosis", {})["enabled"] = False
from src.api.ml_pipeline import MLPipeline
pipe = MLPipeline(cfg)
threshold = cfg["answer"]["min_mean_score"]
specific = [json.loads(line) for line in open("data/generated_eval.jsonl", encoding="utf-8")]
rescue = json.loads(Path("results/v2_rescue.cache.json").read_text(encoding="utf-8"))["queries"]
vague = [{"query": q, "gold_paper_id": pid} for pid, q in rescue.items()]
def run(items: list[dict]) -> tuple[list[float], float | None]:
scores, hits = [], 0
for it in items:
r = pipe.find_papers(it["query"], top_k=10)
scores.append(r["mean_top_score"])
gold = it.get("gold_paper_id")
hits += bool(gold and any(p["paper_id"] == gold for p in r["papers"]))
has_gold = any(it.get("gold_paper_id") for it in items)
return scores, (round(hits / len(items), 4) if has_gold else None)
groups = {}
for name, items in (("specific", specific), ("vague", vague),
("off_topic", [{"query": q} for q in OFF_TOPIC])):
scores, recall = run(items)
groups[name] = {"scores": stats(scores), "recall_at_10": recall,
"pass_gate": round(sum(s >= threshold for s in scores) / len(scores), 4),
"_raw": scores}
print(f"[gate] {name:<9} {groups[name]['scores']} recall@10={recall} "
f"pass@{threshold}={groups[name]['pass_gate']:.1%}", flush=True)
answerable = groups["specific"]["_raw"] + groups["vague"]["_raw"]
off = groups["off_topic"]["_raw"]
t, j = best_threshold(answerable, off)
summary = {
"tag": args.tag, "config": args.config,
"corpus_papers": len(pipe.papers_by_id), "corpus_passages": len(pipe.corpus),
"current_threshold": threshold,
"auc_answerable_vs_off_topic": round(auc(answerable, off), 4),
"auc_specific_vs_off_topic": round(auc(groups["specific"]["_raw"], off), 4),
"youden_threshold": t, "youden_j": j,
# The whole trade-off, not one point: share of each group passing.
"pass_rate_by_threshold": {
f"{th:.2f}": {k: round(sum(x >= th for x in v["_raw"]) / len(v["_raw"]), 4)
for k, v in groups.items()}
for th in (0.05, 0.08, 0.10, 0.12, 0.15, 0.20, 0.25, 0.30, 0.35)
},
"raw_scores": {k: [round(x, 4) for x in v["_raw"]] for k, v in groups.items()},
"groups": {k: {kk: vv for kk, vv in v.items() if kk != "_raw"} for k, v in groups.items()},
}
out = Path(args.out_dir) / f"gate_calibration_{args.tag}.json"
out.write_text(json.dumps(summary, indent=2), encoding="utf-8")
print(json.dumps({k: v for k, v in summary.items() if k not in ("groups", "raw_scores")},
indent=2))
print(f"[gate] saved -> {out}")
if __name__ == "__main__":
main()
|