ResearchOS / scripts /calibrate_gate.py
Kagzy's picture
v2 backend: figures, math prompts, live OpenAlex, 275k corpus via private dataset
e5c059f verified
Raw History Blame Contribute Delete
7.27 kB
"""Measure the confidence gate, and whether retrieval still finds what it used to.
python scripts/calibrate_gate.py --config configs/ml.yaml --tag v1
The gate (`answer.min_mean_score`) decides when ResearchOS says "nothing in
here answers that". Its threshold was set by eye from a handful of queries;
a corpus 35x larger changes what "confident" means - more papers means more
near-misses for any query, which pushes every score distribution up. So this
measures it, on three query groups:
specific 163 questions written from corpus papers (data/generated_eval.jsonl)
vague 250 half-remembered descriptions of corpus papers (the rescue eval's)
off-topic questions this tool should decline
and reports each group's mean-of-top-5 distribution, the AUC separating
answerable from off-topic, the threshold that maximises Youden's J, and
recall@10 on both gold-labelled groups - the regression check that adding
papers did not bury the ones users could already find.
Live sources and diagnosis are off: deterministic, and costs nothing.
"""
from __future__ import annotations
import argparse
import json
import sys
from pathlib import Path
sys.path.insert(0, str(Path(__file__).resolve().parents[1]))
if hasattr(sys.stdout, "reconfigure"):
sys.stdout.reconfigure(encoding="utf-8", errors="replace")
from src.config import load_config # noqa: E402
OFF_TOPIC = [
"how long should sourdough bread proof in the fridge",
"best exercises for lower back pain",
"what caused the fall of the roman empire",
"how to file taxes as a freelancer in the uk",
"symptoms of vitamin d deficiency in adults",
"how do tides work",
"recipe for authentic pad thai",
"how to grow tomatoes in containers",
"who won the 1966 football world cup",
"how does a mortgage refinance work",
"what is the plot of pride and prejudice",
"how to remove red wine stains from carpet",
"treatment for a sprained ankle",
"history of the byzantine empire",
"how to train a puppy to sit",
"what is the capital of mongolia",
"how do vaccines trigger immunity",
"best time to visit kyoto",
"how to change a car tyre",
"what is impressionism in painting",
"how does compound interest work",
"causes of the first world war",
"how to brew pour over coffee",
"symptoms of seasonal allergies",
"how do volcanoes form",
"what is a sonnet",
"how to negotiate a salary offer",
"how does photosynthesis work in plants",
"rules of cricket for beginners",
"how to knit a scarf",
"what is the keto diet",
"how do solar panels work at home",
"history of jazz music",
"how to care for succulents",
"what is inflation and why does it happen",
"how to write a cover letter",
"migration patterns of monarch butterflies",
"how to make cold brew coffee",
"what is stoic philosophy",
"how to repair a leaking tap",
]
def stats(xs: list[float]) -> dict[str, float]:
xs = sorted(xs)
pick = lambda q: xs[min(len(xs) - 1, int(q * (len(xs) - 1)))] # noqa: E731
return {"n": len(xs), "min": round(xs[0], 4), "p05": round(pick(0.05), 4),
"median": round(pick(0.5), 4), "p95": round(pick(0.95), 4), "max": round(xs[-1], 4)}
def auc(pos: list[float], neg: list[float]) -> float:
wins = sum((p > n) + 0.5 * (p == n) for p in pos for n in neg)
return wins / (len(pos) * len(neg))
def best_threshold(pos: list[float], neg: list[float]) -> tuple[float, float]:
"""Threshold maximising Youden's J = TPR - FPR (answerable = positive)."""
best = (0.0, -1.0)
for t in sorted(set(pos + neg)):
j = sum(p >= t for p in pos) / len(pos) - sum(n >= t for n in neg) / len(neg)
if j > best[1]:
best = (t, j)
return round(best[0], 4), round(best[1], 4)
def main() -> None:
ap = argparse.ArgumentParser(description=__doc__)
ap.add_argument("--config", default="configs/ml.yaml")
ap.add_argument("--tag", required=True)
ap.add_argument("--out-dir", default="results")
args = ap.parse_args()
cfg = load_config(args.config)
cfg["citation"]["allow_remote"] = False
cfg.setdefault("diagnosis", {})["enabled"] = False
from src.api.ml_pipeline import MLPipeline
pipe = MLPipeline(cfg)
threshold = cfg["answer"]["min_mean_score"]
specific = [json.loads(line) for line in open("data/generated_eval.jsonl", encoding="utf-8")]
rescue = json.loads(Path("results/v2_rescue.cache.json").read_text(encoding="utf-8"))["queries"]
vague = [{"query": q, "gold_paper_id": pid} for pid, q in rescue.items()]
def run(items: list[dict]) -> tuple[list[float], float | None]:
scores, hits = [], 0
for it in items:
r = pipe.find_papers(it["query"], top_k=10)
scores.append(r["mean_top_score"])
gold = it.get("gold_paper_id")
hits += bool(gold and any(p["paper_id"] == gold for p in r["papers"]))
has_gold = any(it.get("gold_paper_id") for it in items)
return scores, (round(hits / len(items), 4) if has_gold else None)
groups = {}
for name, items in (("specific", specific), ("vague", vague),
("off_topic", [{"query": q} for q in OFF_TOPIC])):
scores, recall = run(items)
groups[name] = {"scores": stats(scores), "recall_at_10": recall,
"pass_gate": round(sum(s >= threshold for s in scores) / len(scores), 4),
"_raw": scores}
print(f"[gate] {name:<9} {groups[name]['scores']} recall@10={recall} "
f"pass@{threshold}={groups[name]['pass_gate']:.1%}", flush=True)
answerable = groups["specific"]["_raw"] + groups["vague"]["_raw"]
off = groups["off_topic"]["_raw"]
t, j = best_threshold(answerable, off)
summary = {
"tag": args.tag, "config": args.config,
"corpus_papers": len(pipe.papers_by_id), "corpus_passages": len(pipe.corpus),
"current_threshold": threshold,
"auc_answerable_vs_off_topic": round(auc(answerable, off), 4),
"auc_specific_vs_off_topic": round(auc(groups["specific"]["_raw"], off), 4),
"youden_threshold": t, "youden_j": j,
# The whole trade-off, not one point: share of each group passing.
"pass_rate_by_threshold": {
f"{th:.2f}": {k: round(sum(x >= th for x in v["_raw"]) / len(v["_raw"]), 4)
for k, v in groups.items()}
for th in (0.05, 0.08, 0.10, 0.12, 0.15, 0.20, 0.25, 0.30, 0.35)
},
"raw_scores": {k: [round(x, 4) for x in v["_raw"]] for k, v in groups.items()},
"groups": {k: {kk: vv for kk, vv in v.items() if kk != "_raw"} for k, v in groups.items()},
}
out = Path(args.out_dir) / f"gate_calibration_{args.tag}.json"
out.write_text(json.dumps(summary, indent=2), encoding="utf-8")
print(json.dumps({k: v for k, v in summary.items() if k not in ("groups", "raw_scores")},
indent=2))
print(f"[gate] saved -> {out}")
if __name__ == "__main__":
main()