Spaces:
Running on Zero
Running on Zero
Download scripts/calibrate_gate.py from Kagzy/ResearchOS: direct link, hf CLI and curl.
- Browser
- Download file 7.27 kB
-
https://huggingface.co/spaces/Kagzy/ResearchOS/resolve/main/scripts/calibrate_gate.py
- Command line
-
hf download hf://spaces/Kagzy/ResearchOS/scripts/calibrate_gate.py
-
curl -L -o calibrate_gate.py https://huggingface.co/spaces/Kagzy/ResearchOS/resolve/main/scripts/calibrate_gate.py
7.27 kB
| """Measure the confidence gate, and whether retrieval still finds what it used to. | |
| python scripts/calibrate_gate.py --config configs/ml.yaml --tag v1 | |
| The gate (`answer.min_mean_score`) decides when ResearchOS says "nothing in | |
| here answers that". Its threshold was set by eye from a handful of queries; | |
| a corpus 35x larger changes what "confident" means - more papers means more | |
| near-misses for any query, which pushes every score distribution up. So this | |
| measures it, on three query groups: | |
| specific 163 questions written from corpus papers (data/generated_eval.jsonl) | |
| vague 250 half-remembered descriptions of corpus papers (the rescue eval's) | |
| off-topic questions this tool should decline | |
| and reports each group's mean-of-top-5 distribution, the AUC separating | |
| answerable from off-topic, the threshold that maximises Youden's J, and | |
| recall@10 on both gold-labelled groups - the regression check that adding | |
| papers did not bury the ones users could already find. | |
| Live sources and diagnosis are off: deterministic, and costs nothing. | |
| """ | |
| from __future__ import annotations | |
| import argparse | |
| import json | |
| import sys | |
| from pathlib import Path | |
| sys.path.insert(0, str(Path(__file__).resolve().parents[1])) | |
| if hasattr(sys.stdout, "reconfigure"): | |
| sys.stdout.reconfigure(encoding="utf-8", errors="replace") | |
| from src.config import load_config # noqa: E402 | |
| OFF_TOPIC = [ | |
| "how long should sourdough bread proof in the fridge", | |
| "best exercises for lower back pain", | |
| "what caused the fall of the roman empire", | |
| "how to file taxes as a freelancer in the uk", | |
| "symptoms of vitamin d deficiency in adults", | |
| "how do tides work", | |
| "recipe for authentic pad thai", | |
| "how to grow tomatoes in containers", | |
| "who won the 1966 football world cup", | |
| "how does a mortgage refinance work", | |
| "what is the plot of pride and prejudice", | |
| "how to remove red wine stains from carpet", | |
| "treatment for a sprained ankle", | |
| "history of the byzantine empire", | |
| "how to train a puppy to sit", | |
| "what is the capital of mongolia", | |
| "how do vaccines trigger immunity", | |
| "best time to visit kyoto", | |
| "how to change a car tyre", | |
| "what is impressionism in painting", | |
| "how does compound interest work", | |
| "causes of the first world war", | |
| "how to brew pour over coffee", | |
| "symptoms of seasonal allergies", | |
| "how do volcanoes form", | |
| "what is a sonnet", | |
| "how to negotiate a salary offer", | |
| "how does photosynthesis work in plants", | |
| "rules of cricket for beginners", | |
| "how to knit a scarf", | |
| "what is the keto diet", | |
| "how do solar panels work at home", | |
| "history of jazz music", | |
| "how to care for succulents", | |
| "what is inflation and why does it happen", | |
| "how to write a cover letter", | |
| "migration patterns of monarch butterflies", | |
| "how to make cold brew coffee", | |
| "what is stoic philosophy", | |
| "how to repair a leaking tap", | |
| ] | |
| def stats(xs: list[float]) -> dict[str, float]: | |
| xs = sorted(xs) | |
| pick = lambda q: xs[min(len(xs) - 1, int(q * (len(xs) - 1)))] # noqa: E731 | |
| return {"n": len(xs), "min": round(xs[0], 4), "p05": round(pick(0.05), 4), | |
| "median": round(pick(0.5), 4), "p95": round(pick(0.95), 4), "max": round(xs[-1], 4)} | |
| def auc(pos: list[float], neg: list[float]) -> float: | |
| wins = sum((p > n) + 0.5 * (p == n) for p in pos for n in neg) | |
| return wins / (len(pos) * len(neg)) | |
| def best_threshold(pos: list[float], neg: list[float]) -> tuple[float, float]: | |
| """Threshold maximising Youden's J = TPR - FPR (answerable = positive).""" | |
| best = (0.0, -1.0) | |
| for t in sorted(set(pos + neg)): | |
| j = sum(p >= t for p in pos) / len(pos) - sum(n >= t for n in neg) / len(neg) | |
| if j > best[1]: | |
| best = (t, j) | |
| return round(best[0], 4), round(best[1], 4) | |
| def main() -> None: | |
| ap = argparse.ArgumentParser(description=__doc__) | |
| ap.add_argument("--config", default="configs/ml.yaml") | |
| ap.add_argument("--tag", required=True) | |
| ap.add_argument("--out-dir", default="results") | |
| args = ap.parse_args() | |
| cfg = load_config(args.config) | |
| cfg["citation"]["allow_remote"] = False | |
| cfg.setdefault("diagnosis", {})["enabled"] = False | |
| from src.api.ml_pipeline import MLPipeline | |
| pipe = MLPipeline(cfg) | |
| threshold = cfg["answer"]["min_mean_score"] | |
| specific = [json.loads(line) for line in open("data/generated_eval.jsonl", encoding="utf-8")] | |
| rescue = json.loads(Path("results/v2_rescue.cache.json").read_text(encoding="utf-8"))["queries"] | |
| vague = [{"query": q, "gold_paper_id": pid} for pid, q in rescue.items()] | |
| def run(items: list[dict]) -> tuple[list[float], float | None]: | |
| scores, hits = [], 0 | |
| for it in items: | |
| r = pipe.find_papers(it["query"], top_k=10) | |
| scores.append(r["mean_top_score"]) | |
| gold = it.get("gold_paper_id") | |
| hits += bool(gold and any(p["paper_id"] == gold for p in r["papers"])) | |
| has_gold = any(it.get("gold_paper_id") for it in items) | |
| return scores, (round(hits / len(items), 4) if has_gold else None) | |
| groups = {} | |
| for name, items in (("specific", specific), ("vague", vague), | |
| ("off_topic", [{"query": q} for q in OFF_TOPIC])): | |
| scores, recall = run(items) | |
| groups[name] = {"scores": stats(scores), "recall_at_10": recall, | |
| "pass_gate": round(sum(s >= threshold for s in scores) / len(scores), 4), | |
| "_raw": scores} | |
| print(f"[gate] {name:<9} {groups[name]['scores']} recall@10={recall} " | |
| f"pass@{threshold}={groups[name]['pass_gate']:.1%}", flush=True) | |
| answerable = groups["specific"]["_raw"] + groups["vague"]["_raw"] | |
| off = groups["off_topic"]["_raw"] | |
| t, j = best_threshold(answerable, off) | |
| summary = { | |
| "tag": args.tag, "config": args.config, | |
| "corpus_papers": len(pipe.papers_by_id), "corpus_passages": len(pipe.corpus), | |
| "current_threshold": threshold, | |
| "auc_answerable_vs_off_topic": round(auc(answerable, off), 4), | |
| "auc_specific_vs_off_topic": round(auc(groups["specific"]["_raw"], off), 4), | |
| "youden_threshold": t, "youden_j": j, | |
| # The whole trade-off, not one point: share of each group passing. | |
| "pass_rate_by_threshold": { | |
| f"{th:.2f}": {k: round(sum(x >= th for x in v["_raw"]) / len(v["_raw"]), 4) | |
| for k, v in groups.items()} | |
| for th in (0.05, 0.08, 0.10, 0.12, 0.15, 0.20, 0.25, 0.30, 0.35) | |
| }, | |
| "raw_scores": {k: [round(x, 4) for x in v["_raw"]] for k, v in groups.items()}, | |
| "groups": {k: {kk: vv for kk, vv in v.items() if kk != "_raw"} for k, v in groups.items()}, | |
| } | |
| out = Path(args.out_dir) / f"gate_calibration_{args.tag}.json" | |
| out.write_text(json.dumps(summary, indent=2), encoding="utf-8") | |
| print(json.dumps({k: v for k, v in summary.items() if k not in ("groups", "raw_scores")}, | |
| indent=2)) | |
| print(f"[gate] saved -> {out}") | |
| if __name__ == "__main__": | |
| main() | |