File size: 7,268 Bytes
e5c059f
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
"""Measure the confidence gate, and whether retrieval still finds what it used to.



    python scripts/calibrate_gate.py --config configs/ml.yaml --tag v1



The gate (`answer.min_mean_score`) decides when ResearchOS says "nothing in

here answers that". Its threshold was set by eye from a handful of queries;

a corpus 35x larger changes what "confident" means - more papers means more

near-misses for any query, which pushes every score distribution up. So this

measures it, on three query groups:



  specific   163 questions written from corpus papers (data/generated_eval.jsonl)

  vague      250 half-remembered descriptions of corpus papers (the rescue eval's)

  off-topic  questions this tool should decline



and reports each group's mean-of-top-5 distribution, the AUC separating

answerable from off-topic, the threshold that maximises Youden's J, and

recall@10 on both gold-labelled groups - the regression check that adding

papers did not bury the ones users could already find.



Live sources and diagnosis are off: deterministic, and costs nothing.

"""

from __future__ import annotations

import argparse
import json
import sys
from pathlib import Path

sys.path.insert(0, str(Path(__file__).resolve().parents[1]))
if hasattr(sys.stdout, "reconfigure"):
    sys.stdout.reconfigure(encoding="utf-8", errors="replace")

from src.config import load_config  # noqa: E402

OFF_TOPIC = [
    "how long should sourdough bread proof in the fridge",
    "best exercises for lower back pain",
    "what caused the fall of the roman empire",
    "how to file taxes as a freelancer in the uk",
    "symptoms of vitamin d deficiency in adults",
    "how do tides work",
    "recipe for authentic pad thai",
    "how to grow tomatoes in containers",
    "who won the 1966 football world cup",
    "how does a mortgage refinance work",
    "what is the plot of pride and prejudice",
    "how to remove red wine stains from carpet",
    "treatment for a sprained ankle",
    "history of the byzantine empire",
    "how to train a puppy to sit",
    "what is the capital of mongolia",
    "how do vaccines trigger immunity",
    "best time to visit kyoto",
    "how to change a car tyre",
    "what is impressionism in painting",
    "how does compound interest work",
    "causes of the first world war",
    "how to brew pour over coffee",
    "symptoms of seasonal allergies",
    "how do volcanoes form",
    "what is a sonnet",
    "how to negotiate a salary offer",
    "how does photosynthesis work in plants",
    "rules of cricket for beginners",
    "how to knit a scarf",
    "what is the keto diet",
    "how do solar panels work at home",
    "history of jazz music",
    "how to care for succulents",
    "what is inflation and why does it happen",
    "how to write a cover letter",
    "migration patterns of monarch butterflies",
    "how to make cold brew coffee",
    "what is stoic philosophy",
    "how to repair a leaking tap",
]


def stats(xs: list[float]) -> dict[str, float]:
    xs = sorted(xs)
    pick = lambda q: xs[min(len(xs) - 1, int(q * (len(xs) - 1)))]  # noqa: E731
    return {"n": len(xs), "min": round(xs[0], 4), "p05": round(pick(0.05), 4),
            "median": round(pick(0.5), 4), "p95": round(pick(0.95), 4), "max": round(xs[-1], 4)}


def auc(pos: list[float], neg: list[float]) -> float:
    wins = sum((p > n) + 0.5 * (p == n) for p in pos for n in neg)
    return wins / (len(pos) * len(neg))


def best_threshold(pos: list[float], neg: list[float]) -> tuple[float, float]:
    """Threshold maximising Youden's J = TPR - FPR (answerable = positive)."""
    best = (0.0, -1.0)
    for t in sorted(set(pos + neg)):
        j = sum(p >= t for p in pos) / len(pos) - sum(n >= t for n in neg) / len(neg)
        if j > best[1]:
            best = (t, j)
    return round(best[0], 4), round(best[1], 4)


def main() -> None:
    ap = argparse.ArgumentParser(description=__doc__)
    ap.add_argument("--config", default="configs/ml.yaml")
    ap.add_argument("--tag", required=True)
    ap.add_argument("--out-dir", default="results")
    args = ap.parse_args()

    cfg = load_config(args.config)
    cfg["citation"]["allow_remote"] = False
    cfg.setdefault("diagnosis", {})["enabled"] = False
    from src.api.ml_pipeline import MLPipeline
    pipe = MLPipeline(cfg)
    threshold = cfg["answer"]["min_mean_score"]

    specific = [json.loads(line) for line in open("data/generated_eval.jsonl", encoding="utf-8")]
    rescue = json.loads(Path("results/v2_rescue.cache.json").read_text(encoding="utf-8"))["queries"]
    vague = [{"query": q, "gold_paper_id": pid} for pid, q in rescue.items()]

    def run(items: list[dict]) -> tuple[list[float], float | None]:
        scores, hits = [], 0
        for it in items:
            r = pipe.find_papers(it["query"], top_k=10)
            scores.append(r["mean_top_score"])
            gold = it.get("gold_paper_id")
            hits += bool(gold and any(p["paper_id"] == gold for p in r["papers"]))
        has_gold = any(it.get("gold_paper_id") for it in items)
        return scores, (round(hits / len(items), 4) if has_gold else None)

    groups = {}
    for name, items in (("specific", specific), ("vague", vague),
                        ("off_topic", [{"query": q} for q in OFF_TOPIC])):
        scores, recall = run(items)
        groups[name] = {"scores": stats(scores), "recall_at_10": recall,
                        "pass_gate": round(sum(s >= threshold for s in scores) / len(scores), 4),
                        "_raw": scores}
        print(f"[gate] {name:<9} {groups[name]['scores']}  recall@10={recall}  "
              f"pass@{threshold}={groups[name]['pass_gate']:.1%}", flush=True)

    answerable = groups["specific"]["_raw"] + groups["vague"]["_raw"]
    off = groups["off_topic"]["_raw"]
    t, j = best_threshold(answerable, off)
    summary = {
        "tag": args.tag, "config": args.config,
        "corpus_papers": len(pipe.papers_by_id), "corpus_passages": len(pipe.corpus),
        "current_threshold": threshold,
        "auc_answerable_vs_off_topic": round(auc(answerable, off), 4),
        "auc_specific_vs_off_topic": round(auc(groups["specific"]["_raw"], off), 4),
        "youden_threshold": t, "youden_j": j,
        # The whole trade-off, not one point: share of each group passing.
        "pass_rate_by_threshold": {
            f"{th:.2f}": {k: round(sum(x >= th for x in v["_raw"]) / len(v["_raw"]), 4)
                          for k, v in groups.items()}
            for th in (0.05, 0.08, 0.10, 0.12, 0.15, 0.20, 0.25, 0.30, 0.35)
        },
        "raw_scores": {k: [round(x, 4) for x in v["_raw"]] for k, v in groups.items()},
        "groups": {k: {kk: vv for kk, vv in v.items() if kk != "_raw"} for k, v in groups.items()},
    }
    out = Path(args.out_dir) / f"gate_calibration_{args.tag}.json"
    out.write_text(json.dumps(summary, indent=2), encoding="utf-8")
    print(json.dumps({k: v for k, v in summary.items() if k not in ("groups", "raw_scores")},
                     indent=2))
    print(f"[gate] saved -> {out}")


if __name__ == "__main__":
    main()