Kagzy commited on
Commit
ea0fb4a
·
verified ·
1 Parent(s): 273dddc

calibrated confidence gate; indexed title lookup

Browse files
Files changed (3) hide show
  1. configs/ml.cpu.yaml +9 -2
  2. configs/ml.yaml +9 -2
  3. src/data/citation.py +68 -7
configs/ml.cpu.yaml CHANGED
@@ -79,9 +79,16 @@ answer:
79
  # hits 0.67 against a bone-densitometry paper) while the mean stays at 0.19.
80
  # Scale is [0,1] because CrossEncoder.predict applies a sigmoid. The ONNX
81
  # backend returns raw logits instead - rescale if you switch.
82
- # Measured: in-corpus mean5 0.67-0.996, out-of-corpus 0.001-0.19.
 
 
 
 
 
 
 
83
  confidence_top_n: 5
84
- min_mean_score: 0.35
85
  # Below this (but above min_mean_score) the corpus has *something* but not
86
  # much, so live arXiv results are added alongside it and the answer draws on
87
  # both. Above it, a strong corpus hit answers without touching the network.
 
79
  # hits 0.67 against a bone-densitometry paper) while the mean stays at 0.19.
80
  # Scale is [0,1] because CrossEncoder.predict applies a sigmoid. The ONNX
81
  # backend returns raw logits instead - rescale if you switch.
82
+ # Measured, not eyeballed: scripts/calibrate_gate.py (results/gate_calibration_*.json).
83
+ # The old 0.35 came from a few hand-picked queries ("in-corpus 0.67-0.996") and
84
+ # refused 49% of specific questions whose answer paper WAS retrieved
85
+ # (recall@10 = 100%). Pass rates, specific / vague / off-topic:
86
+ # 0.35 -> 51% / 31% / 2.5% 0.20 -> 94% / 59% / 20%
87
+ # A wrongly refused good question costs more than an off-topic one let
88
+ # through: the answer prompt still says plainly when passages do not
89
+ # cover the question. Recalibrate whenever the corpus changes size.
90
  confidence_top_n: 5
91
+ min_mean_score: 0.20
92
  # Below this (but above min_mean_score) the corpus has *something* but not
93
  # much, so live arXiv results are added alongside it and the answer draws on
94
  # both. Above it, a strong corpus hit answers without touching the network.
configs/ml.yaml CHANGED
@@ -69,9 +69,16 @@ answer:
69
  # hits 0.67 against a bone-densitometry paper) while the mean stays at 0.19.
70
  # Scale is [0,1] because CrossEncoder.predict applies a sigmoid. The ONNX
71
  # backend returns raw logits instead - rescale if you switch.
72
- # Measured: in-corpus mean5 0.67-0.996, out-of-corpus 0.001-0.19.
 
 
 
 
 
 
 
73
  confidence_top_n: 5
74
- min_mean_score: 0.35
75
  # Below this (but above min_mean_score) the corpus has *something* but not
76
  # much, so live arXiv results are added alongside it and the answer draws on
77
  # both. Above it, a strong corpus hit answers without touching the network.
 
69
  # hits 0.67 against a bone-densitometry paper) while the mean stays at 0.19.
70
  # Scale is [0,1] because CrossEncoder.predict applies a sigmoid. The ONNX
71
  # backend returns raw logits instead - rescale if you switch.
72
+ # Measured, not eyeballed: scripts/calibrate_gate.py (results/gate_calibration_*.json).
73
+ # The old 0.35 came from a few hand-picked queries ("in-corpus 0.67-0.996") and
74
+ # refused 49% of specific questions whose answer paper WAS retrieved
75
+ # (recall@10 = 100%). Pass rates, specific / vague / off-topic:
76
+ # 0.35 -> 51% / 31% / 2.5% 0.20 -> 94% / 59% / 20%
77
+ # A wrongly refused good question costs more than an off-topic one let
78
+ # through: the answer prompt still says plainly when passages do not
79
+ # cover the question. Recalibrate whenever the corpus changes size.
80
  confidence_top_n: 5
81
+ min_mean_score: 0.20
82
  # Below this (but above min_mean_score) the corpus has *something* but not
83
  # much, so live arXiv results are added alongside it and the answer draws on
84
  # both. Above it, a strong corpus hit answers without touching the network.
src/data/citation.py CHANGED
@@ -252,17 +252,78 @@ def _weighted_coverage(query: str, title: str, idf: dict[str, float], default_id
252
  return (matched / total) * length_penalty
253
 
254
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
255
  def find_in_corpus(
256
  query: str, papers_by_id: dict[str, dict[str, Any]], threshold: float = 0.72
257
  ) -> tuple[dict[str, Any] | None, list[dict[str, str]]]:
258
  """Best local title match, plus runners-up for disambiguation."""
259
- idf = _build_idf(papers_by_id)
260
- default_idf = max(idf.values(), default=1.0)
261
- scored = [
262
- (_weighted_coverage(query, meta.get("title", ""), idf, default_idf), meta)
263
- for meta in papers_by_id.values()
264
- ]
265
- scored.sort(key=lambda kv: kv[0], reverse=True)
266
  candidates = [
267
  {"paper_id": m.get("paper_id", ""), "title": m.get("title", ""), "score": f"{s:.2f}"}
268
  for s, m in scored[:5]
 
252
  return (matched / total) * length_penalty
253
 
254
 
255
+ class TitleIndex:
256
+ """Inverted index over corpus titles, built once per corpus.
257
+
258
+ find_in_corpus used to rebuild the IDF table and score every title on
259
+ every lookup: fine at 8k papers, seconds per lookup at 300k. This keeps
260
+ the scoring identical and only changes which titles get scored.
261
+
262
+ Pruning is exact, not heuristic. A title that contains none of a set S of
263
+ query tokens can match at most (total - w(S)) of the query's IDF weight,
264
+ so if w(S) >= (1 - floor) * total it cannot score above `floor`. Taking
265
+ S as the rarest query tokens until that holds, the postings of S contain
266
+ every title that could clear the floor - the same results as a full scan.
267
+ """
268
+
269
+ def __init__(self, papers_by_id: dict[str, dict[str, Any]]) -> None:
270
+ import math
271
+
272
+ self.papers = papers_by_id
273
+ self.tokens: dict[str, set[str]] = {}
274
+ postings: dict[str, list[str]] = {}
275
+ for pid, meta in papers_by_id.items():
276
+ toks = _tokens(meta.get("title", ""))
277
+ self.tokens[pid] = toks
278
+ for t in toks:
279
+ postings.setdefault(t, []).append(pid)
280
+ self.postings = postings
281
+ n = max(1, len(papers_by_id))
282
+ self.idf = {t: math.log(n / (1 + len(ids))) for t, ids in postings.items()}
283
+ self.default_idf = max(self.idf.values(), default=1.0)
284
+
285
+ def search(self, query: str, floor: float = 0.2) -> list[tuple[float, dict[str, Any]]]:
286
+ qt = _tokens(query)
287
+ if not qt:
288
+ return []
289
+ weight = {t: self.idf.get(t, self.default_idf) for t in qt}
290
+ total = sum(weight.values())
291
+ needed, covered, pivots = (1.0 - floor) * total, 0.0, []
292
+ for t in sorted(qt, key=weight.get, reverse=True):
293
+ pivots.append(t)
294
+ covered += weight[t]
295
+ if covered >= needed:
296
+ break
297
+ candidates = {pid for t in pivots for pid in self.postings.get(t, ())}
298
+ scored = []
299
+ for pid in candidates:
300
+ tt = self.tokens[pid]
301
+ matched = sum(weight[t] for t in qt & tt)
302
+ score = (matched / total) * min(1.0, len(qt) / max(1, len(tt))) ** 0.5
303
+ scored.append((score, self.papers[pid]))
304
+ scored.sort(key=lambda kv: kv[0], reverse=True)
305
+ return scored
306
+
307
+
308
+ _TITLE_INDEX: dict[int, tuple[int, TitleIndex]] = {}
309
+
310
+
311
+ def _title_index(papers_by_id: dict[str, dict[str, Any]]) -> TitleIndex:
312
+ """One index per corpus dict, rebuilt only if the corpus changes size."""
313
+ key = id(papers_by_id)
314
+ cached = _TITLE_INDEX.get(key)
315
+ if cached is None or cached[0] != len(papers_by_id):
316
+ _TITLE_INDEX.clear() # one live corpus per process
317
+ cached = (len(papers_by_id), TitleIndex(papers_by_id))
318
+ _TITLE_INDEX[key] = cached
319
+ return cached[1]
320
+
321
+
322
  def find_in_corpus(
323
  query: str, papers_by_id: dict[str, dict[str, Any]], threshold: float = 0.72
324
  ) -> tuple[dict[str, Any] | None, list[dict[str, str]]]:
325
  """Best local title match, plus runners-up for disambiguation."""
326
+ scored = _title_index(papers_by_id).search(query, floor=0.2)
 
 
 
 
 
 
327
  candidates = [
328
  {"paper_id": m.get("paper_id", ""), "title": m.get("title", ""), "score": f"{s:.2f}"}
329
  for s, m in scored[:5]