Spaces:
Running on Zero
Running on Zero
File size: 43,454 Bytes
e8c5cc2 2299777 e8c5cc2 5eec49d 2299777 e8c5cc2 af38999 a3c7ac6 e8c5cc2 a3c7ac6 e8c5cc2 a3c7ac6 2299777 a3c7ac6 e8c5cc2 2299777 e8c5cc2 2299777 e8c5cc2 2299777 e8c5cc2 2299777 e8c5cc2 2299777 e8c5cc2 2299777 e8c5cc2 2299777 e8c5cc2 2299777 e8c5cc2 a3c7ac6 e8c5cc2 a3c7ac6 e8c5cc2 a3c7ac6 e8c5cc2 a3c7ac6 e8c5cc2 5eec49d e8c5cc2 5eec49d e8c5cc2 5eec49d e8c5cc2 a3c7ac6 e8c5cc2 a3c7ac6 e8c5cc2 a3c7ac6 e8c5cc2 a3c7ac6 2299777 a3c7ac6 2299777 a3c7ac6 e8c5cc2 af38999 e8c5cc2 a3c7ac6 e8c5cc2 a3c7ac6 e8c5cc2 5eec49d e8c5cc2 af38999 e8c5cc2 af38999 a3c7ac6 af38999 e8c5cc2 2299777 e8c5cc2 5eec49d e8c5cc2 2299777 e8c5cc2 5eec49d 2299777 5eec49d e8c5cc2 2299777 e8c5cc2 5eec49d e8c5cc2 af38999 e8c5cc2 5eec49d e8c5cc2 5eec49d e8c5cc2 a3c7ac6 e8c5cc2 5eec49d e8c5cc2 5eec49d e8c5cc2 a3c7ac6 e8c5cc2 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 151 152 153 154 155 156 157 158 159 160 161 162 163 164 165 166 167 168 169 170 171 172 173 174 175 176 177 178 179 180 181 182 183 184 185 186 187 188 189 190 191 192 193 194 195 196 197 198 199 200 201 202 203 204 205 206 207 208 209 210 211 212 213 214 215 216 217 218 219 220 221 222 223 224 225 226 227 228 229 230 231 232 233 234 235 236 237 238 239 240 241 242 243 244 245 246 247 248 249 250 251 252 253 254 255 256 257 258 259 260 261 262 263 264 265 266 267 268 269 270 271 272 273 274 275 276 277 278 279 280 281 282 283 284 285 286 287 288 289 290 291 292 293 294 295 296 297 298 299 300 301 302 303 304 305 306 307 308 309 310 311 312 313 314 315 316 317 318 319 320 321 322 323 324 325 326 327 328 329 330 331 332 333 334 335 336 337 338 339 340 341 342 343 344 345 346 347 348 349 350 351 352 353 354 355 356 357 358 359 360 361 362 363 364 365 366 367 368 369 370 371 372 373 374 375 376 377 378 379 380 381 382 383 384 385 386 387 388 389 390 391 392 393 394 395 396 397 398 399 400 401 402 403 404 405 406 407 408 409 410 411 412 413 414 415 416 417 418 419 420 421 422 423 424 425 426 427 428 429 430 431 432 433 434 435 436 437 438 439 440 441 442 443 444 445 446 447 448 449 450 451 452 453 454 455 456 457 458 459 460 461 462 463 464 465 466 467 468 469 470 471 472 473 474 475 476 477 478 479 480 481 482 483 484 485 486 487 488 489 490 491 492 493 494 495 496 497 498 499 500 501 502 503 504 505 506 507 508 509 510 511 512 513 514 515 516 517 518 519 520 521 522 523 524 525 526 527 528 529 530 531 532 533 534 535 536 537 538 539 540 541 542 543 544 545 546 547 548 549 550 551 552 553 554 555 556 557 558 559 560 561 562 563 564 565 566 567 568 569 570 571 572 573 574 575 576 577 578 579 580 581 582 583 584 585 586 587 588 589 590 591 592 593 594 595 596 597 598 599 600 601 602 603 604 605 606 607 608 609 610 611 612 613 614 615 616 617 618 619 620 621 622 623 624 625 626 627 628 629 630 631 632 633 634 635 636 637 638 639 640 641 642 643 644 645 646 647 648 649 650 651 652 653 654 655 656 657 658 659 660 661 662 663 664 665 666 667 668 669 670 671 672 673 674 675 676 677 678 679 680 681 682 683 684 685 686 687 688 689 690 691 692 693 694 695 696 697 698 699 700 701 702 703 704 705 706 707 708 709 710 711 712 713 714 715 716 717 718 719 720 721 722 723 724 725 726 727 728 729 730 731 732 733 734 735 736 737 738 739 740 741 742 743 744 745 746 747 748 749 750 751 752 753 754 755 756 757 758 759 760 761 762 763 764 765 766 767 768 769 770 771 772 773 774 775 776 777 778 779 780 781 782 783 784 785 786 787 788 789 790 791 792 793 794 795 796 797 798 799 800 801 802 803 804 805 806 807 808 809 810 811 812 813 814 815 816 817 818 819 820 821 822 823 824 825 826 827 828 829 830 831 832 833 834 835 836 837 838 839 840 841 842 843 844 845 846 847 848 849 850 851 852 853 854 855 856 857 858 859 860 861 862 863 864 865 866 867 868 869 870 871 872 873 874 875 876 877 878 879 880 881 882 883 884 885 886 887 888 889 890 891 892 893 894 895 896 897 898 899 900 901 902 903 904 905 906 907 908 909 910 911 912 913 914 915 916 917 918 919 920 921 922 923 924 925 926 927 928 | """ResearchOS-ML pipeline: retrieve -> fuse -> rerank -> diversify -> answer.
Two capabilities:
ask() question answering over the ingested corpus
explain_citation() resolve a citation you saw in a paper, then explain it
Three things differ from the parent project's pipeline, all forced by the corpus:
FR4.2 diversification. One paper produces dozens of chunks, so the top-scoring
five are routinely five paragraphs of the same paper. `max_per_paper` caps that,
which is what makes "explain" and "compare" answers multi-source.
US5 rejection. The threshold is on the MEAN of the top-N reranker scores, not
the maximum, because the maximum is not discriminative. Measured on this corpus:
query max mean-of-5
explain JEPA 0.998 0.995 in corpus
how does JEPA differ from MAE 0.898 0.671 in corpus
what causes knee osteoarthritis 0.670 0.190 NOT in corpus
sourdough hydration ratio 0.001 0.001 NOT in corpus
The osteoarthritis query hits 0.670 on max because the corpus really does
contain a self-supervised bone-densitometry paper - one genuine partial match
defeats a max-based threshold. The mean separates cleanly, because a query that
belongs to this corpus matches *many* passages, not one.
Scale note: CrossEncoder.predict applies a sigmoid, so torch-backend scores are
probabilities in [0,1]. The ONNX backend returns raw logits - rescale if you
switch backends.
Citation resolution is local-first, then arXiv on demand, so the tool is not
limited to whatever was ingested.
"""
from __future__ import annotations
import re
import threading
import time
from collections import OrderedDict
from concurrent.futures import FIRST_COMPLETED, ThreadPoolExecutor
from concurrent.futures import wait as _futures_wait
from typing import Any
from src.api.diagnose import diagnose
from src.api.llm import (
explain_paper, general_answer, generate_answer, is_configured, is_not_covered,
)
from src.config import device
from src.data.arxiv_source import (
ArxivUnavailable, Paper, fetch_metadata_bounded, read_papers,
)
from src.data import openalex_source as openalex
from src.data.chunking import chunk_paper, chunks_to_corpus, read_chunks
from src.data.citation import looks_like_a_question, paper_url, resolve
from src.retrieval.bm25 import BM25Retriever
from src.retrieval.dense import DenseRetriever
from src.retrieval.diversify import cap_per_paper, distinct_papers
from src.retrieval.fusion import reciprocal_rank_fusion
PSEUDO_QID = "q"
# Live sources run side by side under one deadline; their own clients bound
# each call, so this pool only has to hold the two waits of one request.
_LIVE_POOL = ThreadPoolExecutor(max_workers=8, thread_name_prefix="live-search")
_SEARCH_POOL = ThreadPoolExecutor(max_workers=2, thread_name_prefix="first-stage")
RANK_CACHE_MAX = 512
def _dedup_key(item: dict[str, Any]) -> str:
"""arXiv id when known, else a normalised title: one paper, one entry."""
pid = item.get("paper_id") or ""
if pid and not pid.startswith("openalex:"):
return pid
return re.sub(r"[^a-z0-9]", "", (item.get("title") or "").lower())[:120]
_QUERY_STOPWORDS = {
"the", "and", "for", "with", "from", "what", "how", "why", "does", "explain",
"when", "are", "was", "that", "this", "there", "which", "about", "into",
"some", "can", "you", "its", "their", "using", "used", "based",
}
class MLPipeline:
"""Loads chunk corpus and indices once, then answers questions."""
def __init__(self, cfg: dict[str, Any]) -> None:
self.cfg = cfg
paths, rc = cfg["paths"], cfg["retrieval"]
chunks = read_chunks(paths["chunks"])
self.corpus = chunks_to_corpus(chunks)
self.chunks_by_paper: dict[str, list[str]] = {}
for chunk in chunks:
self.chunks_by_paper.setdefault(chunk.paper_id, []).append(chunk.chunk_id)
# Full paper records, for citation resolution and metadata display.
self.papers_by_id = {p.paper_id: vars(p) for p in read_papers(paths["papers"])}
# An abstract-only paper holds its abstract twice - once in its paper
# record, once as its single passage - as two equal strings. At 270k
# papers that is ~400 MB of RAM for nothing. Point both at one object.
for entry in self.corpus.values():
if entry["section"] == "abstract":
record = self.papers_by_id.get(entry["paper_id"])
if record is not None and record.get("abstract") == entry["text"]:
entry["text"] = record["abstract"]
doc_ids = list(self.corpus.keys())
doc_texts = [
f"{self.corpus[d]['title']} {self.corpus[d]['text']}".strip() for d in doc_ids
]
# The lexical arm is mandatory here: bge-small has frozen weights and has
# never seen "JEPA" or "GRPO", while BM25 gives a rare token a very high
# IDF weight. It is the only defence against vocabulary coined after the
# embedding model was trained.
self.bm25 = BM25Retriever(
k1=rc["bm25"]["k1"], b=rc["bm25"]["b"], stemmer_lang=rc["bm25"]["stemmer"]
)
self.bm25.index(doc_ids, doc_texts, cache_dir=paths["index_cache"])
self.dense = DenseRetriever(
model_name=rc["dense"]["model"],
batch_size=rc["dense"]["batch_size"],
max_length=rc["dense"]["max_length"],
query_prefix=rc["dense"]["query_prefix"],
device=device(),
)
self.dense.index_corpus(doc_ids, doc_texts, cache_dir=paths["index_cache"])
from src.rerank.model import Reranker
mc = cfg["reranker"]
self.reranker = Reranker(
model_name_or_path=mc["model"],
max_length=mc["max_length"],
batch_size=mc["batch_size"],
device=device(),
fp16=mc.get("fp16", True) and device() == "cuda",
)
# One GPU, one model, many HTTP threads. FastAPI runs sync endpoints in
# a threadpool, so without this two concurrent users would drive
# forward passes through the same CUDA module simultaneously - which
# risks wrong results and VRAM spikes. Retrieval is fast enough
# (sub-second) that serialising it costs little.
self._gpu_lock = threading.Lock()
self.rrf_k = rc["fusion"]["rrf_k"]
self.retrieve_k = rc["top_k"]
self._rank_cache: OrderedDict[str, dict[str, float]] = OrderedDict()
self._cache_lock = threading.Lock()
self.cache_hits = 0
self.early_exits = 0
print(
f"[ml] ready | {len(self.corpus)} chunks from {len(self.papers_by_id)} papers "
f"| reranker={mc['model']} | llm={'groq' if is_configured() else 'disabled'}"
)
def warmup(self) -> None:
"""Run one throwaway query so the first real user does not pay for it.
The first call into torch, faiss and the tokenizers allocates buffers
and compiles kernels; on a CPU container that is seconds the first
visitor would otherwise sit through. The result is cached, harmlessly.
"""
t0 = time.perf_counter()
try:
self._ranked("how do transformers use attention for language modelling")
print(f"[ml] warmed up in {time.perf_counter() - t0:.1f}s", flush=True)
except Exception as exc: # noqa: BLE001 - warmup is an optimisation only
print(f"[ml] warmup skipped ({type(exc).__name__})", flush=True)
# ---------------------------------------------------------------- helpers
def _passage(self, chunk_id: str, score: float) -> dict[str, Any]:
entry = self.corpus[chunk_id]
return {
"chunk_id": chunk_id,
"doc_id": chunk_id,
"paper_id": entry["paper_id"],
"arxiv_url": f"https://arxiv.org/abs/{entry['paper_id']}",
"score": float(score),
"title": entry["title"],
"section": entry["section"],
"published": entry["published"],
"text": entry["text"],
}
def _confidence(self, scores: list[float]) -> tuple[float, float, bool]:
ac = self.cfg["answer"]
best = max(scores, default=0.0)
n = min(ac.get("confidence_top_n", 5), len(scores))
mean_top = sum(sorted(scores, reverse=True)[:n]) / n if n else 0.0
return best, mean_top, mean_top >= ac["min_mean_score"]
# ------------------------------------------------------------- retrieval
def _ranked(self, question: str) -> dict[str, float]:
"""Reranked candidate chunks for a query: {chunk_id: score}.
The one place a query is searched, so every endpoint shares its two
latency wins:
* BM25 and dense search run side by side. They are independent, and
both spend their time in native code (numpy, faiss, torch) that
releases the GIL, so the cost is the slower of the two, not the sum.
* Results are cached. The corpus is static and every stage is
deterministic, so a repeated query - a demo, a retry, /papers then
/ask on the same text, a shared link - skips search AND the
reranker, which is the dominant cost on CPU.
"""
key = " ".join(question.lower().split())
with self._cache_lock:
hit = self._rank_cache.get(key)
if hit is not None:
self._rank_cache.move_to_end(key)
self.cache_hits += 1
return dict(hit)
query_map = {PSEUDO_QID: question}
with self._gpu_lock:
bm25_future = _SEARCH_POOL.submit(self.bm25.search, query_map, self.retrieve_k)
dense_run = self.dense.search(query_map, top_k=self.retrieve_k)
fused = reciprocal_rank_fusion(
[bm25_future.result(), dense_run], k=self.rrf_k, top_k=self.retrieve_k
)
ranked = self._rerank_adaptive(query_map, fused[PSEUDO_QID])
with self._cache_lock:
self._rank_cache[key] = dict(ranked)
while len(self._rank_cache) > RANK_CACHE_MAX:
self._rank_cache.popitem(last=False)
return ranked
def _rerank_adaptive(self, query_map: dict[str, str], fused: dict[str, float]) -> dict[str, float]:
"""Cross-encoder scores for the fused candidates, stopping early when sure.
The cross-encoder is the dominant cost of a search and scales with the
candidates scored (seconds on a 2-vCPU CPU). Score the best `first`
fused candidates; if their top-5 mean already clears `tau`, the answer
is obvious and the remaining candidates are not worth scoring.
Otherwise score the rest, so a hard query still sees every candidate.
Measured offline (scripts/eval_early_exit.py): recall@10 unchanged,
~27% fewer candidates scored.
"""
ee = self.cfg["retrieval"].get("early_exit")
if not ee or len(fused) <= ee["first"]:
reranked, _ = self.reranker.rerank(query_map, {PSEUDO_QID: fused}, self.corpus,
top_k=self.retrieve_k)
return reranked[PSEUDO_QID]
order = sorted(fused.items(), key=lambda kv: kv[1], reverse=True)
head = dict(order[: ee["first"]])
scores, _ = self.reranker.rerank(query_map, {PSEUDO_QID: head}, self.corpus,
top_k=len(head))
scores = scores[PSEUDO_QID]
top = sorted(scores.values(), reverse=True)[:5]
if sum(top) / len(top) >= ee["tau"]:
self.early_exits += 1
return scores
tail = dict(order[ee["first"]:])
rest, _ = self.reranker.rerank(query_map, {PSEUDO_QID: tail}, self.corpus,
top_k=len(tail))
return {**scores, **rest[PSEUDO_QID]}
def retrieve(self, question: str, top_k: int | None = None) -> list[dict[str, Any]]:
"""Hybrid retrieve, fuse, rerank, then cap chunks per paper."""
reranked = {PSEUDO_QID: self._ranked(question)}
ac = self.cfg["answer"]
selected = cap_per_paper(
reranked[PSEUDO_QID],
self.corpus,
top_k=top_k or ac["top_k_passages"],
max_per_paper=ac["max_per_paper"],
)
return [self._passage(cid, s) for cid, s in selected]
# -------------------------------------------------------- paper discovery
def find_papers(self, topic: str, top_k: int = 10) -> dict[str, Any]:
"""Topic in, ranked *papers* out.
The ask path returns passages, which is right for answering a question
and wrong for "show me work on X" - you get five paragraphs where you
wanted five papers. Here the per-paper cap is deliberately skipped, a
wide candidate set is reranked, and results are then grouped by paper.
A paper is scored by its single best passage. Summing would reward long
papers with many mediocre chunks over short papers with one excellent
one, which inverts what you actually want from a search result.
"""
start = time.perf_counter()
reranked = {PSEUDO_QID: self._ranked(topic)}
grouped: dict[str, dict[str, Any]] = {}
for chunk_id, score in reranked[PSEUDO_QID].items():
entry = self.corpus[chunk_id]
pid = entry["paper_id"]
bucket = grouped.setdefault(
pid, {"score": float("-inf"), "matches": 0, "best_chunk": None, "sections": []}
)
bucket["matches"] += 1
bucket["sections"].append(entry["section"])
if score > bucket["score"]:
bucket["score"] = float(score)
bucket["best_chunk"] = chunk_id
ordered = sorted(grouped.items(), key=lambda kv: kv[1]["score"], reverse=True)[:top_k]
papers: list[dict[str, Any]] = []
for pid, info in ordered:
meta = self.papers_by_id.get(pid, {})
best = self.corpus[info["best_chunk"]]
papers.append(
{
"paper_id": pid,
"doc_id": info["best_chunk"],
"title": meta.get("title") or best["title"],
"authors": (meta.get("authors") or [])[:6],
"abstract": meta.get("abstract", ""),
"published": meta.get("published") or best["published"],
"primary_category": meta.get("primary_category", ""),
"arxiv_url": f"https://arxiv.org/abs/{pid}",
"score": round(info["score"], 4),
"matched_passages": info["matches"],
"matched_sections": sorted(set(info["sections"]))[:5],
"why": best["text"][:400],
"why_section": best["section"],
"has_full_text": bool(meta.get("full_text")),
}
)
scores = [p["score"] for p in papers]
best_score, mean_top, confident = self._confidence(scores)
# Same live fallback the ask path has. Without it, Find papers is
# limited to the ingested sample - and the corpus holds roughly 1,400
# papers per category against the 50-150/day arXiv publishes, so a
# topic can be well covered on arXiv and absent here. Searching "economic
# world models" returned unrelated robotics papers for exactly this
# reason while the paper sat on arXiv.
source = "corpus"
ac = self.cfg["answer"]
if not confident and self.cfg["citation"].get("allow_remote", True):
live = self.search_live(topic, top_k=top_k)
if live:
live = self._rescore_live(topic, live)
floor = ac.get("live_min_score", 0.30)
kept = [p for p in live if p["score"] >= floor]
if kept:
seen = {p["paper_id"] for p in papers}
fetched = [
{
"paper_id": p["paper_id"],
"doc_id": p["doc_id"],
"title": p["title"],
"authors": p.get("authors", []),
"abstract": p["text"],
"published": p["published"],
"primary_category": "",
"arxiv_url": p["arxiv_url"],
"venue": p.get("venue", ""),
"cited_by_count": p.get("cited_by_count"),
"live_source": p.get("live_source", "arxiv"),
"score": round(p["score"], 4),
"matched_passages": 1,
"matched_sections": ["abstract"],
"why": p["text"][:400],
"why_section": "abstract",
"has_full_text": False,
"from_arxiv": True,
}
for p in kept
if p["paper_id"] not in seen
]
if fetched:
# Corpus papers only fill remaining slots if they clear
# the same bar - otherwise a 0.08 match sat under a
# result labelled confident.
papers = (fetched + [p for p in papers if p["score"] >= floor])[:top_k]
source = "arxiv_live"
confident = True
mean_top = sum(p["score"] for p in fetched) / len(fetched)
best_score = max(p["score"] for p in fetched)
# v2: nothing convincing in the corpus or on arXiv - diagnose why.
diagnosis, diagnosis_ms = None, 0.0
if not confident:
diagnosis, diagnosis_ms = self._diagnose(topic, papers, None)
return {
"topic": topic,
"source": source,
"papers": papers,
"confident": confident,
"top_score": round(best_score, 3),
"mean_top_score": round(mean_top, 3),
"diagnosis": diagnosis,
"timing_ms": {"total": round((time.perf_counter() - start) * 1000.0, 1),
"diagnosis": round(diagnosis_ms, 1)},
}
def _rescore_live(
self, question: str, live: list[dict[str, Any]]
) -> list[dict[str, Any]]:
"""Score live arXiv abstracts with the same reranker the corpus uses.
arXiv's own relevance ranking is lexical and generous. Running the
cross-encoder over the abstracts puts live results on the same scale as
corpus results, so one confidence threshold governs both.
"""
pairs = [(question, f"{p['title']} {p['text']}".strip()) for p in live]
if not pairs:
return []
with self._gpu_lock:
scores = self.reranker.score(pairs)
for passage, score in zip(live, scores):
passage["score"] = float(score)
return sorted(live, key=lambda p: p["score"], reverse=True)
def answer_from_document(
self, question: str, passages: list[dict[str, Any]], title: str, generate: bool = True
) -> dict[str, Any]:
"""Answer a question against a single uploaded paper.
Every chunk of the document is scored directly by the cross-encoder -
no BM25, no embeddings, no index. A paper is only 20-40 chunks, so
exhaustive scoring is both cheap and strictly better than retrieval,
which can only approximate it.
"""
start = time.perf_counter()
if not passages:
return {
"question": question,
"answer": "That document has no readable passages.",
"confident": False,
"source": "upload",
"top_score": 0.0,
"mean_top_score": 0.0,
"citations": [],
"distinct_papers": 0,
"generation_configured": is_configured(),
"timing_ms": {"retrieval_and_rerank": 0.0, "generation": 0.0, "total": 0.0},
}
pairs = [(question, f"{p['title']} {p['text']}".strip()) for p in passages]
with self._gpu_lock:
scores = self.reranker.score(pairs)
for passage, score in zip(passages, scores):
passage["score"] = float(score)
ranked = sorted(passages, key=lambda p: p["score"], reverse=True)
ac = self.cfg["answer"]
top = ranked[: ac["top_k_passages"]]
best, mean_top, _ = self._confidence([p["score"] for p in top])
retrieval_ms = (time.perf_counter() - start) * 1000.0
answer_text: str | None = None
generation_ms = 0.0
if generate:
gen_start = time.perf_counter()
answer_text = generate_answer(
f"About the paper '{title}': {question}",
top,
model=self._answer_models(),
max_tokens=ac["max_tokens"],
)
generation_ms = (time.perf_counter() - gen_start) * 1000.0
return {
"question": question,
"answer": answer_text,
# The user chose this document, so we answer from it even when the
# match is weak - declining would be second-guessing them.
"confident": True,
"source": "upload",
"top_score": round(best, 3),
"mean_top_score": round(mean_top, 3),
"citations": top,
"distinct_papers": 1,
"generation_configured": is_configured(),
"timing_ms": {
"retrieval_and_rerank": round(retrieval_ms, 1),
"generation": round(generation_ms, 1),
"total": round(retrieval_ms + generation_ms, 1),
},
}
# ------------------------------------------------------------ live search
def search_live(self, question: str, top_k: int = 6) -> list[dict[str, Any]]:
"""Search arXiv and OpenAlex live, for questions the corpus cannot answer.
The corpus is a sample, not coverage: ~1,400 papers per category across
nine months, against the 50-150/day arXiv actually publishes. Most
papers a user asks about were never ingested, and telling them "not in
this corpus" is honest but useless when the paper plainly exists.
Two sources, because each misses what the other has. arXiv is the
freshest for preprints; OpenAlex adds conference and journal papers
that never had one. Both run in parallel under ONE wall-clock budget
(`citation.live_budget_s`), so adding a source never adds latency:
whatever has answered when the budget expires is used, and a source
that is down or paused costs nothing. The same paper found by both
collapses to one entry, keyed on its arXiv id or normalised title.
Abstracts only - no PDF fetch. Results are rescored by the corpus
reranker afterwards, so one confidence bar governs every source.
"""
sources = self.cfg["citation"].get("live_sources", ["arxiv", "openalex"])
budget = float(self.cfg["citation"].get("live_budget_s", 5.0))
deadline = time.monotonic() + budget
jobs = {}
if "arxiv" in sources:
jobs["arxiv"] = _LIVE_POOL.submit(self._live_arxiv, question, top_k, deadline)
if "openalex" in sources:
jobs["openalex"] = _LIVE_POOL.submit(self._live_openalex, question, top_k, deadline)
# Do not wait for the slowest source. Once one has results, the other
# gets a short grace and is then left behind: arXiv often takes 3-5 s
# while OpenAlex answers in ~1 s, and its extra papers are not worth
# holding the whole response. A source that returns nothing (down,
# paused, empty) does not start the clock - the other keeps its full
# budget.
grace = float(self.cfg["citation"].get("live_grace_s", 0.75))
found: dict[str, list[dict[str, Any]]] = {}
name_of = {job: name for name, job in jobs.items()}
pending = set(jobs.values())
cutoff = deadline
while pending and time.monotonic() < cutoff:
done, pending = _futures_wait(
pending, timeout=max(cutoff - time.monotonic(), 0.01),
return_when=FIRST_COMPLETED)
for job in done:
try:
found[name_of[job]] = job.result()
except Exception: # noqa: BLE001 - a failed source is an empty source
found[name_of[job]] = []
if found[name_of[job]]:
cutoff = min(cutoff, time.monotonic() + grace)
merged: list[dict[str, Any]] = []
seen: dict[str, dict[str, Any]] = {}
# arXiv first: on a duplicate, its record wins (fresher version), and
# OpenAlex contributes the venue and citation count it lacks.
for name in ("arxiv", "openalex"):
for item in found.get(name, []):
key = _dedup_key(item)
if key in seen:
kept = seen[key]
kept["venue"] = kept.get("venue") or item.get("venue", "")
kept["cited_by_count"] = max(kept.get("cited_by_count") or 0,
item.get("cited_by_count") or 0)
kept["live_source"] = "arxiv+openalex"
continue
seen[key] = item
merged.append(item)
return merged
# Kept for callers that predate OpenAlex.
search_arxiv_live = search_live
def _live_arxiv(self, question: str, top_k: int, deadline: float) -> list[dict[str, Any]]:
terms = [w for w in re.findall(r"[A-Za-z][A-Za-z0-9-]{2,}", question)
if w.lower() not in _QUERY_STOPWORDS][:12]
if not terms:
return []
queries = [
"all:" + " AND ".join(f'"{t}"' for t in terms[:4]), # precise first
"all:" + " ".join(terms[:8]), # then looser
]
found: list[Paper] = []
for query in queries:
remaining = deadline - time.monotonic()
if remaining <= 0.25:
break
try:
found = fetch_metadata_bounded(query, max_results=top_k,
budget_s=remaining)
except ArxivUnavailable:
return [] # a failed enrichment is not a failed request
if found:
break
return [
{
"chunk_id": f"live:{p.paper_id}",
"doc_id": f"live:{p.paper_id}",
"paper_id": p.paper_id,
"arxiv_url": f"https://arxiv.org/abs/{p.paper_id}",
"score": 1.0,
"title": p.title,
"section": "abstract",
"published": p.published,
"text": p.abstract,
"authors": p.authors[:8],
"venue": "arXiv",
"cited_by_count": None,
"live_source": "arxiv",
}
for p in found
]
def _live_openalex(self, question: str, top_k: int, deadline: float) -> list[dict[str, Any]]:
remaining = deadline - time.monotonic()
if remaining <= 0.25:
return []
try:
works = openalex.search_works(question, max_results=top_k, budget_s=remaining)
except openalex.OpenAlexUnavailable:
return []
return [
{
"chunk_id": f"live:{w['paper_id']}",
"doc_id": f"live:{w['paper_id']}",
"paper_id": w["paper_id"],
# Named arxiv_url for compatibility with every client; for a
# paper with no arXiv copy it is the DOI or landing page.
"arxiv_url": w["url"],
"score": 1.0,
"title": w["title"],
"section": "abstract",
"published": w["published"],
"text": w["abstract"],
"authors": w["authors"][:8],
"venue": w["venue"],
"cited_by_count": w["cited_by_count"],
"doi": w["doi"],
"live_source": "openalex",
}
for w in works
]
# ------------------------------------------------------------------- ask
def answer(self, question: str, generate: bool = True) -> dict[str, Any]:
"""Question answering, including the US5 rejection path."""
start = time.perf_counter()
passages = self.retrieve(question)
retrieval_ms = (time.perf_counter() - start) * 1000.0
# Kept for diagnosis: if everything below fails, these weak corpus hits
# are exactly what the critic needs to see to say what went wrong.
corpus_passages = passages
ac = self.cfg["answer"]
best, mean_top, confident = self._confidence([p["score"] for p in passages])
# Blend in live arXiv results when the corpus alone is thin.
#
# The corpus is a sample (~1,400 papers per category against the 50-150
# a day arXiv publishes), so a question can be perfectly answerable from
# the literature and unanswerable from here. Two triggers:
#
# below min_mean_score the corpus has nothing - live results replace
# below enrich_below the corpus has something thin - live results
# are added, so the answer draws on both
#
# A strong corpus hit skips the network entirely and stays sub-second.
answer_text: str | None = None
generation_ms = 0.0
source = "corpus"
remote_ok = self.cfg["citation"].get("allow_remote", True)
enrich_below = ac.get("enrich_below", 0.0)
# Timed on its own: this was the one stage outside every timer, which is
# how a 45s arXiv stall reported itself as "retrieval_and_rerank: 1783ms".
live_ms = 0.0
if remote_ok and (not confident or mean_top < enrich_below):
live_start = time.perf_counter()
live = self.search_live(question)
live_ms = (time.perf_counter() - live_start) * 1000.0
if live:
# arXiv always returns *something*, so live results must clear
# the same confidence bar as the corpus. Without this the
# honest "I do not know" is replaced by confident nonsense -
# a sourdough question came back with quantum-physics papers.
# Filter live results PER PASSAGE, not on their mean.
#
# The mean-of-top-5 rule works for the corpus because a query
# that belongs to it matches many passages. An arXiv keyword
# search returns a mixed bag by construction - three loose hits
# would drag the mean under and bury the one paper that is
# actually the answer. So each live passage clears its own bar.
live = self._rescore_live(question, live)
floor = ac.get("live_min_score", 0.30)
live = [p for p in live if p["score"] >= floor]
seen = {p["paper_id"] for p in passages}
extra = [p for p in live if p["paper_id"] not in seen]
if confident and extra:
passages = passages + extra # enrich
source = "corpus+arxiv_live"
elif extra:
passages = extra # replace
source = "arxiv_live"
confident = True
mean_top = sum(p["score"] for p in extra) / len(extra)
elif not confident:
passages = [] # genuinely unanswerable
# Answer from the papers when they cover the question. When they do
# not - nothing retrieved, or the answerer replies NOT_COVERED - fall
# back to the model's own knowledge, but only for AI questions and
# labelled as not grounded in the papers. Everything else is declined.
grounded = True
uncovered = not passages
if passages and generate:
gen_start = time.perf_counter()
answer_text = generate_answer(
question, passages, model=self._answer_models(), max_tokens=ac["max_tokens"]
)
generation_ms = (time.perf_counter() - gen_start) * 1000.0
if is_not_covered(answer_text):
uncovered, answer_text = True, None
if uncovered and generate and ac.get("general_knowledge", True):
gen_start = time.perf_counter()
try:
general = general_answer(question, model=self._answer_models(),
max_tokens=ac["max_tokens"])
except Exception as exc: # noqa: BLE001 - fall back to the plain decline
print(f"[ml] general answer failed ({type(exc).__name__})", flush=True)
general = None
generation_ms += (time.perf_counter() - gen_start) * 1000.0
if general:
answer_text, source, grounded, confident = general, "general_knowledge", False, False
if uncovered and answer_text is None:
answer_text = (
"Nothing answers that - not in the local corpus, and not in a live "
"arXiv search either. If this is an AI/ML topic the paper may simply "
"be too new or too obscure; if it is not, this tool only covers AI research."
if not passages else
"The papers found for this do not contain an answer to it."
)
# v2: the gate fired and nothing rescued it, so say what went wrong.
diagnosis, diagnosis_ms = None, 0.0
if not passages and source != "general_knowledge":
diagnosis, diagnosis_ms = self._diagnose(question, corpus_passages, None)
return {
"question": question,
"answer": answer_text,
"confident": confident,
"source": source,
"grounded": grounded,
"top_score": round(best, 3),
"mean_top_score": round(mean_top, 3),
"citations": passages,
"distinct_papers": distinct_papers(passages),
"generation_configured": is_configured(),
"diagnosis": diagnosis,
"timing_ms": {
"retrieval_and_rerank": round(retrieval_ms, 1),
"arxiv_live": round(live_ms, 1),
"generation": round(generation_ms, 1),
"diagnosis": round(diagnosis_ms, 1),
"total": round(retrieval_ms + live_ms + generation_ms + diagnosis_ms, 1),
},
}
def _answer_models(self) -> list[str]:
"""Primary answer model, then fallbacks for when its daily cap is spent."""
ac = self.cfg["answer"]
return [ac["model"], *ac.get("fallback_models", [])]
def _diagnose(self, question: str, passages: list[dict[str, Any]],
answer: str | None) -> tuple[dict[str, Any] | None, float]:
"""Run the v2 failure diagnosis, if enabled. Never raises."""
dc = self.cfg.get("diagnosis", {})
if not dc.get("enabled", True):
return None, 0.0
t0 = time.perf_counter()
d = diagnose(question, passages, answer,
model=dc.get("models") or dc.get("model", self.cfg["answer"]["model"]),
budget_s=float(dc.get("budget_s", 6.0)))
return (d.to_json() if d else None), (time.perf_counter() - t0) * 1000.0
# ------------------------------------------------------- citation lookup
def _passages_for_paper(self, paper: Paper, from_corpus: bool) -> list[dict[str, Any]]:
"""Passages describing one specific paper.
Indexed papers use their stored chunks. A paper fetched on demand is
chunked in memory - it is not added to the index, because rebuilding the
index per lookup would be far slower than the lookup itself.
"""
if from_corpus and paper.paper_id in self.chunks_by_paper:
chunk_ids = self.chunks_by_paper[paper.paper_id]
ordered = sorted(chunk_ids, key=lambda c: self.corpus[c]["section"] != "abstract")
return [self._passage(cid, 1.0) for cid in ordered[:8]]
chunks = chunk_paper(
paper,
max_words=self.cfg["chunking"]["max_words"],
overlap_words=self.cfg["chunking"]["overlap_words"],
min_words=self.cfg["chunking"]["min_words"],
)[:8]
return [
{
"chunk_id": c.chunk_id,
"doc_id": c.chunk_id,
"paper_id": c.paper_id,
"arxiv_url": paper_url(paper),
"score": 1.0,
"title": c.title,
"section": c.section,
"published": c.published,
"text": c.text,
}
for c in chunks
]
def explain_citation(self, citation: str, generate: bool = True) -> dict[str, Any]:
"""Resolve a citation string to a paper and explain it.
Returns the explanation plus `related`: what else in the corpus talks
about this paper, which is where the "and here is the wider context"
half of the answer comes from.
"""
start = time.perf_counter()
# A question is not a citation. Trying to resolve one costs two arXiv
# searches that cannot succeed, and arXiv answers 429 - which is how a
# concept question turned into a minute of waiting and then a failure.
if looks_like_a_question(citation):
answer = self.answer(citation, generate=generate)
return {
"citation": citation,
"resolved": False,
"method": "question",
"query_used": citation,
"candidates": [],
"paper": None,
"explanation": answer["answer"],
"passages": answer["citations"],
"related": [],
"answered_as_question": True,
"source": answer["source"],
"generation_configured": is_configured(),
"timing_ms": answer["timing_ms"],
"diagnosis": answer.get("diagnosis"),
# The complete /ask payload, so the client can render it
# directly. Without this the web client discarded the answer
# computed here and called /ask, re-running retrieval,
# reranking and generation - every Explain-mode question cost
# two full pipeline passes and two Groq calls.
"answer_result": answer,
}
resolution = resolve(
citation,
self.papers_by_id,
allow_remote=self.cfg["citation"]["allow_remote"],
fetch_full_text=self.cfg["citation"]["fetch_full_text"],
pdf_dir=self.cfg["paths"]["pdf_cache"],
text_dir=self.cfg["paths"]["text_cache"],
max_cache_gb=self.cfg["citation"].get("max_pdf_cache_gb", 5.0),
)
if resolution.paper is None:
return {
"citation": citation,
"resolved": False,
"method": resolution.method,
"query_used": resolution.query_used,
"candidates": resolution.candidates,
"explanation": (
"Could not resolve that citation to a paper. Try the arXiv id, or the "
"paper title on its own."
),
"paper": None,
"passages": [],
"related": [],
"generation_configured": is_configured(),
"timing_ms": {"total": round((time.perf_counter() - start) * 1000.0, 1)},
}
paper = resolution.paper
passages = self._passages_for_paper(paper, resolution.from_corpus)
# What else in the corpus discusses this work - the wider-context half.
related_query = f"{paper.title} {paper.abstract[:300]}"
related = [
p
for p in self.retrieve(related_query, top_k=6)
if p["paper_id"] != paper.paper_id
][:4]
explanation: str | None = None
gen_ms = 0.0
if generate:
gen_start = time.perf_counter()
explanation = explain_paper(
paper.title,
passages + related,
model=self._answer_models(),
max_tokens=self.cfg["citation"]["max_tokens"],
)
gen_ms = (time.perf_counter() - gen_start) * 1000.0
return {
"citation": citation,
"resolved": True,
"method": resolution.method,
"query_used": resolution.query_used,
"from_corpus": resolution.from_corpus,
"candidates": resolution.candidates,
"paper": {
"paper_id": paper.paper_id,
"title": paper.title,
"abstract": paper.abstract,
"authors": paper.authors[:8],
"published": paper.published,
"primary_category": paper.primary_category,
"arxiv_url": paper_url(paper),
"venue": (paper.metadata or {}).get("venue", ""),
"doi": (paper.metadata or {}).get("doi", ""),
"has_full_text": bool(paper.full_text),
},
"explanation": explanation,
"passages": passages,
"related": related,
"generation_configured": is_configured(),
"timing_ms": {
"total": round((time.perf_counter() - start) * 1000.0, 1),
"generation": round(gen_ms, 1),
},
}
|