Spaces:
Running on Zero
Running on Zero
Download src/api/ml_pipeline.py from Kagzy/ResearchOS: direct link, hf CLI and curl.
- Browser
- Download file 43.5 kB
-
https://huggingface.co/spaces/Kagzy/ResearchOS/resolve/main/src/api/ml_pipeline.py
- Command line
-
hf download hf://spaces/Kagzy/ResearchOS/src/api/ml_pipeline.py
-
curl -L -o ml_pipeline.py https://huggingface.co/spaces/Kagzy/ResearchOS/resolve/main/src/api/ml_pipeline.py
43.5 kB
| """ResearchOS-ML pipeline: retrieve -> fuse -> rerank -> diversify -> answer. | |
| Two capabilities: | |
| ask() question answering over the ingested corpus | |
| explain_citation() resolve a citation you saw in a paper, then explain it | |
| Three things differ from the parent project's pipeline, all forced by the corpus: | |
| FR4.2 diversification. One paper produces dozens of chunks, so the top-scoring | |
| five are routinely five paragraphs of the same paper. `max_per_paper` caps that, | |
| which is what makes "explain" and "compare" answers multi-source. | |
| US5 rejection. The threshold is on the MEAN of the top-N reranker scores, not | |
| the maximum, because the maximum is not discriminative. Measured on this corpus: | |
| query max mean-of-5 | |
| explain JEPA 0.998 0.995 in corpus | |
| how does JEPA differ from MAE 0.898 0.671 in corpus | |
| what causes knee osteoarthritis 0.670 0.190 NOT in corpus | |
| sourdough hydration ratio 0.001 0.001 NOT in corpus | |
| The osteoarthritis query hits 0.670 on max because the corpus really does | |
| contain a self-supervised bone-densitometry paper - one genuine partial match | |
| defeats a max-based threshold. The mean separates cleanly, because a query that | |
| belongs to this corpus matches *many* passages, not one. | |
| Scale note: CrossEncoder.predict applies a sigmoid, so torch-backend scores are | |
| probabilities in [0,1]. The ONNX backend returns raw logits - rescale if you | |
| switch backends. | |
| Citation resolution is local-first, then arXiv on demand, so the tool is not | |
| limited to whatever was ingested. | |
| """ | |
| from __future__ import annotations | |
| import re | |
| import threading | |
| import time | |
| from collections import OrderedDict | |
| from concurrent.futures import FIRST_COMPLETED, ThreadPoolExecutor | |
| from concurrent.futures import wait as _futures_wait | |
| from typing import Any | |
| from src.api.diagnose import diagnose | |
| from src.api.llm import ( | |
| explain_paper, general_answer, generate_answer, is_configured, is_not_covered, | |
| ) | |
| from src.config import device | |
| from src.data.arxiv_source import ( | |
| ArxivUnavailable, Paper, fetch_metadata_bounded, read_papers, | |
| ) | |
| from src.data import openalex_source as openalex | |
| from src.data.chunking import chunk_paper, chunks_to_corpus, read_chunks | |
| from src.data.citation import looks_like_a_question, paper_url, resolve | |
| from src.retrieval.bm25 import BM25Retriever | |
| from src.retrieval.dense import DenseRetriever | |
| from src.retrieval.diversify import cap_per_paper, distinct_papers | |
| from src.retrieval.fusion import reciprocal_rank_fusion | |
| PSEUDO_QID = "q" | |
| # Live sources run side by side under one deadline; their own clients bound | |
| # each call, so this pool only has to hold the two waits of one request. | |
| _LIVE_POOL = ThreadPoolExecutor(max_workers=8, thread_name_prefix="live-search") | |
| _SEARCH_POOL = ThreadPoolExecutor(max_workers=2, thread_name_prefix="first-stage") | |
| RANK_CACHE_MAX = 512 | |
| def _dedup_key(item: dict[str, Any]) -> str: | |
| """arXiv id when known, else a normalised title: one paper, one entry.""" | |
| pid = item.get("paper_id") or "" | |
| if pid and not pid.startswith("openalex:"): | |
| return pid | |
| return re.sub(r"[^a-z0-9]", "", (item.get("title") or "").lower())[:120] | |
| _QUERY_STOPWORDS = { | |
| "the", "and", "for", "with", "from", "what", "how", "why", "does", "explain", | |
| "when", "are", "was", "that", "this", "there", "which", "about", "into", | |
| "some", "can", "you", "its", "their", "using", "used", "based", | |
| } | |
| class MLPipeline: | |
| """Loads chunk corpus and indices once, then answers questions.""" | |
| def __init__(self, cfg: dict[str, Any]) -> None: | |
| self.cfg = cfg | |
| paths, rc = cfg["paths"], cfg["retrieval"] | |
| chunks = read_chunks(paths["chunks"]) | |
| self.corpus = chunks_to_corpus(chunks) | |
| self.chunks_by_paper: dict[str, list[str]] = {} | |
| for chunk in chunks: | |
| self.chunks_by_paper.setdefault(chunk.paper_id, []).append(chunk.chunk_id) | |
| # Full paper records, for citation resolution and metadata display. | |
| self.papers_by_id = {p.paper_id: vars(p) for p in read_papers(paths["papers"])} | |
| # An abstract-only paper holds its abstract twice - once in its paper | |
| # record, once as its single passage - as two equal strings. At 270k | |
| # papers that is ~400 MB of RAM for nothing. Point both at one object. | |
| for entry in self.corpus.values(): | |
| if entry["section"] == "abstract": | |
| record = self.papers_by_id.get(entry["paper_id"]) | |
| if record is not None and record.get("abstract") == entry["text"]: | |
| entry["text"] = record["abstract"] | |
| doc_ids = list(self.corpus.keys()) | |
| doc_texts = [ | |
| f"{self.corpus[d]['title']} {self.corpus[d]['text']}".strip() for d in doc_ids | |
| ] | |
| # The lexical arm is mandatory here: bge-small has frozen weights and has | |
| # never seen "JEPA" or "GRPO", while BM25 gives a rare token a very high | |
| # IDF weight. It is the only defence against vocabulary coined after the | |
| # embedding model was trained. | |
| self.bm25 = BM25Retriever( | |
| k1=rc["bm25"]["k1"], b=rc["bm25"]["b"], stemmer_lang=rc["bm25"]["stemmer"] | |
| ) | |
| self.bm25.index(doc_ids, doc_texts, cache_dir=paths["index_cache"]) | |
| self.dense = DenseRetriever( | |
| model_name=rc["dense"]["model"], | |
| batch_size=rc["dense"]["batch_size"], | |
| max_length=rc["dense"]["max_length"], | |
| query_prefix=rc["dense"]["query_prefix"], | |
| device=device(), | |
| ) | |
| self.dense.index_corpus(doc_ids, doc_texts, cache_dir=paths["index_cache"]) | |
| from src.rerank.model import Reranker | |
| mc = cfg["reranker"] | |
| self.reranker = Reranker( | |
| model_name_or_path=mc["model"], | |
| max_length=mc["max_length"], | |
| batch_size=mc["batch_size"], | |
| device=device(), | |
| fp16=mc.get("fp16", True) and device() == "cuda", | |
| ) | |
| # One GPU, one model, many HTTP threads. FastAPI runs sync endpoints in | |
| # a threadpool, so without this two concurrent users would drive | |
| # forward passes through the same CUDA module simultaneously - which | |
| # risks wrong results and VRAM spikes. Retrieval is fast enough | |
| # (sub-second) that serialising it costs little. | |
| self._gpu_lock = threading.Lock() | |
| self.rrf_k = rc["fusion"]["rrf_k"] | |
| self.retrieve_k = rc["top_k"] | |
| self._rank_cache: OrderedDict[str, dict[str, float]] = OrderedDict() | |
| self._cache_lock = threading.Lock() | |
| self.cache_hits = 0 | |
| self.early_exits = 0 | |
| print( | |
| f"[ml] ready | {len(self.corpus)} chunks from {len(self.papers_by_id)} papers " | |
| f"| reranker={mc['model']} | llm={'groq' if is_configured() else 'disabled'}" | |
| ) | |
| def warmup(self) -> None: | |
| """Run one throwaway query so the first real user does not pay for it. | |
| The first call into torch, faiss and the tokenizers allocates buffers | |
| and compiles kernels; on a CPU container that is seconds the first | |
| visitor would otherwise sit through. The result is cached, harmlessly. | |
| """ | |
| t0 = time.perf_counter() | |
| try: | |
| self._ranked("how do transformers use attention for language modelling") | |
| print(f"[ml] warmed up in {time.perf_counter() - t0:.1f}s", flush=True) | |
| except Exception as exc: # noqa: BLE001 - warmup is an optimisation only | |
| print(f"[ml] warmup skipped ({type(exc).__name__})", flush=True) | |
| # ---------------------------------------------------------------- helpers | |
| def _passage(self, chunk_id: str, score: float) -> dict[str, Any]: | |
| entry = self.corpus[chunk_id] | |
| return { | |
| "chunk_id": chunk_id, | |
| "doc_id": chunk_id, | |
| "paper_id": entry["paper_id"], | |
| "arxiv_url": f"https://arxiv.org/abs/{entry['paper_id']}", | |
| "score": float(score), | |
| "title": entry["title"], | |
| "section": entry["section"], | |
| "published": entry["published"], | |
| "text": entry["text"], | |
| } | |
| def _confidence(self, scores: list[float]) -> tuple[float, float, bool]: | |
| ac = self.cfg["answer"] | |
| best = max(scores, default=0.0) | |
| n = min(ac.get("confidence_top_n", 5), len(scores)) | |
| mean_top = sum(sorted(scores, reverse=True)[:n]) / n if n else 0.0 | |
| return best, mean_top, mean_top >= ac["min_mean_score"] | |
| # ------------------------------------------------------------- retrieval | |
| def _ranked(self, question: str) -> dict[str, float]: | |
| """Reranked candidate chunks for a query: {chunk_id: score}. | |
| The one place a query is searched, so every endpoint shares its two | |
| latency wins: | |
| * BM25 and dense search run side by side. They are independent, and | |
| both spend their time in native code (numpy, faiss, torch) that | |
| releases the GIL, so the cost is the slower of the two, not the sum. | |
| * Results are cached. The corpus is static and every stage is | |
| deterministic, so a repeated query - a demo, a retry, /papers then | |
| /ask on the same text, a shared link - skips search AND the | |
| reranker, which is the dominant cost on CPU. | |
| """ | |
| key = " ".join(question.lower().split()) | |
| with self._cache_lock: | |
| hit = self._rank_cache.get(key) | |
| if hit is not None: | |
| self._rank_cache.move_to_end(key) | |
| self.cache_hits += 1 | |
| return dict(hit) | |
| query_map = {PSEUDO_QID: question} | |
| with self._gpu_lock: | |
| bm25_future = _SEARCH_POOL.submit(self.bm25.search, query_map, self.retrieve_k) | |
| dense_run = self.dense.search(query_map, top_k=self.retrieve_k) | |
| fused = reciprocal_rank_fusion( | |
| [bm25_future.result(), dense_run], k=self.rrf_k, top_k=self.retrieve_k | |
| ) | |
| ranked = self._rerank_adaptive(query_map, fused[PSEUDO_QID]) | |
| with self._cache_lock: | |
| self._rank_cache[key] = dict(ranked) | |
| while len(self._rank_cache) > RANK_CACHE_MAX: | |
| self._rank_cache.popitem(last=False) | |
| return ranked | |
| def _rerank_adaptive(self, query_map: dict[str, str], fused: dict[str, float]) -> dict[str, float]: | |
| """Cross-encoder scores for the fused candidates, stopping early when sure. | |
| The cross-encoder is the dominant cost of a search and scales with the | |
| candidates scored (seconds on a 2-vCPU CPU). Score the best `first` | |
| fused candidates; if their top-5 mean already clears `tau`, the answer | |
| is obvious and the remaining candidates are not worth scoring. | |
| Otherwise score the rest, so a hard query still sees every candidate. | |
| Measured offline (scripts/eval_early_exit.py): recall@10 unchanged, | |
| ~27% fewer candidates scored. | |
| """ | |
| ee = self.cfg["retrieval"].get("early_exit") | |
| if not ee or len(fused) <= ee["first"]: | |
| reranked, _ = self.reranker.rerank(query_map, {PSEUDO_QID: fused}, self.corpus, | |
| top_k=self.retrieve_k) | |
| return reranked[PSEUDO_QID] | |
| order = sorted(fused.items(), key=lambda kv: kv[1], reverse=True) | |
| head = dict(order[: ee["first"]]) | |
| scores, _ = self.reranker.rerank(query_map, {PSEUDO_QID: head}, self.corpus, | |
| top_k=len(head)) | |
| scores = scores[PSEUDO_QID] | |
| top = sorted(scores.values(), reverse=True)[:5] | |
| if sum(top) / len(top) >= ee["tau"]: | |
| self.early_exits += 1 | |
| return scores | |
| tail = dict(order[ee["first"]:]) | |
| rest, _ = self.reranker.rerank(query_map, {PSEUDO_QID: tail}, self.corpus, | |
| top_k=len(tail)) | |
| return {**scores, **rest[PSEUDO_QID]} | |
| def retrieve(self, question: str, top_k: int | None = None) -> list[dict[str, Any]]: | |
| """Hybrid retrieve, fuse, rerank, then cap chunks per paper.""" | |
| reranked = {PSEUDO_QID: self._ranked(question)} | |
| ac = self.cfg["answer"] | |
| selected = cap_per_paper( | |
| reranked[PSEUDO_QID], | |
| self.corpus, | |
| top_k=top_k or ac["top_k_passages"], | |
| max_per_paper=ac["max_per_paper"], | |
| ) | |
| return [self._passage(cid, s) for cid, s in selected] | |
| # -------------------------------------------------------- paper discovery | |
| def find_papers(self, topic: str, top_k: int = 10) -> dict[str, Any]: | |
| """Topic in, ranked *papers* out. | |
| The ask path returns passages, which is right for answering a question | |
| and wrong for "show me work on X" - you get five paragraphs where you | |
| wanted five papers. Here the per-paper cap is deliberately skipped, a | |
| wide candidate set is reranked, and results are then grouped by paper. | |
| A paper is scored by its single best passage. Summing would reward long | |
| papers with many mediocre chunks over short papers with one excellent | |
| one, which inverts what you actually want from a search result. | |
| """ | |
| start = time.perf_counter() | |
| reranked = {PSEUDO_QID: self._ranked(topic)} | |
| grouped: dict[str, dict[str, Any]] = {} | |
| for chunk_id, score in reranked[PSEUDO_QID].items(): | |
| entry = self.corpus[chunk_id] | |
| pid = entry["paper_id"] | |
| bucket = grouped.setdefault( | |
| pid, {"score": float("-inf"), "matches": 0, "best_chunk": None, "sections": []} | |
| ) | |
| bucket["matches"] += 1 | |
| bucket["sections"].append(entry["section"]) | |
| if score > bucket["score"]: | |
| bucket["score"] = float(score) | |
| bucket["best_chunk"] = chunk_id | |
| ordered = sorted(grouped.items(), key=lambda kv: kv[1]["score"], reverse=True)[:top_k] | |
| papers: list[dict[str, Any]] = [] | |
| for pid, info in ordered: | |
| meta = self.papers_by_id.get(pid, {}) | |
| best = self.corpus[info["best_chunk"]] | |
| papers.append( | |
| { | |
| "paper_id": pid, | |
| "doc_id": info["best_chunk"], | |
| "title": meta.get("title") or best["title"], | |
| "authors": (meta.get("authors") or [])[:6], | |
| "abstract": meta.get("abstract", ""), | |
| "published": meta.get("published") or best["published"], | |
| "primary_category": meta.get("primary_category", ""), | |
| "arxiv_url": f"https://arxiv.org/abs/{pid}", | |
| "score": round(info["score"], 4), | |
| "matched_passages": info["matches"], | |
| "matched_sections": sorted(set(info["sections"]))[:5], | |
| "why": best["text"][:400], | |
| "why_section": best["section"], | |
| "has_full_text": bool(meta.get("full_text")), | |
| } | |
| ) | |
| scores = [p["score"] for p in papers] | |
| best_score, mean_top, confident = self._confidence(scores) | |
| # Same live fallback the ask path has. Without it, Find papers is | |
| # limited to the ingested sample - and the corpus holds roughly 1,400 | |
| # papers per category against the 50-150/day arXiv publishes, so a | |
| # topic can be well covered on arXiv and absent here. Searching "economic | |
| # world models" returned unrelated robotics papers for exactly this | |
| # reason while the paper sat on arXiv. | |
| source = "corpus" | |
| ac = self.cfg["answer"] | |
| if not confident and self.cfg["citation"].get("allow_remote", True): | |
| live = self.search_live(topic, top_k=top_k) | |
| if live: | |
| live = self._rescore_live(topic, live) | |
| floor = ac.get("live_min_score", 0.30) | |
| kept = [p for p in live if p["score"] >= floor] | |
| if kept: | |
| seen = {p["paper_id"] for p in papers} | |
| fetched = [ | |
| { | |
| "paper_id": p["paper_id"], | |
| "doc_id": p["doc_id"], | |
| "title": p["title"], | |
| "authors": p.get("authors", []), | |
| "abstract": p["text"], | |
| "published": p["published"], | |
| "primary_category": "", | |
| "arxiv_url": p["arxiv_url"], | |
| "venue": p.get("venue", ""), | |
| "cited_by_count": p.get("cited_by_count"), | |
| "live_source": p.get("live_source", "arxiv"), | |
| "score": round(p["score"], 4), | |
| "matched_passages": 1, | |
| "matched_sections": ["abstract"], | |
| "why": p["text"][:400], | |
| "why_section": "abstract", | |
| "has_full_text": False, | |
| "from_arxiv": True, | |
| } | |
| for p in kept | |
| if p["paper_id"] not in seen | |
| ] | |
| if fetched: | |
| # Corpus papers only fill remaining slots if they clear | |
| # the same bar - otherwise a 0.08 match sat under a | |
| # result labelled confident. | |
| papers = (fetched + [p for p in papers if p["score"] >= floor])[:top_k] | |
| source = "arxiv_live" | |
| confident = True | |
| mean_top = sum(p["score"] for p in fetched) / len(fetched) | |
| best_score = max(p["score"] for p in fetched) | |
| # v2: nothing convincing in the corpus or on arXiv - diagnose why. | |
| diagnosis, diagnosis_ms = None, 0.0 | |
| if not confident: | |
| diagnosis, diagnosis_ms = self._diagnose(topic, papers, None) | |
| return { | |
| "topic": topic, | |
| "source": source, | |
| "papers": papers, | |
| "confident": confident, | |
| "top_score": round(best_score, 3), | |
| "mean_top_score": round(mean_top, 3), | |
| "diagnosis": diagnosis, | |
| "timing_ms": {"total": round((time.perf_counter() - start) * 1000.0, 1), | |
| "diagnosis": round(diagnosis_ms, 1)}, | |
| } | |
| def _rescore_live( | |
| self, question: str, live: list[dict[str, Any]] | |
| ) -> list[dict[str, Any]]: | |
| """Score live arXiv abstracts with the same reranker the corpus uses. | |
| arXiv's own relevance ranking is lexical and generous. Running the | |
| cross-encoder over the abstracts puts live results on the same scale as | |
| corpus results, so one confidence threshold governs both. | |
| """ | |
| pairs = [(question, f"{p['title']} {p['text']}".strip()) for p in live] | |
| if not pairs: | |
| return [] | |
| with self._gpu_lock: | |
| scores = self.reranker.score(pairs) | |
| for passage, score in zip(live, scores): | |
| passage["score"] = float(score) | |
| return sorted(live, key=lambda p: p["score"], reverse=True) | |
| def answer_from_document( | |
| self, question: str, passages: list[dict[str, Any]], title: str, generate: bool = True | |
| ) -> dict[str, Any]: | |
| """Answer a question against a single uploaded paper. | |
| Every chunk of the document is scored directly by the cross-encoder - | |
| no BM25, no embeddings, no index. A paper is only 20-40 chunks, so | |
| exhaustive scoring is both cheap and strictly better than retrieval, | |
| which can only approximate it. | |
| """ | |
| start = time.perf_counter() | |
| if not passages: | |
| return { | |
| "question": question, | |
| "answer": "That document has no readable passages.", | |
| "confident": False, | |
| "source": "upload", | |
| "top_score": 0.0, | |
| "mean_top_score": 0.0, | |
| "citations": [], | |
| "distinct_papers": 0, | |
| "generation_configured": is_configured(), | |
| "timing_ms": {"retrieval_and_rerank": 0.0, "generation": 0.0, "total": 0.0}, | |
| } | |
| pairs = [(question, f"{p['title']} {p['text']}".strip()) for p in passages] | |
| with self._gpu_lock: | |
| scores = self.reranker.score(pairs) | |
| for passage, score in zip(passages, scores): | |
| passage["score"] = float(score) | |
| ranked = sorted(passages, key=lambda p: p["score"], reverse=True) | |
| ac = self.cfg["answer"] | |
| top = ranked[: ac["top_k_passages"]] | |
| best, mean_top, _ = self._confidence([p["score"] for p in top]) | |
| retrieval_ms = (time.perf_counter() - start) * 1000.0 | |
| answer_text: str | None = None | |
| generation_ms = 0.0 | |
| if generate: | |
| gen_start = time.perf_counter() | |
| answer_text = generate_answer( | |
| f"About the paper '{title}': {question}", | |
| top, | |
| model=self._answer_models(), | |
| max_tokens=ac["max_tokens"], | |
| ) | |
| generation_ms = (time.perf_counter() - gen_start) * 1000.0 | |
| return { | |
| "question": question, | |
| "answer": answer_text, | |
| # The user chose this document, so we answer from it even when the | |
| # match is weak - declining would be second-guessing them. | |
| "confident": True, | |
| "source": "upload", | |
| "top_score": round(best, 3), | |
| "mean_top_score": round(mean_top, 3), | |
| "citations": top, | |
| "distinct_papers": 1, | |
| "generation_configured": is_configured(), | |
| "timing_ms": { | |
| "retrieval_and_rerank": round(retrieval_ms, 1), | |
| "generation": round(generation_ms, 1), | |
| "total": round(retrieval_ms + generation_ms, 1), | |
| }, | |
| } | |
| # ------------------------------------------------------------ live search | |
| def search_live(self, question: str, top_k: int = 6) -> list[dict[str, Any]]: | |
| """Search arXiv and OpenAlex live, for questions the corpus cannot answer. | |
| The corpus is a sample, not coverage: ~1,400 papers per category across | |
| nine months, against the 50-150/day arXiv actually publishes. Most | |
| papers a user asks about were never ingested, and telling them "not in | |
| this corpus" is honest but useless when the paper plainly exists. | |
| Two sources, because each misses what the other has. arXiv is the | |
| freshest for preprints; OpenAlex adds conference and journal papers | |
| that never had one. Both run in parallel under ONE wall-clock budget | |
| (`citation.live_budget_s`), so adding a source never adds latency: | |
| whatever has answered when the budget expires is used, and a source | |
| that is down or paused costs nothing. The same paper found by both | |
| collapses to one entry, keyed on its arXiv id or normalised title. | |
| Abstracts only - no PDF fetch. Results are rescored by the corpus | |
| reranker afterwards, so one confidence bar governs every source. | |
| """ | |
| sources = self.cfg["citation"].get("live_sources", ["arxiv", "openalex"]) | |
| budget = float(self.cfg["citation"].get("live_budget_s", 5.0)) | |
| deadline = time.monotonic() + budget | |
| jobs = {} | |
| if "arxiv" in sources: | |
| jobs["arxiv"] = _LIVE_POOL.submit(self._live_arxiv, question, top_k, deadline) | |
| if "openalex" in sources: | |
| jobs["openalex"] = _LIVE_POOL.submit(self._live_openalex, question, top_k, deadline) | |
| # Do not wait for the slowest source. Once one has results, the other | |
| # gets a short grace and is then left behind: arXiv often takes 3-5 s | |
| # while OpenAlex answers in ~1 s, and its extra papers are not worth | |
| # holding the whole response. A source that returns nothing (down, | |
| # paused, empty) does not start the clock - the other keeps its full | |
| # budget. | |
| grace = float(self.cfg["citation"].get("live_grace_s", 0.75)) | |
| found: dict[str, list[dict[str, Any]]] = {} | |
| name_of = {job: name for name, job in jobs.items()} | |
| pending = set(jobs.values()) | |
| cutoff = deadline | |
| while pending and time.monotonic() < cutoff: | |
| done, pending = _futures_wait( | |
| pending, timeout=max(cutoff - time.monotonic(), 0.01), | |
| return_when=FIRST_COMPLETED) | |
| for job in done: | |
| try: | |
| found[name_of[job]] = job.result() | |
| except Exception: # noqa: BLE001 - a failed source is an empty source | |
| found[name_of[job]] = [] | |
| if found[name_of[job]]: | |
| cutoff = min(cutoff, time.monotonic() + grace) | |
| merged: list[dict[str, Any]] = [] | |
| seen: dict[str, dict[str, Any]] = {} | |
| # arXiv first: on a duplicate, its record wins (fresher version), and | |
| # OpenAlex contributes the venue and citation count it lacks. | |
| for name in ("arxiv", "openalex"): | |
| for item in found.get(name, []): | |
| key = _dedup_key(item) | |
| if key in seen: | |
| kept = seen[key] | |
| kept["venue"] = kept.get("venue") or item.get("venue", "") | |
| kept["cited_by_count"] = max(kept.get("cited_by_count") or 0, | |
| item.get("cited_by_count") or 0) | |
| kept["live_source"] = "arxiv+openalex" | |
| continue | |
| seen[key] = item | |
| merged.append(item) | |
| return merged | |
| # Kept for callers that predate OpenAlex. | |
| search_arxiv_live = search_live | |
| def _live_arxiv(self, question: str, top_k: int, deadline: float) -> list[dict[str, Any]]: | |
| terms = [w for w in re.findall(r"[A-Za-z][A-Za-z0-9-]{2,}", question) | |
| if w.lower() not in _QUERY_STOPWORDS][:12] | |
| if not terms: | |
| return [] | |
| queries = [ | |
| "all:" + " AND ".join(f'"{t}"' for t in terms[:4]), # precise first | |
| "all:" + " ".join(terms[:8]), # then looser | |
| ] | |
| found: list[Paper] = [] | |
| for query in queries: | |
| remaining = deadline - time.monotonic() | |
| if remaining <= 0.25: | |
| break | |
| try: | |
| found = fetch_metadata_bounded(query, max_results=top_k, | |
| budget_s=remaining) | |
| except ArxivUnavailable: | |
| return [] # a failed enrichment is not a failed request | |
| if found: | |
| break | |
| return [ | |
| { | |
| "chunk_id": f"live:{p.paper_id}", | |
| "doc_id": f"live:{p.paper_id}", | |
| "paper_id": p.paper_id, | |
| "arxiv_url": f"https://arxiv.org/abs/{p.paper_id}", | |
| "score": 1.0, | |
| "title": p.title, | |
| "section": "abstract", | |
| "published": p.published, | |
| "text": p.abstract, | |
| "authors": p.authors[:8], | |
| "venue": "arXiv", | |
| "cited_by_count": None, | |
| "live_source": "arxiv", | |
| } | |
| for p in found | |
| ] | |
| def _live_openalex(self, question: str, top_k: int, deadline: float) -> list[dict[str, Any]]: | |
| remaining = deadline - time.monotonic() | |
| if remaining <= 0.25: | |
| return [] | |
| try: | |
| works = openalex.search_works(question, max_results=top_k, budget_s=remaining) | |
| except openalex.OpenAlexUnavailable: | |
| return [] | |
| return [ | |
| { | |
| "chunk_id": f"live:{w['paper_id']}", | |
| "doc_id": f"live:{w['paper_id']}", | |
| "paper_id": w["paper_id"], | |
| # Named arxiv_url for compatibility with every client; for a | |
| # paper with no arXiv copy it is the DOI or landing page. | |
| "arxiv_url": w["url"], | |
| "score": 1.0, | |
| "title": w["title"], | |
| "section": "abstract", | |
| "published": w["published"], | |
| "text": w["abstract"], | |
| "authors": w["authors"][:8], | |
| "venue": w["venue"], | |
| "cited_by_count": w["cited_by_count"], | |
| "doi": w["doi"], | |
| "live_source": "openalex", | |
| } | |
| for w in works | |
| ] | |
| # ------------------------------------------------------------------- ask | |
| def answer(self, question: str, generate: bool = True) -> dict[str, Any]: | |
| """Question answering, including the US5 rejection path.""" | |
| start = time.perf_counter() | |
| passages = self.retrieve(question) | |
| retrieval_ms = (time.perf_counter() - start) * 1000.0 | |
| # Kept for diagnosis: if everything below fails, these weak corpus hits | |
| # are exactly what the critic needs to see to say what went wrong. | |
| corpus_passages = passages | |
| ac = self.cfg["answer"] | |
| best, mean_top, confident = self._confidence([p["score"] for p in passages]) | |
| # Blend in live arXiv results when the corpus alone is thin. | |
| # | |
| # The corpus is a sample (~1,400 papers per category against the 50-150 | |
| # a day arXiv publishes), so a question can be perfectly answerable from | |
| # the literature and unanswerable from here. Two triggers: | |
| # | |
| # below min_mean_score the corpus has nothing - live results replace | |
| # below enrich_below the corpus has something thin - live results | |
| # are added, so the answer draws on both | |
| # | |
| # A strong corpus hit skips the network entirely and stays sub-second. | |
| answer_text: str | None = None | |
| generation_ms = 0.0 | |
| source = "corpus" | |
| remote_ok = self.cfg["citation"].get("allow_remote", True) | |
| enrich_below = ac.get("enrich_below", 0.0) | |
| # Timed on its own: this was the one stage outside every timer, which is | |
| # how a 45s arXiv stall reported itself as "retrieval_and_rerank: 1783ms". | |
| live_ms = 0.0 | |
| if remote_ok and (not confident or mean_top < enrich_below): | |
| live_start = time.perf_counter() | |
| live = self.search_live(question) | |
| live_ms = (time.perf_counter() - live_start) * 1000.0 | |
| if live: | |
| # arXiv always returns *something*, so live results must clear | |
| # the same confidence bar as the corpus. Without this the | |
| # honest "I do not know" is replaced by confident nonsense - | |
| # a sourdough question came back with quantum-physics papers. | |
| # Filter live results PER PASSAGE, not on their mean. | |
| # | |
| # The mean-of-top-5 rule works for the corpus because a query | |
| # that belongs to it matches many passages. An arXiv keyword | |
| # search returns a mixed bag by construction - three loose hits | |
| # would drag the mean under and bury the one paper that is | |
| # actually the answer. So each live passage clears its own bar. | |
| live = self._rescore_live(question, live) | |
| floor = ac.get("live_min_score", 0.30) | |
| live = [p for p in live if p["score"] >= floor] | |
| seen = {p["paper_id"] for p in passages} | |
| extra = [p for p in live if p["paper_id"] not in seen] | |
| if confident and extra: | |
| passages = passages + extra # enrich | |
| source = "corpus+arxiv_live" | |
| elif extra: | |
| passages = extra # replace | |
| source = "arxiv_live" | |
| confident = True | |
| mean_top = sum(p["score"] for p in extra) / len(extra) | |
| elif not confident: | |
| passages = [] # genuinely unanswerable | |
| # Answer from the papers when they cover the question. When they do | |
| # not - nothing retrieved, or the answerer replies NOT_COVERED - fall | |
| # back to the model's own knowledge, but only for AI questions and | |
| # labelled as not grounded in the papers. Everything else is declined. | |
| grounded = True | |
| uncovered = not passages | |
| if passages and generate: | |
| gen_start = time.perf_counter() | |
| answer_text = generate_answer( | |
| question, passages, model=self._answer_models(), max_tokens=ac["max_tokens"] | |
| ) | |
| generation_ms = (time.perf_counter() - gen_start) * 1000.0 | |
| if is_not_covered(answer_text): | |
| uncovered, answer_text = True, None | |
| if uncovered and generate and ac.get("general_knowledge", True): | |
| gen_start = time.perf_counter() | |
| try: | |
| general = general_answer(question, model=self._answer_models(), | |
| max_tokens=ac["max_tokens"]) | |
| except Exception as exc: # noqa: BLE001 - fall back to the plain decline | |
| print(f"[ml] general answer failed ({type(exc).__name__})", flush=True) | |
| general = None | |
| generation_ms += (time.perf_counter() - gen_start) * 1000.0 | |
| if general: | |
| answer_text, source, grounded, confident = general, "general_knowledge", False, False | |
| if uncovered and answer_text is None: | |
| answer_text = ( | |
| "Nothing answers that - not in the local corpus, and not in a live " | |
| "arXiv search either. If this is an AI/ML topic the paper may simply " | |
| "be too new or too obscure; if it is not, this tool only covers AI research." | |
| if not passages else | |
| "The papers found for this do not contain an answer to it." | |
| ) | |
| # v2: the gate fired and nothing rescued it, so say what went wrong. | |
| diagnosis, diagnosis_ms = None, 0.0 | |
| if not passages and source != "general_knowledge": | |
| diagnosis, diagnosis_ms = self._diagnose(question, corpus_passages, None) | |
| return { | |
| "question": question, | |
| "answer": answer_text, | |
| "confident": confident, | |
| "source": source, | |
| "grounded": grounded, | |
| "top_score": round(best, 3), | |
| "mean_top_score": round(mean_top, 3), | |
| "citations": passages, | |
| "distinct_papers": distinct_papers(passages), | |
| "generation_configured": is_configured(), | |
| "diagnosis": diagnosis, | |
| "timing_ms": { | |
| "retrieval_and_rerank": round(retrieval_ms, 1), | |
| "arxiv_live": round(live_ms, 1), | |
| "generation": round(generation_ms, 1), | |
| "diagnosis": round(diagnosis_ms, 1), | |
| "total": round(retrieval_ms + live_ms + generation_ms + diagnosis_ms, 1), | |
| }, | |
| } | |
| def _answer_models(self) -> list[str]: | |
| """Primary answer model, then fallbacks for when its daily cap is spent.""" | |
| ac = self.cfg["answer"] | |
| return [ac["model"], *ac.get("fallback_models", [])] | |
| def _diagnose(self, question: str, passages: list[dict[str, Any]], | |
| answer: str | None) -> tuple[dict[str, Any] | None, float]: | |
| """Run the v2 failure diagnosis, if enabled. Never raises.""" | |
| dc = self.cfg.get("diagnosis", {}) | |
| if not dc.get("enabled", True): | |
| return None, 0.0 | |
| t0 = time.perf_counter() | |
| d = diagnose(question, passages, answer, | |
| model=dc.get("models") or dc.get("model", self.cfg["answer"]["model"]), | |
| budget_s=float(dc.get("budget_s", 6.0))) | |
| return (d.to_json() if d else None), (time.perf_counter() - t0) * 1000.0 | |
| # ------------------------------------------------------- citation lookup | |
| def _passages_for_paper(self, paper: Paper, from_corpus: bool) -> list[dict[str, Any]]: | |
| """Passages describing one specific paper. | |
| Indexed papers use their stored chunks. A paper fetched on demand is | |
| chunked in memory - it is not added to the index, because rebuilding the | |
| index per lookup would be far slower than the lookup itself. | |
| """ | |
| if from_corpus and paper.paper_id in self.chunks_by_paper: | |
| chunk_ids = self.chunks_by_paper[paper.paper_id] | |
| ordered = sorted(chunk_ids, key=lambda c: self.corpus[c]["section"] != "abstract") | |
| return [self._passage(cid, 1.0) for cid in ordered[:8]] | |
| chunks = chunk_paper( | |
| paper, | |
| max_words=self.cfg["chunking"]["max_words"], | |
| overlap_words=self.cfg["chunking"]["overlap_words"], | |
| min_words=self.cfg["chunking"]["min_words"], | |
| )[:8] | |
| return [ | |
| { | |
| "chunk_id": c.chunk_id, | |
| "doc_id": c.chunk_id, | |
| "paper_id": c.paper_id, | |
| "arxiv_url": paper_url(paper), | |
| "score": 1.0, | |
| "title": c.title, | |
| "section": c.section, | |
| "published": c.published, | |
| "text": c.text, | |
| } | |
| for c in chunks | |
| ] | |
| def explain_citation(self, citation: str, generate: bool = True) -> dict[str, Any]: | |
| """Resolve a citation string to a paper and explain it. | |
| Returns the explanation plus `related`: what else in the corpus talks | |
| about this paper, which is where the "and here is the wider context" | |
| half of the answer comes from. | |
| """ | |
| start = time.perf_counter() | |
| # A question is not a citation. Trying to resolve one costs two arXiv | |
| # searches that cannot succeed, and arXiv answers 429 - which is how a | |
| # concept question turned into a minute of waiting and then a failure. | |
| if looks_like_a_question(citation): | |
| answer = self.answer(citation, generate=generate) | |
| return { | |
| "citation": citation, | |
| "resolved": False, | |
| "method": "question", | |
| "query_used": citation, | |
| "candidates": [], | |
| "paper": None, | |
| "explanation": answer["answer"], | |
| "passages": answer["citations"], | |
| "related": [], | |
| "answered_as_question": True, | |
| "source": answer["source"], | |
| "generation_configured": is_configured(), | |
| "timing_ms": answer["timing_ms"], | |
| "diagnosis": answer.get("diagnosis"), | |
| # The complete /ask payload, so the client can render it | |
| # directly. Without this the web client discarded the answer | |
| # computed here and called /ask, re-running retrieval, | |
| # reranking and generation - every Explain-mode question cost | |
| # two full pipeline passes and two Groq calls. | |
| "answer_result": answer, | |
| } | |
| resolution = resolve( | |
| citation, | |
| self.papers_by_id, | |
| allow_remote=self.cfg["citation"]["allow_remote"], | |
| fetch_full_text=self.cfg["citation"]["fetch_full_text"], | |
| pdf_dir=self.cfg["paths"]["pdf_cache"], | |
| text_dir=self.cfg["paths"]["text_cache"], | |
| max_cache_gb=self.cfg["citation"].get("max_pdf_cache_gb", 5.0), | |
| ) | |
| if resolution.paper is None: | |
| return { | |
| "citation": citation, | |
| "resolved": False, | |
| "method": resolution.method, | |
| "query_used": resolution.query_used, | |
| "candidates": resolution.candidates, | |
| "explanation": ( | |
| "Could not resolve that citation to a paper. Try the arXiv id, or the " | |
| "paper title on its own." | |
| ), | |
| "paper": None, | |
| "passages": [], | |
| "related": [], | |
| "generation_configured": is_configured(), | |
| "timing_ms": {"total": round((time.perf_counter() - start) * 1000.0, 1)}, | |
| } | |
| paper = resolution.paper | |
| passages = self._passages_for_paper(paper, resolution.from_corpus) | |
| # What else in the corpus discusses this work - the wider-context half. | |
| related_query = f"{paper.title} {paper.abstract[:300]}" | |
| related = [ | |
| p | |
| for p in self.retrieve(related_query, top_k=6) | |
| if p["paper_id"] != paper.paper_id | |
| ][:4] | |
| explanation: str | None = None | |
| gen_ms = 0.0 | |
| if generate: | |
| gen_start = time.perf_counter() | |
| explanation = explain_paper( | |
| paper.title, | |
| passages + related, | |
| model=self._answer_models(), | |
| max_tokens=self.cfg["citation"]["max_tokens"], | |
| ) | |
| gen_ms = (time.perf_counter() - gen_start) * 1000.0 | |
| return { | |
| "citation": citation, | |
| "resolved": True, | |
| "method": resolution.method, | |
| "query_used": resolution.query_used, | |
| "from_corpus": resolution.from_corpus, | |
| "candidates": resolution.candidates, | |
| "paper": { | |
| "paper_id": paper.paper_id, | |
| "title": paper.title, | |
| "abstract": paper.abstract, | |
| "authors": paper.authors[:8], | |
| "published": paper.published, | |
| "primary_category": paper.primary_category, | |
| "arxiv_url": paper_url(paper), | |
| "venue": (paper.metadata or {}).get("venue", ""), | |
| "doi": (paper.metadata or {}).get("doi", ""), | |
| "has_full_text": bool(paper.full_text), | |
| }, | |
| "explanation": explanation, | |
| "passages": passages, | |
| "related": related, | |
| "generation_configured": is_configured(), | |
| "timing_ms": { | |
| "total": round((time.perf_counter() - start) * 1000.0, 1), | |
| "generation": round(gen_ms, 1), | |
| }, | |
| } | |