File size: 43,454 Bytes
e8c5cc2
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
2299777
 
 
e8c5cc2
 
5eec49d
2299777
 
 
e8c5cc2
af38999
 
 
a3c7ac6
e8c5cc2
a3c7ac6
e8c5cc2
 
 
 
 
 
 
a3c7ac6
 
 
2299777
 
a3c7ac6
 
 
 
 
 
 
 
 
e8c5cc2
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
2299777
 
 
 
 
 
 
 
 
e8c5cc2
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
2299777
 
 
 
e8c5cc2
 
 
 
 
2299777
 
 
 
 
 
 
 
 
 
 
 
 
 
e8c5cc2
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
2299777
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
e8c5cc2
 
2299777
e8c5cc2
 
2299777
e8c5cc2
2299777
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
e8c5cc2
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
2299777
e8c5cc2
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
a3c7ac6
e8c5cc2
 
 
 
 
 
 
 
 
 
 
a3c7ac6
e8c5cc2
 
 
 
a3c7ac6
 
 
e8c5cc2
 
 
 
 
 
 
 
 
 
 
 
a3c7ac6
 
 
 
e8c5cc2
 
 
 
 
5eec49d
 
 
 
 
e8c5cc2
 
 
 
 
 
 
5eec49d
 
 
e8c5cc2
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
5eec49d
e8c5cc2
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
a3c7ac6
e8c5cc2
a3c7ac6
 
e8c5cc2
 
 
 
 
 
a3c7ac6
 
 
 
 
 
 
 
 
 
e8c5cc2
a3c7ac6
 
 
 
 
 
 
 
 
2299777
 
 
 
 
 
 
a3c7ac6
2299777
 
 
 
 
 
 
 
 
 
 
 
 
 
a3c7ac6
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
e8c5cc2
 
 
 
af38999
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
e8c5cc2
 
 
 
 
 
 
 
 
 
 
a3c7ac6
 
 
 
e8c5cc2
 
 
 
a3c7ac6
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
e8c5cc2
 
 
 
 
 
 
5eec49d
 
 
e8c5cc2
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
af38999
 
 
e8c5cc2
af38999
a3c7ac6
af38999
e8c5cc2
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
2299777
 
 
 
 
 
 
e8c5cc2
 
5eec49d
e8c5cc2
 
2299777
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
e8c5cc2
5eec49d
 
2299777
5eec49d
 
e8c5cc2
 
 
 
 
2299777
e8c5cc2
 
 
 
 
5eec49d
e8c5cc2
 
af38999
e8c5cc2
5eec49d
 
e8c5cc2
 
 
5eec49d
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
e8c5cc2
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
a3c7ac6
e8c5cc2
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
5eec49d
 
 
 
 
 
 
e8c5cc2
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
5eec49d
e8c5cc2
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
a3c7ac6
 
 
e8c5cc2
 
 
 
 
 
 
 
 
 
 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
534
535
536
537
538
539
540
541
542
543
544
545
546
547
548
549
550
551
552
553
554
555
556
557
558
559
560
561
562
563
564
565
566
567
568
569
570
571
572
573
574
575
576
577
578
579
580
581
582
583
584
585
586
587
588
589
590
591
592
593
594
595
596
597
598
599
600
601
602
603
604
605
606
607
608
609
610
611
612
613
614
615
616
617
618
619
620
621
622
623
624
625
626
627
628
629
630
631
632
633
634
635
636
637
638
639
640
641
642
643
644
645
646
647
648
649
650
651
652
653
654
655
656
657
658
659
660
661
662
663
664
665
666
667
668
669
670
671
672
673
674
675
676
677
678
679
680
681
682
683
684
685
686
687
688
689
690
691
692
693
694
695
696
697
698
699
700
701
702
703
704
705
706
707
708
709
710
711
712
713
714
715
716
717
718
719
720
721
722
723
724
725
726
727
728
729
730
731
732
733
734
735
736
737
738
739
740
741
742
743
744
745
746
747
748
749
750
751
752
753
754
755
756
757
758
759
760
761
762
763
764
765
766
767
768
769
770
771
772
773
774
775
776
777
778
779
780
781
782
783
784
785
786
787
788
789
790
791
792
793
794
795
796
797
798
799
800
801
802
803
804
805
806
807
808
809
810
811
812
813
814
815
816
817
818
819
820
821
822
823
824
825
826
827
828
829
830
831
832
833
834
835
836
837
838
839
840
841
842
843
844
845
846
847
848
849
850
851
852
853
854
855
856
857
858
859
860
861
862
863
864
865
866
867
868
869
870
871
872
873
874
875
876
877
878
879
880
881
882
883
884
885
886
887
888
889
890
891
892
893
894
895
896
897
898
899
900
901
902
903
904
905
906
907
908
909
910
911
912
913
914
915
916
917
918
919
920
921
922
923
924
925
926
927
928
"""ResearchOS-ML pipeline: retrieve -> fuse -> rerank -> diversify -> answer.



Two capabilities:



  ask()              question answering over the ingested corpus

  explain_citation() resolve a citation you saw in a paper, then explain it



Three things differ from the parent project's pipeline, all forced by the corpus:



FR4.2 diversification. One paper produces dozens of chunks, so the top-scoring

five are routinely five paragraphs of the same paper. `max_per_paper` caps that,

which is what makes "explain" and "compare" answers multi-source.



US5 rejection. The threshold is on the MEAN of the top-N reranker scores, not

the maximum, because the maximum is not discriminative. Measured on this corpus:



    query                           max     mean-of-5

    explain JEPA                    0.998     0.995     in corpus

    how does JEPA differ from MAE   0.898     0.671     in corpus

    what causes knee osteoarthritis 0.670     0.190     NOT in corpus

    sourdough hydration ratio       0.001     0.001     NOT in corpus



The osteoarthritis query hits 0.670 on max because the corpus really does

contain a self-supervised bone-densitometry paper - one genuine partial match

defeats a max-based threshold. The mean separates cleanly, because a query that

belongs to this corpus matches *many* passages, not one.



Scale note: CrossEncoder.predict applies a sigmoid, so torch-backend scores are

probabilities in [0,1]. The ONNX backend returns raw logits - rescale if you

switch backends.



Citation resolution is local-first, then arXiv on demand, so the tool is not

limited to whatever was ingested.

"""

from __future__ import annotations

import re
import threading
import time
from collections import OrderedDict
from concurrent.futures import FIRST_COMPLETED, ThreadPoolExecutor
from concurrent.futures import wait as _futures_wait
from typing import Any

from src.api.diagnose import diagnose
from src.api.llm import (
    explain_paper, general_answer, generate_answer, is_configured, is_not_covered,
)
from src.config import device
from src.data.arxiv_source import (
    ArxivUnavailable, Paper, fetch_metadata_bounded, read_papers,
)
from src.data import openalex_source as openalex
from src.data.chunking import chunk_paper, chunks_to_corpus, read_chunks
from src.data.citation import looks_like_a_question, paper_url, resolve
from src.retrieval.bm25 import BM25Retriever
from src.retrieval.dense import DenseRetriever
from src.retrieval.diversify import cap_per_paper, distinct_papers
from src.retrieval.fusion import reciprocal_rank_fusion

PSEUDO_QID = "q"

# Live sources run side by side under one deadline; their own clients bound
# each call, so this pool only has to hold the two waits of one request.
_LIVE_POOL = ThreadPoolExecutor(max_workers=8, thread_name_prefix="live-search")
_SEARCH_POOL = ThreadPoolExecutor(max_workers=2, thread_name_prefix="first-stage")
RANK_CACHE_MAX = 512


def _dedup_key(item: dict[str, Any]) -> str:
    """arXiv id when known, else a normalised title: one paper, one entry."""
    pid = item.get("paper_id") or ""
    if pid and not pid.startswith("openalex:"):
        return pid
    return re.sub(r"[^a-z0-9]", "", (item.get("title") or "").lower())[:120]

_QUERY_STOPWORDS = {
    "the", "and", "for", "with", "from", "what", "how", "why", "does", "explain",
    "when", "are", "was", "that", "this", "there", "which", "about", "into",
    "some", "can", "you", "its", "their", "using", "used", "based",
}


class MLPipeline:
    """Loads chunk corpus and indices once, then answers questions."""

    def __init__(self, cfg: dict[str, Any]) -> None:
        self.cfg = cfg
        paths, rc = cfg["paths"], cfg["retrieval"]

        chunks = read_chunks(paths["chunks"])
        self.corpus = chunks_to_corpus(chunks)
        self.chunks_by_paper: dict[str, list[str]] = {}
        for chunk in chunks:
            self.chunks_by_paper.setdefault(chunk.paper_id, []).append(chunk.chunk_id)

        # Full paper records, for citation resolution and metadata display.
        self.papers_by_id = {p.paper_id: vars(p) for p in read_papers(paths["papers"])}

        # An abstract-only paper holds its abstract twice - once in its paper
        # record, once as its single passage - as two equal strings. At 270k
        # papers that is ~400 MB of RAM for nothing. Point both at one object.
        for entry in self.corpus.values():
            if entry["section"] == "abstract":
                record = self.papers_by_id.get(entry["paper_id"])
                if record is not None and record.get("abstract") == entry["text"]:
                    entry["text"] = record["abstract"]

        doc_ids = list(self.corpus.keys())
        doc_texts = [
            f"{self.corpus[d]['title']} {self.corpus[d]['text']}".strip() for d in doc_ids
        ]

        # The lexical arm is mandatory here: bge-small has frozen weights and has
        # never seen "JEPA" or "GRPO", while BM25 gives a rare token a very high
        # IDF weight. It is the only defence against vocabulary coined after the
        # embedding model was trained.
        self.bm25 = BM25Retriever(
            k1=rc["bm25"]["k1"], b=rc["bm25"]["b"], stemmer_lang=rc["bm25"]["stemmer"]
        )
        self.bm25.index(doc_ids, doc_texts, cache_dir=paths["index_cache"])

        self.dense = DenseRetriever(
            model_name=rc["dense"]["model"],
            batch_size=rc["dense"]["batch_size"],
            max_length=rc["dense"]["max_length"],
            query_prefix=rc["dense"]["query_prefix"],
            device=device(),
        )
        self.dense.index_corpus(doc_ids, doc_texts, cache_dir=paths["index_cache"])

        from src.rerank.model import Reranker

        mc = cfg["reranker"]
        self.reranker = Reranker(
            model_name_or_path=mc["model"],
            max_length=mc["max_length"],
            batch_size=mc["batch_size"],
            device=device(),
            fp16=mc.get("fp16", True) and device() == "cuda",
        )

        # One GPU, one model, many HTTP threads. FastAPI runs sync endpoints in
        # a threadpool, so without this two concurrent users would drive
        # forward passes through the same CUDA module simultaneously - which
        # risks wrong results and VRAM spikes. Retrieval is fast enough
        # (sub-second) that serialising it costs little.
        self._gpu_lock = threading.Lock()
        self.rrf_k = rc["fusion"]["rrf_k"]
        self.retrieve_k = rc["top_k"]
        self._rank_cache: OrderedDict[str, dict[str, float]] = OrderedDict()
        self._cache_lock = threading.Lock()
        self.cache_hits = 0
        self.early_exits = 0
        print(
            f"[ml] ready | {len(self.corpus)} chunks from {len(self.papers_by_id)} papers "
            f"| reranker={mc['model']} | llm={'groq' if is_configured() else 'disabled'}"
        )

    def warmup(self) -> None:
        """Run one throwaway query so the first real user does not pay for it.



        The first call into torch, faiss and the tokenizers allocates buffers

        and compiles kernels; on a CPU container that is seconds the first

        visitor would otherwise sit through. The result is cached, harmlessly.

        """
        t0 = time.perf_counter()
        try:
            self._ranked("how do transformers use attention for language modelling")
            print(f"[ml] warmed up in {time.perf_counter() - t0:.1f}s", flush=True)
        except Exception as exc:  # noqa: BLE001 - warmup is an optimisation only
            print(f"[ml] warmup skipped ({type(exc).__name__})", flush=True)

    # ---------------------------------------------------------------- helpers

    def _passage(self, chunk_id: str, score: float) -> dict[str, Any]:
        entry = self.corpus[chunk_id]
        return {
            "chunk_id": chunk_id,
            "doc_id": chunk_id,
            "paper_id": entry["paper_id"],
            "arxiv_url": f"https://arxiv.org/abs/{entry['paper_id']}",
            "score": float(score),
            "title": entry["title"],
            "section": entry["section"],
            "published": entry["published"],
            "text": entry["text"],
        }

    def _confidence(self, scores: list[float]) -> tuple[float, float, bool]:
        ac = self.cfg["answer"]
        best = max(scores, default=0.0)
        n = min(ac.get("confidence_top_n", 5), len(scores))
        mean_top = sum(sorted(scores, reverse=True)[:n]) / n if n else 0.0
        return best, mean_top, mean_top >= ac["min_mean_score"]

    # ------------------------------------------------------------- retrieval

    def _ranked(self, question: str) -> dict[str, float]:
        """Reranked candidate chunks for a query: {chunk_id: score}.



        The one place a query is searched, so every endpoint shares its two

        latency wins:



        * BM25 and dense search run side by side. They are independent, and

          both spend their time in native code (numpy, faiss, torch) that

          releases the GIL, so the cost is the slower of the two, not the sum.

        * Results are cached. The corpus is static and every stage is

          deterministic, so a repeated query - a demo, a retry, /papers then

          /ask on the same text, a shared link - skips search AND the

          reranker, which is the dominant cost on CPU.

        """
        key = " ".join(question.lower().split())
        with self._cache_lock:
            hit = self._rank_cache.get(key)
            if hit is not None:
                self._rank_cache.move_to_end(key)
                self.cache_hits += 1
                return dict(hit)
        query_map = {PSEUDO_QID: question}
        with self._gpu_lock:
            bm25_future = _SEARCH_POOL.submit(self.bm25.search, query_map, self.retrieve_k)
            dense_run = self.dense.search(query_map, top_k=self.retrieve_k)
            fused = reciprocal_rank_fusion(
                [bm25_future.result(), dense_run], k=self.rrf_k, top_k=self.retrieve_k
            )
            ranked = self._rerank_adaptive(query_map, fused[PSEUDO_QID])
        with self._cache_lock:
            self._rank_cache[key] = dict(ranked)
            while len(self._rank_cache) > RANK_CACHE_MAX:
                self._rank_cache.popitem(last=False)
        return ranked

    def _rerank_adaptive(self, query_map: dict[str, str], fused: dict[str, float]) -> dict[str, float]:
        """Cross-encoder scores for the fused candidates, stopping early when sure.



        The cross-encoder is the dominant cost of a search and scales with the

        candidates scored (seconds on a 2-vCPU CPU). Score the best `first`

        fused candidates; if their top-5 mean already clears `tau`, the answer

        is obvious and the remaining candidates are not worth scoring.

        Otherwise score the rest, so a hard query still sees every candidate.



        Measured offline (scripts/eval_early_exit.py): recall@10 unchanged,

        ~27% fewer candidates scored.

        """
        ee = self.cfg["retrieval"].get("early_exit")
        if not ee or len(fused) <= ee["first"]:
            reranked, _ = self.reranker.rerank(query_map, {PSEUDO_QID: fused}, self.corpus,
                                               top_k=self.retrieve_k)
            return reranked[PSEUDO_QID]
        order = sorted(fused.items(), key=lambda kv: kv[1], reverse=True)
        head = dict(order[: ee["first"]])
        scores, _ = self.reranker.rerank(query_map, {PSEUDO_QID: head}, self.corpus,
                                         top_k=len(head))
        scores = scores[PSEUDO_QID]
        top = sorted(scores.values(), reverse=True)[:5]
        if sum(top) / len(top) >= ee["tau"]:
            self.early_exits += 1
            return scores
        tail = dict(order[ee["first"]:])
        rest, _ = self.reranker.rerank(query_map, {PSEUDO_QID: tail}, self.corpus,
                                       top_k=len(tail))
        return {**scores, **rest[PSEUDO_QID]}

    def retrieve(self, question: str, top_k: int | None = None) -> list[dict[str, Any]]:
        """Hybrid retrieve, fuse, rerank, then cap chunks per paper."""
        reranked = {PSEUDO_QID: self._ranked(question)}

        ac = self.cfg["answer"]
        selected = cap_per_paper(
            reranked[PSEUDO_QID],
            self.corpus,
            top_k=top_k or ac["top_k_passages"],
            max_per_paper=ac["max_per_paper"],
        )
        return [self._passage(cid, s) for cid, s in selected]

    # -------------------------------------------------------- paper discovery

    def find_papers(self, topic: str, top_k: int = 10) -> dict[str, Any]:
        """Topic in, ranked *papers* out.



        The ask path returns passages, which is right for answering a question

        and wrong for "show me work on X" - you get five paragraphs where you

        wanted five papers. Here the per-paper cap is deliberately skipped, a

        wide candidate set is reranked, and results are then grouped by paper.



        A paper is scored by its single best passage. Summing would reward long

        papers with many mediocre chunks over short papers with one excellent

        one, which inverts what you actually want from a search result.

        """
        start = time.perf_counter()
        reranked = {PSEUDO_QID: self._ranked(topic)}

        grouped: dict[str, dict[str, Any]] = {}
        for chunk_id, score in reranked[PSEUDO_QID].items():
            entry = self.corpus[chunk_id]
            pid = entry["paper_id"]
            bucket = grouped.setdefault(
                pid, {"score": float("-inf"), "matches": 0, "best_chunk": None, "sections": []}
            )
            bucket["matches"] += 1
            bucket["sections"].append(entry["section"])
            if score > bucket["score"]:
                bucket["score"] = float(score)
                bucket["best_chunk"] = chunk_id

        ordered = sorted(grouped.items(), key=lambda kv: kv[1]["score"], reverse=True)[:top_k]

        papers: list[dict[str, Any]] = []
        for pid, info in ordered:
            meta = self.papers_by_id.get(pid, {})
            best = self.corpus[info["best_chunk"]]
            papers.append(
                {
                    "paper_id": pid,
                    "doc_id": info["best_chunk"],
                    "title": meta.get("title") or best["title"],
                    "authors": (meta.get("authors") or [])[:6],
                    "abstract": meta.get("abstract", ""),
                    "published": meta.get("published") or best["published"],
                    "primary_category": meta.get("primary_category", ""),
                    "arxiv_url": f"https://arxiv.org/abs/{pid}",
                    "score": round(info["score"], 4),
                    "matched_passages": info["matches"],
                    "matched_sections": sorted(set(info["sections"]))[:5],
                    "why": best["text"][:400],
                    "why_section": best["section"],
                    "has_full_text": bool(meta.get("full_text")),
                }
            )

        scores = [p["score"] for p in papers]
        best_score, mean_top, confident = self._confidence(scores)

        # Same live fallback the ask path has. Without it, Find papers is
        # limited to the ingested sample - and the corpus holds roughly 1,400
        # papers per category against the 50-150/day arXiv publishes, so a
        # topic can be well covered on arXiv and absent here. Searching "economic
        # world models" returned unrelated robotics papers for exactly this
        # reason while the paper sat on arXiv.
        source = "corpus"
        ac = self.cfg["answer"]
        if not confident and self.cfg["citation"].get("allow_remote", True):
            live = self.search_live(topic, top_k=top_k)
            if live:
                live = self._rescore_live(topic, live)
                floor = ac.get("live_min_score", 0.30)
                kept = [p for p in live if p["score"] >= floor]
                if kept:
                    seen = {p["paper_id"] for p in papers}
                    fetched = [
                        {
                            "paper_id": p["paper_id"],
                            "doc_id": p["doc_id"],
                            "title": p["title"],
                            "authors": p.get("authors", []),
                            "abstract": p["text"],
                            "published": p["published"],
                            "primary_category": "",
                            "arxiv_url": p["arxiv_url"],
                            "venue": p.get("venue", ""),
                            "cited_by_count": p.get("cited_by_count"),
                            "live_source": p.get("live_source", "arxiv"),
                            "score": round(p["score"], 4),
                            "matched_passages": 1,
                            "matched_sections": ["abstract"],
                            "why": p["text"][:400],
                            "why_section": "abstract",
                            "has_full_text": False,
                            "from_arxiv": True,
                        }
                        for p in kept
                        if p["paper_id"] not in seen
                    ]
                    if fetched:
                        # Corpus papers only fill remaining slots if they clear
                        # the same bar - otherwise a 0.08 match sat under a
                        # result labelled confident.
                        papers = (fetched + [p for p in papers if p["score"] >= floor])[:top_k]
                        source = "arxiv_live"
                        confident = True
                        mean_top = sum(p["score"] for p in fetched) / len(fetched)
                        best_score = max(p["score"] for p in fetched)

        # v2: nothing convincing in the corpus or on arXiv - diagnose why.
        diagnosis, diagnosis_ms = None, 0.0
        if not confident:
            diagnosis, diagnosis_ms = self._diagnose(topic, papers, None)

        return {
            "topic": topic,
            "source": source,
            "papers": papers,
            "confident": confident,
            "top_score": round(best_score, 3),
            "mean_top_score": round(mean_top, 3),
            "diagnosis": diagnosis,
            "timing_ms": {"total": round((time.perf_counter() - start) * 1000.0, 1),
                          "diagnosis": round(diagnosis_ms, 1)},
        }

    def _rescore_live(

        self, question: str, live: list[dict[str, Any]]

    ) -> list[dict[str, Any]]:
        """Score live arXiv abstracts with the same reranker the corpus uses.



        arXiv's own relevance ranking is lexical and generous. Running the

        cross-encoder over the abstracts puts live results on the same scale as

        corpus results, so one confidence threshold governs both.

        """
        pairs = [(question, f"{p['title']} {p['text']}".strip()) for p in live]
        if not pairs:
            return []
        with self._gpu_lock:
            scores = self.reranker.score(pairs)
        for passage, score in zip(live, scores):
            passage["score"] = float(score)
        return sorted(live, key=lambda p: p["score"], reverse=True)

    def answer_from_document(

        self, question: str, passages: list[dict[str, Any]], title: str, generate: bool = True

    ) -> dict[str, Any]:
        """Answer a question against a single uploaded paper.



        Every chunk of the document is scored directly by the cross-encoder -

        no BM25, no embeddings, no index. A paper is only 20-40 chunks, so

        exhaustive scoring is both cheap and strictly better than retrieval,

        which can only approximate it.

        """
        start = time.perf_counter()
        if not passages:
            return {
                "question": question,
                "answer": "That document has no readable passages.",
                "confident": False,
                "source": "upload",
                "top_score": 0.0,
                "mean_top_score": 0.0,
                "citations": [],
                "distinct_papers": 0,
                "generation_configured": is_configured(),
                "timing_ms": {"retrieval_and_rerank": 0.0, "generation": 0.0, "total": 0.0},
            }

        pairs = [(question, f"{p['title']} {p['text']}".strip()) for p in passages]
        with self._gpu_lock:
            scores = self.reranker.score(pairs)
        for passage, score in zip(passages, scores):
            passage["score"] = float(score)
        ranked = sorted(passages, key=lambda p: p["score"], reverse=True)

        ac = self.cfg["answer"]
        top = ranked[: ac["top_k_passages"]]
        best, mean_top, _ = self._confidence([p["score"] for p in top])
        retrieval_ms = (time.perf_counter() - start) * 1000.0

        answer_text: str | None = None
        generation_ms = 0.0
        if generate:
            gen_start = time.perf_counter()
            answer_text = generate_answer(
                f"About the paper '{title}': {question}",
                top,
                model=self._answer_models(),
                max_tokens=ac["max_tokens"],
            )
            generation_ms = (time.perf_counter() - gen_start) * 1000.0

        return {
            "question": question,
            "answer": answer_text,
            # The user chose this document, so we answer from it even when the
            # match is weak - declining would be second-guessing them.
            "confident": True,
            "source": "upload",
            "top_score": round(best, 3),
            "mean_top_score": round(mean_top, 3),
            "citations": top,
            "distinct_papers": 1,
            "generation_configured": is_configured(),
            "timing_ms": {
                "retrieval_and_rerank": round(retrieval_ms, 1),
                "generation": round(generation_ms, 1),
                "total": round(retrieval_ms + generation_ms, 1),
            },
        }

    # ------------------------------------------------------------ live search

    def search_live(self, question: str, top_k: int = 6) -> list[dict[str, Any]]:
        """Search arXiv and OpenAlex live, for questions the corpus cannot answer.



        The corpus is a sample, not coverage: ~1,400 papers per category across

        nine months, against the 50-150/day arXiv actually publishes. Most

        papers a user asks about were never ingested, and telling them "not in

        this corpus" is honest but useless when the paper plainly exists.



        Two sources, because each misses what the other has. arXiv is the

        freshest for preprints; OpenAlex adds conference and journal papers

        that never had one. Both run in parallel under ONE wall-clock budget

        (`citation.live_budget_s`), so adding a source never adds latency:

        whatever has answered when the budget expires is used, and a source

        that is down or paused costs nothing. The same paper found by both

        collapses to one entry, keyed on its arXiv id or normalised title.



        Abstracts only - no PDF fetch. Results are rescored by the corpus

        reranker afterwards, so one confidence bar governs every source.

        """
        sources = self.cfg["citation"].get("live_sources", ["arxiv", "openalex"])
        budget = float(self.cfg["citation"].get("live_budget_s", 5.0))
        deadline = time.monotonic() + budget
        jobs = {}
        if "arxiv" in sources:
            jobs["arxiv"] = _LIVE_POOL.submit(self._live_arxiv, question, top_k, deadline)
        if "openalex" in sources:
            jobs["openalex"] = _LIVE_POOL.submit(self._live_openalex, question, top_k, deadline)

        # Do not wait for the slowest source. Once one has results, the other
        # gets a short grace and is then left behind: arXiv often takes 3-5 s
        # while OpenAlex answers in ~1 s, and its extra papers are not worth
        # holding the whole response. A source that returns nothing (down,
        # paused, empty) does not start the clock - the other keeps its full
        # budget.
        grace = float(self.cfg["citation"].get("live_grace_s", 0.75))
        found: dict[str, list[dict[str, Any]]] = {}
        name_of = {job: name for name, job in jobs.items()}
        pending = set(jobs.values())
        cutoff = deadline
        while pending and time.monotonic() < cutoff:
            done, pending = _futures_wait(
                pending, timeout=max(cutoff - time.monotonic(), 0.01),
                return_when=FIRST_COMPLETED)
            for job in done:
                try:
                    found[name_of[job]] = job.result()
                except Exception:  # noqa: BLE001 - a failed source is an empty source
                    found[name_of[job]] = []
                if found[name_of[job]]:
                    cutoff = min(cutoff, time.monotonic() + grace)

        merged: list[dict[str, Any]] = []
        seen: dict[str, dict[str, Any]] = {}
        # arXiv first: on a duplicate, its record wins (fresher version), and
        # OpenAlex contributes the venue and citation count it lacks.
        for name in ("arxiv", "openalex"):
            for item in found.get(name, []):
                key = _dedup_key(item)
                if key in seen:
                    kept = seen[key]
                    kept["venue"] = kept.get("venue") or item.get("venue", "")
                    kept["cited_by_count"] = max(kept.get("cited_by_count") or 0,
                                                 item.get("cited_by_count") or 0)
                    kept["live_source"] = "arxiv+openalex"
                    continue
                seen[key] = item
                merged.append(item)
        return merged

    # Kept for callers that predate OpenAlex.
    search_arxiv_live = search_live

    def _live_arxiv(self, question: str, top_k: int, deadline: float) -> list[dict[str, Any]]:
        terms = [w for w in re.findall(r"[A-Za-z][A-Za-z0-9-]{2,}", question)
                 if w.lower() not in _QUERY_STOPWORDS][:12]
        if not terms:
            return []
        queries = [
            "all:" + " AND ".join(f'"{t}"' for t in terms[:4]),  # precise first
            "all:" + " ".join(terms[:8]),                         # then looser
        ]
        found: list[Paper] = []
        for query in queries:
            remaining = deadline - time.monotonic()
            if remaining <= 0.25:
                break
            try:
                found = fetch_metadata_bounded(query, max_results=top_k,
                                               budget_s=remaining)
            except ArxivUnavailable:
                return []  # a failed enrichment is not a failed request
            if found:
                break
        return [
            {
                "chunk_id": f"live:{p.paper_id}",
                "doc_id": f"live:{p.paper_id}",
                "paper_id": p.paper_id,
                "arxiv_url": f"https://arxiv.org/abs/{p.paper_id}",
                "score": 1.0,
                "title": p.title,
                "section": "abstract",
                "published": p.published,
                "text": p.abstract,
                "authors": p.authors[:8],
                "venue": "arXiv",
                "cited_by_count": None,
                "live_source": "arxiv",
            }
            for p in found
        ]

    def _live_openalex(self, question: str, top_k: int, deadline: float) -> list[dict[str, Any]]:
        remaining = deadline - time.monotonic()
        if remaining <= 0.25:
            return []
        try:
            works = openalex.search_works(question, max_results=top_k, budget_s=remaining)
        except openalex.OpenAlexUnavailable:
            return []
        return [
            {
                "chunk_id": f"live:{w['paper_id']}",
                "doc_id": f"live:{w['paper_id']}",
                "paper_id": w["paper_id"],
                # Named arxiv_url for compatibility with every client; for a
                # paper with no arXiv copy it is the DOI or landing page.
                "arxiv_url": w["url"],
                "score": 1.0,
                "title": w["title"],
                "section": "abstract",
                "published": w["published"],
                "text": w["abstract"],
                "authors": w["authors"][:8],
                "venue": w["venue"],
                "cited_by_count": w["cited_by_count"],
                "doi": w["doi"],
                "live_source": "openalex",
            }
            for w in works
        ]

    # ------------------------------------------------------------------- ask

    def answer(self, question: str, generate: bool = True) -> dict[str, Any]:
        """Question answering, including the US5 rejection path."""
        start = time.perf_counter()
        passages = self.retrieve(question)
        retrieval_ms = (time.perf_counter() - start) * 1000.0
        # Kept for diagnosis: if everything below fails, these weak corpus hits
        # are exactly what the critic needs to see to say what went wrong.
        corpus_passages = passages

        ac = self.cfg["answer"]
        best, mean_top, confident = self._confidence([p["score"] for p in passages])

        # Blend in live arXiv results when the corpus alone is thin.
        #
        # The corpus is a sample (~1,400 papers per category against the 50-150
        # a day arXiv publishes), so a question can be perfectly answerable from
        # the literature and unanswerable from here. Two triggers:
        #
        #   below min_mean_score   the corpus has nothing - live results replace
        #   below enrich_below     the corpus has something thin - live results
        #                          are added, so the answer draws on both
        #
        # A strong corpus hit skips the network entirely and stays sub-second.
        answer_text: str | None = None
        generation_ms = 0.0
        source = "corpus"
        remote_ok = self.cfg["citation"].get("allow_remote", True)
        enrich_below = ac.get("enrich_below", 0.0)

        # Timed on its own: this was the one stage outside every timer, which is
        # how a 45s arXiv stall reported itself as "retrieval_and_rerank: 1783ms".
        live_ms = 0.0
        if remote_ok and (not confident or mean_top < enrich_below):
            live_start = time.perf_counter()
            live = self.search_live(question)
            live_ms = (time.perf_counter() - live_start) * 1000.0
            if live:
                # arXiv always returns *something*, so live results must clear
                # the same confidence bar as the corpus. Without this the
                # honest "I do not know" is replaced by confident nonsense -
                # a sourdough question came back with quantum-physics papers.
                # Filter live results PER PASSAGE, not on their mean.
                #
                # The mean-of-top-5 rule works for the corpus because a query
                # that belongs to it matches many passages. An arXiv keyword
                # search returns a mixed bag by construction - three loose hits
                # would drag the mean under and bury the one paper that is
                # actually the answer. So each live passage clears its own bar.
                live = self._rescore_live(question, live)
                floor = ac.get("live_min_score", 0.30)
                live = [p for p in live if p["score"] >= floor]
                seen = {p["paper_id"] for p in passages}
                extra = [p for p in live if p["paper_id"] not in seen]
                if confident and extra:
                    passages = passages + extra          # enrich
                    source = "corpus+arxiv_live"
                elif extra:
                    passages = extra                     # replace
                    source = "arxiv_live"
                    confident = True
                    mean_top = sum(p["score"] for p in extra) / len(extra)
                elif not confident:
                    passages = []                        # genuinely unanswerable

        # Answer from the papers when they cover the question. When they do
        # not - nothing retrieved, or the answerer replies NOT_COVERED - fall
        # back to the model's own knowledge, but only for AI questions and
        # labelled as not grounded in the papers. Everything else is declined.
        grounded = True
        uncovered = not passages
        if passages and generate:
            gen_start = time.perf_counter()
            answer_text = generate_answer(
                question, passages, model=self._answer_models(), max_tokens=ac["max_tokens"]
            )
            generation_ms = (time.perf_counter() - gen_start) * 1000.0
            if is_not_covered(answer_text):
                uncovered, answer_text = True, None
        if uncovered and generate and ac.get("general_knowledge", True):
            gen_start = time.perf_counter()
            try:
                general = general_answer(question, model=self._answer_models(),
                                         max_tokens=ac["max_tokens"])
            except Exception as exc:  # noqa: BLE001 - fall back to the plain decline
                print(f"[ml] general answer failed ({type(exc).__name__})", flush=True)
                general = None
            generation_ms += (time.perf_counter() - gen_start) * 1000.0
            if general:
                answer_text, source, grounded, confident = general, "general_knowledge", False, False
        if uncovered and answer_text is None:
            answer_text = (
                "Nothing answers that - not in the local corpus, and not in a live "
                "arXiv search either. If this is an AI/ML topic the paper may simply "
                "be too new or too obscure; if it is not, this tool only covers AI research."
                if not passages else
                "The papers found for this do not contain an answer to it."
            )

        # v2: the gate fired and nothing rescued it, so say what went wrong.
        diagnosis, diagnosis_ms = None, 0.0
        if not passages and source != "general_knowledge":
            diagnosis, diagnosis_ms = self._diagnose(question, corpus_passages, None)

        return {
            "question": question,
            "answer": answer_text,
            "confident": confident,
            "source": source,
            "grounded": grounded,
            "top_score": round(best, 3),
            "mean_top_score": round(mean_top, 3),
            "citations": passages,
            "distinct_papers": distinct_papers(passages),
            "generation_configured": is_configured(),
            "diagnosis": diagnosis,
            "timing_ms": {
                "retrieval_and_rerank": round(retrieval_ms, 1),
                "arxiv_live": round(live_ms, 1),
                "generation": round(generation_ms, 1),
                "diagnosis": round(diagnosis_ms, 1),
                "total": round(retrieval_ms + live_ms + generation_ms + diagnosis_ms, 1),
            },
        }

    def _answer_models(self) -> list[str]:
        """Primary answer model, then fallbacks for when its daily cap is spent."""
        ac = self.cfg["answer"]
        return [ac["model"], *ac.get("fallback_models", [])]

    def _diagnose(self, question: str, passages: list[dict[str, Any]],

                  answer: str | None) -> tuple[dict[str, Any] | None, float]:
        """Run the v2 failure diagnosis, if enabled. Never raises."""
        dc = self.cfg.get("diagnosis", {})
        if not dc.get("enabled", True):
            return None, 0.0
        t0 = time.perf_counter()
        d = diagnose(question, passages, answer,
                     model=dc.get("models") or dc.get("model", self.cfg["answer"]["model"]),
                     budget_s=float(dc.get("budget_s", 6.0)))
        return (d.to_json() if d else None), (time.perf_counter() - t0) * 1000.0

    # ------------------------------------------------------- citation lookup

    def _passages_for_paper(self, paper: Paper, from_corpus: bool) -> list[dict[str, Any]]:
        """Passages describing one specific paper.



        Indexed papers use their stored chunks. A paper fetched on demand is

        chunked in memory - it is not added to the index, because rebuilding the

        index per lookup would be far slower than the lookup itself.

        """
        if from_corpus and paper.paper_id in self.chunks_by_paper:
            chunk_ids = self.chunks_by_paper[paper.paper_id]
            ordered = sorted(chunk_ids, key=lambda c: self.corpus[c]["section"] != "abstract")
            return [self._passage(cid, 1.0) for cid in ordered[:8]]

        chunks = chunk_paper(
            paper,
            max_words=self.cfg["chunking"]["max_words"],
            overlap_words=self.cfg["chunking"]["overlap_words"],
            min_words=self.cfg["chunking"]["min_words"],
        )[:8]
        return [
            {
                "chunk_id": c.chunk_id,
                "doc_id": c.chunk_id,
                "paper_id": c.paper_id,
                "arxiv_url": paper_url(paper),
                "score": 1.0,
                "title": c.title,
                "section": c.section,
                "published": c.published,
                "text": c.text,
            }
            for c in chunks
        ]

    def explain_citation(self, citation: str, generate: bool = True) -> dict[str, Any]:
        """Resolve a citation string to a paper and explain it.



        Returns the explanation plus `related`: what else in the corpus talks

        about this paper, which is where the "and here is the wider context"

        half of the answer comes from.

        """
        start = time.perf_counter()

        # A question is not a citation. Trying to resolve one costs two arXiv
        # searches that cannot succeed, and arXiv answers 429 - which is how a
        # concept question turned into a minute of waiting and then a failure.
        if looks_like_a_question(citation):
            answer = self.answer(citation, generate=generate)
            return {
                "citation": citation,
                "resolved": False,
                "method": "question",
                "query_used": citation,
                "candidates": [],
                "paper": None,
                "explanation": answer["answer"],
                "passages": answer["citations"],
                "related": [],
                "answered_as_question": True,
                "source": answer["source"],
                "generation_configured": is_configured(),
                "timing_ms": answer["timing_ms"],
                "diagnosis": answer.get("diagnosis"),
                # The complete /ask payload, so the client can render it
                # directly. Without this the web client discarded the answer
                # computed here and called /ask, re-running retrieval,
                # reranking and generation - every Explain-mode question cost
                # two full pipeline passes and two Groq calls.
                "answer_result": answer,
            }

        resolution = resolve(
            citation,
            self.papers_by_id,
            allow_remote=self.cfg["citation"]["allow_remote"],
            fetch_full_text=self.cfg["citation"]["fetch_full_text"],
            pdf_dir=self.cfg["paths"]["pdf_cache"],
            text_dir=self.cfg["paths"]["text_cache"],
            max_cache_gb=self.cfg["citation"].get("max_pdf_cache_gb", 5.0),
        )

        if resolution.paper is None:
            return {
                "citation": citation,
                "resolved": False,
                "method": resolution.method,
                "query_used": resolution.query_used,
                "candidates": resolution.candidates,
                "explanation": (
                    "Could not resolve that citation to a paper. Try the arXiv id, or the "
                    "paper title on its own."
                ),
                "paper": None,
                "passages": [],
                "related": [],
                "generation_configured": is_configured(),
                "timing_ms": {"total": round((time.perf_counter() - start) * 1000.0, 1)},
            }

        paper = resolution.paper
        passages = self._passages_for_paper(paper, resolution.from_corpus)

        # What else in the corpus discusses this work - the wider-context half.
        related_query = f"{paper.title} {paper.abstract[:300]}"
        related = [
            p
            for p in self.retrieve(related_query, top_k=6)
            if p["paper_id"] != paper.paper_id
        ][:4]

        explanation: str | None = None
        gen_ms = 0.0
        if generate:
            gen_start = time.perf_counter()
            explanation = explain_paper(
                paper.title,
                passages + related,
                model=self._answer_models(),
                max_tokens=self.cfg["citation"]["max_tokens"],
            )
            gen_ms = (time.perf_counter() - gen_start) * 1000.0

        return {
            "citation": citation,
            "resolved": True,
            "method": resolution.method,
            "query_used": resolution.query_used,
            "from_corpus": resolution.from_corpus,
            "candidates": resolution.candidates,
            "paper": {
                "paper_id": paper.paper_id,
                "title": paper.title,
                "abstract": paper.abstract,
                "authors": paper.authors[:8],
                "published": paper.published,
                "primary_category": paper.primary_category,
                "arxiv_url": paper_url(paper),
                "venue": (paper.metadata or {}).get("venue", ""),
                "doi": (paper.metadata or {}).get("doi", ""),
                "has_full_text": bool(paper.full_text),
            },
            "explanation": explanation,
            "passages": passages,
            "related": related,
            "generation_configured": is_configured(),
            "timing_ms": {
                "total": round((time.perf_counter() - start) * 1000.0, 1),
                "generation": round(gen_ms, 1),
            },
        }