Aissugur Kurganbekov Claude Opus 4.8 commited on
Commit
ba9f2db
·
1 Parent(s): 81e0253

Add two-stage retrieval: TF-IDF -> optional FlashRank reranking

Browse files

Adapts the hybrid-search RAG pattern from awesome-llm-apps (Apache-2.0) as a
second retrieval stage. A small cross-encoder (FlashRank, ms-marco-TinyBERT,
~3 MB CPU) reorders the TF-IDF candidates so the most relevant evidence is
cited first and fed to the LLM first.

- reranking.py: lazy, cached ranker with graceful fallback — if flashrank or
the model is unavailable it returns the TF-IDF order, so the app never breaks
and still runs fully offline.
- answering.py: pull a larger TF-IDF candidate pool when reranking is on, but
keep the abstention DECISION on the original top-K, so behaviour and the
golden eval are unchanged whether or not the reranker is present.
- config: RERANK_ENABLED / RERANK_CANDIDATES / RERANK_MODEL (env-tunable).
- Tests for the disabled/unavailable fallbacks and the reorder path (28 pass,
golden eval still 100% citation coverage / 0 unsupported).

Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>

ARCHITECTURE.md CHANGED
@@ -33,7 +33,8 @@ container; in local dev, Vite proxies `/api` to `http://127.0.0.1:8000`.
33
  | `schemas.py` | Pydantic request/response models (typed I/O, Swagger) |
34
  | `parsing.py` | Text extraction: TXT (whole file) and PDF (per page, via `pypdf`) |
35
  | `chunking.py` | Overlapping word-window chunking |
36
- | `retrieval.py` | Tokenizer + TF-IDF ranker (`rank_chunks`) |
 
37
  | `answering.py` | Abstention gates, focused extractive answer, citation builder |
38
  | `llm.py` | Optional OpenAI/Anthropic synthesis with graceful fallback |
39
  | `ingest.py` | Shared parse → chunk → persist pipeline |
 
33
  | `schemas.py` | Pydantic request/response models (typed I/O, Swagger) |
34
  | `parsing.py` | Text extraction: TXT (whole file) and PDF (per page, via `pypdf`) |
35
  | `chunking.py` | Overlapping word-window chunking |
36
+ | `retrieval.py` | Tokenizer + TF-IDF ranker (`rank_chunks`) — offline first stage |
37
+ | `reranking.py` | Optional cross-encoder reranking (FlashRank) of the TF-IDF candidates; graceful fallback to TF-IDF order when unavailable |
38
  | `answering.py` | Abstention gates, focused extractive answer, citation builder |
39
  | `llm.py` | Optional OpenAI/Anthropic synthesis with graceful fallback |
40
  | `ingest.py` | Shared parse → chunk → persist pipeline |
backend/app/answering.py CHANGED
@@ -12,7 +12,7 @@ evidence (citations) so the user can verify.
12
  import logging
13
  from typing import List, Optional, Set, Tuple
14
 
15
- from . import config, llm
16
  from .models import Chunk
17
  from .retrieval import ScoredChunk, rank_chunks, tokenize
18
  from .schemas import Citation
@@ -107,11 +107,17 @@ def answer_question(question: str, chunks: List[Chunk], mode: str = "fast"):
107
  or the call fails. The returned mode reflects what actually
108
  produced the answer ("fast" | "thinking").
109
  """
110
- scored = rank_chunks(question, chunks, top_k=config.TOP_K)
111
- top_score = scored[0].score if scored else 0.0
 
 
 
 
 
 
112
 
113
  # Anti-hallucination guard #1: the best evidence must clear a score floor.
114
- if not scored or top_score < config.SCORE_THRESHOLD:
115
  logger.info("Abstain (low score %.3f) for question: %s", top_score, question)
116
  return config.ABSTAIN_MESSAGE, True, mode, []
117
 
@@ -121,7 +127,7 @@ def answer_question(question: str, chunks: List[Chunk], mode: str = "fast"):
121
  # and matching only the word "number") from producing a confident answer.
122
  q_terms = set(tokenize(question))
123
  evidence_terms: Set[str] = set()
124
- for s in scored:
125
  evidence_terms |= set(tokenize(s.chunk.text))
126
  matched = len(q_terms & evidence_terms)
127
  required = min(2, len(q_terms)) # 1-word questions only need 1 match
@@ -134,8 +140,11 @@ def answer_question(question: str, chunks: List[Chunk], mode: str = "fast"):
134
  )
135
  return config.ABSTAIN_MESSAGE, True, mode, []
136
 
137
- # Keep only chunks that carry real signal for the citations/context.
138
- relevant = [s for s in scored if s.score >= config.SCORE_THRESHOLD] or scored[:1]
 
 
 
139
  citations = build_citations(relevant, question)
140
 
141
  # Thinking mode: let the LLM reason over the same grounded context.
 
12
  import logging
13
  from typing import List, Optional, Set, Tuple
14
 
15
+ from . import config, llm, reranking
16
  from .models import Chunk
17
  from .retrieval import ScoredChunk, rank_chunks, tokenize
18
  from .schemas import Citation
 
107
  or the call fails. The returned mode reflects what actually
108
  produced the answer ("fast" | "thinking").
109
  """
110
+ # Stage 1: TF-IDF retrieval. Pull a larger candidate pool when reranking is
111
+ # on, but the abstention DECISION below always uses the original top-K, so the
112
+ # behaviour (and the golden eval) is identical whether or not the reranker is
113
+ # present.
114
+ pool_k = config.RERANK_CANDIDATES if reranking.is_enabled() else config.TOP_K
115
+ scored = rank_chunks(question, chunks, top_k=pool_k)
116
+ gate_chunks = scored[: config.TOP_K]
117
+ top_score = gate_chunks[0].score if gate_chunks else 0.0
118
 
119
  # Anti-hallucination guard #1: the best evidence must clear a score floor.
120
+ if not gate_chunks or top_score < config.SCORE_THRESHOLD:
121
  logger.info("Abstain (low score %.3f) for question: %s", top_score, question)
122
  return config.ABSTAIN_MESSAGE, True, mode, []
123
 
 
127
  # and matching only the word "number") from producing a confident answer.
128
  q_terms = set(tokenize(question))
129
  evidence_terms: Set[str] = set()
130
+ for s in gate_chunks:
131
  evidence_terms |= set(tokenize(s.chunk.text))
132
  matched = len(q_terms & evidence_terms)
133
  required = min(2, len(q_terms)) # 1-word questions only need 1 match
 
140
  )
141
  return config.ABSTAIN_MESSAGE, True, mode, []
142
 
143
+ # Stage 2: keep the chunks with real signal, then rerank so the most relevant
144
+ # evidence is cited first / fed to the LLM first (graceful no-op fallback to
145
+ # TF-IDF order when the reranker is unavailable).
146
+ relevant_pool = [s for s in scored if s.score >= config.SCORE_THRESHOLD] or scored[:1]
147
+ relevant = reranking.rerank(question, relevant_pool, top_k=config.TOP_K)
148
  citations = build_citations(relevant, question)
149
 
150
  # Thinking mode: let the LLM reason over the same grounded context.
backend/app/config.py CHANGED
@@ -52,6 +52,15 @@ LLM_AVAILABLE = bool(
52
  TOP_K = int(os.getenv("TOP_K", "4"))
53
  SCORE_THRESHOLD = float(os.getenv("SCORE_THRESHOLD", "0.12"))
54
 
 
 
 
 
 
 
 
 
 
55
  # Upload limits / validation.
56
  ALLOWED_EXTENSIONS = {".pdf", ".txt"}
57
  MAX_UPLOAD_BYTES = 10 * 1024 * 1024 # 10 MB
 
52
  TOP_K = int(os.getenv("TOP_K", "4"))
53
  SCORE_THRESHOLD = float(os.getenv("SCORE_THRESHOLD", "0.12"))
54
 
55
+ # Optional second-stage reranking (cross-encoder over the TF-IDF candidates).
56
+ # Pattern adapted from the hybrid-search RAG examples in awesome-llm-apps
57
+ # (Apache-2.0). It degrades gracefully to pure TF-IDF when flashrank or the model
58
+ # is unavailable, and the abstention DECISION always stays on the TF-IDF scores —
59
+ # reranking only reorders/selects the evidence that is cited and fed to the LLM.
60
+ RERANK_ENABLED = os.getenv("RERANK_ENABLED", "1").strip().lower() not in {"0", "false", "no", ""}
61
+ RERANK_CANDIDATES = int(os.getenv("RERANK_CANDIDATES", "12")) # TF-IDF pool to rerank
62
+ RERANK_MODEL = os.getenv("RERANK_MODEL", "ms-marco-TinyBERT-L-2-v2").strip()
63
+
64
  # Upload limits / validation.
65
  ALLOWED_EXTENSIONS = {".pdf", ".txt"}
66
  MAX_UPLOAD_BYTES = 10 * 1024 * 1024 # 10 MB
backend/app/reranking.py ADDED
@@ -0,0 +1,70 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ """Optional second-stage reranking (cross-encoder) over retrieved chunks.
2
+
3
+ Stage 1 is the offline TF-IDF retriever (retrieval.py), which picks candidate
4
+ chunks. Stage 2 — this module — uses a small FlashRank cross-encoder to reorder
5
+ those candidates by true query relevance. The pattern is adapted from the
6
+ hybrid-search RAG examples in github.com/Shubhamsaboo/awesome-llm-apps
7
+ (Apache-2.0), re-authored to fit FinanceFlow's pipeline.
8
+
9
+ Like the LLM layer, it is OPTIONAL and degrades gracefully: if `flashrank` isn't
10
+ installed, the model can't be fetched, or anything fails, we return the original
11
+ TF-IDF order — so the app never breaks and still runs fully offline. Reranking
12
+ only reorders the evidence that gets cited and fed to the LLM; the abstention
13
+ decision still runs on the TF-IDF scores in answering.py.
14
+ """
15
+ import logging
16
+ from typing import List
17
+
18
+ from . import config
19
+ from .retrieval import ScoredChunk
20
+
21
+ logger = logging.getLogger(__name__)
22
+
23
+ _ranker = None # cached FlashRank Ranker instance
24
+ _unavailable = False # latch: once we know it's unusable, stop retrying
25
+
26
+
27
+ def is_enabled() -> bool:
28
+ return config.RERANK_ENABLED
29
+
30
+
31
+ def _get_ranker():
32
+ """Lazily build and cache the FlashRank ranker, or return None if unusable."""
33
+ global _ranker, _unavailable
34
+ if _ranker is not None:
35
+ return _ranker
36
+ if _unavailable:
37
+ return None
38
+ try:
39
+ from flashrank import Ranker # optional dependency, imported lazily
40
+
41
+ _ranker = Ranker(model_name=config.RERANK_MODEL)
42
+ logger.info("Reranker ready (%s).", config.RERANK_MODEL)
43
+ return _ranker
44
+ except Exception as exc: # not installed / download failed / offline
45
+ logger.info("Reranker unavailable (%s); using TF-IDF order.", exc)
46
+ _unavailable = True
47
+ return None
48
+
49
+
50
+ def rerank(question: str, scored: List[ScoredChunk], top_k: int) -> List[ScoredChunk]:
51
+ """Reorder `scored` by cross-encoder relevance and return the top_k.
52
+
53
+ Always safe to call: falls back to the original TF-IDF order (trimmed to
54
+ top_k) when reranking is disabled or unavailable.
55
+ """
56
+ head = scored[:top_k]
57
+ if not config.RERANK_ENABLED or len(scored) <= 1:
58
+ return head
59
+ ranker = _get_ranker()
60
+ if ranker is None:
61
+ return head
62
+ try:
63
+ from flashrank import RerankRequest
64
+
65
+ passages = [{"id": i, "text": s.chunk.text} for i, s in enumerate(scored)]
66
+ ranked = ranker.rerank(RerankRequest(query=question, passages=passages))
67
+ return [scored[r["id"]] for r in ranked][:top_k]
68
+ except Exception as exc: # pragma: no cover - defensive
69
+ logger.warning("Rerank failed (%s); using TF-IDF order.", exc)
70
+ return head
backend/requirements.txt CHANGED
@@ -6,6 +6,11 @@ python-multipart==0.0.9
6
  pypdf==4.3.1
7
  python-dotenv==1.0.1
8
 
 
 
 
 
 
9
  # Optional LLM providers (the app works WITHOUT these installed/configured).
10
  # Install only if you want LLM-synthesized answers:
11
  # pip install openai anthropic
 
6
  pypdf==4.3.1
7
  python-dotenv==1.0.1
8
 
9
+ # Second-stage reranker (cross-encoder). Pattern adapted from awesome-llm-apps.
10
+ # The app degrades gracefully to pure TF-IDF if this is absent; installing it
11
+ # enables the reranking stage. Small CPU model (~3 MB), no GPU needed.
12
+ flashrank==0.2.10
13
+
14
  # Optional LLM providers (the app works WITHOUT these installed/configured).
15
  # Install only if you want LLM-synthesized answers:
16
  # pip install openai anthropic
backend/tests/test_reranking.py ADDED
@@ -0,0 +1,54 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ """Unit tests for the optional second-stage reranker (reranking.py).
2
+
3
+ These cover the graceful-fallback contract (the app must never break when the
4
+ reranker is disabled or unavailable) and that a working ranker actually reorders.
5
+ """
6
+ import pytest
7
+
8
+ from app import config, reranking
9
+ from app.retrieval import ScoredChunk
10
+ from tests.conftest import make_chunk
11
+
12
+ try:
13
+ import flashrank # noqa: F401
14
+
15
+ HAS_FLASHRANK = True
16
+ except Exception:
17
+ HAS_FLASHRANK = False
18
+
19
+
20
+ def _scored():
21
+ # TF-IDF order: alpha(0) > bravo(1) > charlie(2)
22
+ return [
23
+ ScoredChunk(make_chunk("alpha", chunk_index=0), 0.9),
24
+ ScoredChunk(make_chunk("bravo", chunk_index=1), 0.5),
25
+ ScoredChunk(make_chunk("charlie", chunk_index=2), 0.1),
26
+ ]
27
+
28
+
29
+ def test_rerank_disabled_keeps_tfidf_order(monkeypatch):
30
+ monkeypatch.setattr(config, "RERANK_ENABLED", False)
31
+ out = reranking.rerank("anything", _scored(), top_k=2)
32
+ assert [s.chunk.chunk_index for s in out] == [0, 1] # untouched TF-IDF order, trimmed
33
+
34
+
35
+ def test_rerank_unavailable_falls_back_to_tfidf(monkeypatch):
36
+ # Enabled, but the ranker can't be built -> must fall back to TF-IDF order.
37
+ monkeypatch.setattr(config, "RERANK_ENABLED", True)
38
+ monkeypatch.setattr(reranking, "_get_ranker", lambda: None)
39
+ out = reranking.rerank("anything", _scored(), top_k=2)
40
+ assert [s.chunk.chunk_index for s in out] == [0, 1]
41
+
42
+
43
+ @pytest.mark.skipif(not HAS_FLASHRANK, reason="flashrank not installed")
44
+ def test_rerank_reorders_with_a_working_ranker(monkeypatch):
45
+ # A fake ranker that promotes charlie(2) to the top proves we honour its order.
46
+ monkeypatch.setattr(config, "RERANK_ENABLED", True)
47
+
48
+ class FakeRanker:
49
+ def rerank(self, _req):
50
+ return [{"id": 2}, {"id": 0}, {"id": 1}]
51
+
52
+ monkeypatch.setattr(reranking, "_get_ranker", lambda: FakeRanker())
53
+ out = reranking.rerank("anything", _scored(), top_k=2)
54
+ assert [s.chunk.chunk_index for s in out] == [2, 0]