Spaces:
Sleeping
Add two-stage retrieval: TF-IDF -> optional FlashRank reranking
Browse filesAdapts the hybrid-search RAG pattern from awesome-llm-apps (Apache-2.0) as a
second retrieval stage. A small cross-encoder (FlashRank, ms-marco-TinyBERT,
~3 MB CPU) reorders the TF-IDF candidates so the most relevant evidence is
cited first and fed to the LLM first.
- reranking.py: lazy, cached ranker with graceful fallback — if flashrank or
the model is unavailable it returns the TF-IDF order, so the app never breaks
and still runs fully offline.
- answering.py: pull a larger TF-IDF candidate pool when reranking is on, but
keep the abstention DECISION on the original top-K, so behaviour and the
golden eval are unchanged whether or not the reranker is present.
- config: RERANK_ENABLED / RERANK_CANDIDATES / RERANK_MODEL (env-tunable).
- Tests for the disabled/unavailable fallbacks and the reorder path (28 pass,
golden eval still 100% citation coverage / 0 unsupported).
Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>
- ARCHITECTURE.md +2 -1
- backend/app/answering.py +16 -7
- backend/app/config.py +9 -0
- backend/app/reranking.py +70 -0
- backend/requirements.txt +5 -0
- backend/tests/test_reranking.py +54 -0
|
@@ -33,7 +33,8 @@ container; in local dev, Vite proxies `/api` to `http://127.0.0.1:8000`.
|
|
| 33 |
| `schemas.py` | Pydantic request/response models (typed I/O, Swagger) |
|
| 34 |
| `parsing.py` | Text extraction: TXT (whole file) and PDF (per page, via `pypdf`) |
|
| 35 |
| `chunking.py` | Overlapping word-window chunking |
|
| 36 |
-
| `retrieval.py` | Tokenizer + TF-IDF ranker (`rank_chunks`) |
|
|
|
|
| 37 |
| `answering.py` | Abstention gates, focused extractive answer, citation builder |
|
| 38 |
| `llm.py` | Optional OpenAI/Anthropic synthesis with graceful fallback |
|
| 39 |
| `ingest.py` | Shared parse → chunk → persist pipeline |
|
|
|
|
| 33 |
| `schemas.py` | Pydantic request/response models (typed I/O, Swagger) |
|
| 34 |
| `parsing.py` | Text extraction: TXT (whole file) and PDF (per page, via `pypdf`) |
|
| 35 |
| `chunking.py` | Overlapping word-window chunking |
|
| 36 |
+
| `retrieval.py` | Tokenizer + TF-IDF ranker (`rank_chunks`) — offline first stage |
|
| 37 |
+
| `reranking.py` | Optional cross-encoder reranking (FlashRank) of the TF-IDF candidates; graceful fallback to TF-IDF order when unavailable |
|
| 38 |
| `answering.py` | Abstention gates, focused extractive answer, citation builder |
|
| 39 |
| `llm.py` | Optional OpenAI/Anthropic synthesis with graceful fallback |
|
| 40 |
| `ingest.py` | Shared parse → chunk → persist pipeline |
|
|
@@ -12,7 +12,7 @@ evidence (citations) so the user can verify.
|
|
| 12 |
import logging
|
| 13 |
from typing import List, Optional, Set, Tuple
|
| 14 |
|
| 15 |
-
from . import config, llm
|
| 16 |
from .models import Chunk
|
| 17 |
from .retrieval import ScoredChunk, rank_chunks, tokenize
|
| 18 |
from .schemas import Citation
|
|
@@ -107,11 +107,17 @@ def answer_question(question: str, chunks: List[Chunk], mode: str = "fast"):
|
|
| 107 |
or the call fails. The returned mode reflects what actually
|
| 108 |
produced the answer ("fast" | "thinking").
|
| 109 |
"""
|
| 110 |
-
|
| 111 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 112 |
|
| 113 |
# Anti-hallucination guard #1: the best evidence must clear a score floor.
|
| 114 |
-
if not
|
| 115 |
logger.info("Abstain (low score %.3f) for question: %s", top_score, question)
|
| 116 |
return config.ABSTAIN_MESSAGE, True, mode, []
|
| 117 |
|
|
@@ -121,7 +127,7 @@ def answer_question(question: str, chunks: List[Chunk], mode: str = "fast"):
|
|
| 121 |
# and matching only the word "number") from producing a confident answer.
|
| 122 |
q_terms = set(tokenize(question))
|
| 123 |
evidence_terms: Set[str] = set()
|
| 124 |
-
for s in
|
| 125 |
evidence_terms |= set(tokenize(s.chunk.text))
|
| 126 |
matched = len(q_terms & evidence_terms)
|
| 127 |
required = min(2, len(q_terms)) # 1-word questions only need 1 match
|
|
@@ -134,8 +140,11 @@ def answer_question(question: str, chunks: List[Chunk], mode: str = "fast"):
|
|
| 134 |
)
|
| 135 |
return config.ABSTAIN_MESSAGE, True, mode, []
|
| 136 |
|
| 137 |
-
#
|
| 138 |
-
|
|
|
|
|
|
|
|
|
|
| 139 |
citations = build_citations(relevant, question)
|
| 140 |
|
| 141 |
# Thinking mode: let the LLM reason over the same grounded context.
|
|
|
|
| 12 |
import logging
|
| 13 |
from typing import List, Optional, Set, Tuple
|
| 14 |
|
| 15 |
+
from . import config, llm, reranking
|
| 16 |
from .models import Chunk
|
| 17 |
from .retrieval import ScoredChunk, rank_chunks, tokenize
|
| 18 |
from .schemas import Citation
|
|
|
|
| 107 |
or the call fails. The returned mode reflects what actually
|
| 108 |
produced the answer ("fast" | "thinking").
|
| 109 |
"""
|
| 110 |
+
# Stage 1: TF-IDF retrieval. Pull a larger candidate pool when reranking is
|
| 111 |
+
# on, but the abstention DECISION below always uses the original top-K, so the
|
| 112 |
+
# behaviour (and the golden eval) is identical whether or not the reranker is
|
| 113 |
+
# present.
|
| 114 |
+
pool_k = config.RERANK_CANDIDATES if reranking.is_enabled() else config.TOP_K
|
| 115 |
+
scored = rank_chunks(question, chunks, top_k=pool_k)
|
| 116 |
+
gate_chunks = scored[: config.TOP_K]
|
| 117 |
+
top_score = gate_chunks[0].score if gate_chunks else 0.0
|
| 118 |
|
| 119 |
# Anti-hallucination guard #1: the best evidence must clear a score floor.
|
| 120 |
+
if not gate_chunks or top_score < config.SCORE_THRESHOLD:
|
| 121 |
logger.info("Abstain (low score %.3f) for question: %s", top_score, question)
|
| 122 |
return config.ABSTAIN_MESSAGE, True, mode, []
|
| 123 |
|
|
|
|
| 127 |
# and matching only the word "number") from producing a confident answer.
|
| 128 |
q_terms = set(tokenize(question))
|
| 129 |
evidence_terms: Set[str] = set()
|
| 130 |
+
for s in gate_chunks:
|
| 131 |
evidence_terms |= set(tokenize(s.chunk.text))
|
| 132 |
matched = len(q_terms & evidence_terms)
|
| 133 |
required = min(2, len(q_terms)) # 1-word questions only need 1 match
|
|
|
|
| 140 |
)
|
| 141 |
return config.ABSTAIN_MESSAGE, True, mode, []
|
| 142 |
|
| 143 |
+
# Stage 2: keep the chunks with real signal, then rerank so the most relevant
|
| 144 |
+
# evidence is cited first / fed to the LLM first (graceful no-op fallback to
|
| 145 |
+
# TF-IDF order when the reranker is unavailable).
|
| 146 |
+
relevant_pool = [s for s in scored if s.score >= config.SCORE_THRESHOLD] or scored[:1]
|
| 147 |
+
relevant = reranking.rerank(question, relevant_pool, top_k=config.TOP_K)
|
| 148 |
citations = build_citations(relevant, question)
|
| 149 |
|
| 150 |
# Thinking mode: let the LLM reason over the same grounded context.
|
|
@@ -52,6 +52,15 @@ LLM_AVAILABLE = bool(
|
|
| 52 |
TOP_K = int(os.getenv("TOP_K", "4"))
|
| 53 |
SCORE_THRESHOLD = float(os.getenv("SCORE_THRESHOLD", "0.12"))
|
| 54 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 55 |
# Upload limits / validation.
|
| 56 |
ALLOWED_EXTENSIONS = {".pdf", ".txt"}
|
| 57 |
MAX_UPLOAD_BYTES = 10 * 1024 * 1024 # 10 MB
|
|
|
|
| 52 |
TOP_K = int(os.getenv("TOP_K", "4"))
|
| 53 |
SCORE_THRESHOLD = float(os.getenv("SCORE_THRESHOLD", "0.12"))
|
| 54 |
|
| 55 |
+
# Optional second-stage reranking (cross-encoder over the TF-IDF candidates).
|
| 56 |
+
# Pattern adapted from the hybrid-search RAG examples in awesome-llm-apps
|
| 57 |
+
# (Apache-2.0). It degrades gracefully to pure TF-IDF when flashrank or the model
|
| 58 |
+
# is unavailable, and the abstention DECISION always stays on the TF-IDF scores —
|
| 59 |
+
# reranking only reorders/selects the evidence that is cited and fed to the LLM.
|
| 60 |
+
RERANK_ENABLED = os.getenv("RERANK_ENABLED", "1").strip().lower() not in {"0", "false", "no", ""}
|
| 61 |
+
RERANK_CANDIDATES = int(os.getenv("RERANK_CANDIDATES", "12")) # TF-IDF pool to rerank
|
| 62 |
+
RERANK_MODEL = os.getenv("RERANK_MODEL", "ms-marco-TinyBERT-L-2-v2").strip()
|
| 63 |
+
|
| 64 |
# Upload limits / validation.
|
| 65 |
ALLOWED_EXTENSIONS = {".pdf", ".txt"}
|
| 66 |
MAX_UPLOAD_BYTES = 10 * 1024 * 1024 # 10 MB
|
|
@@ -0,0 +1,70 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""Optional second-stage reranking (cross-encoder) over retrieved chunks.
|
| 2 |
+
|
| 3 |
+
Stage 1 is the offline TF-IDF retriever (retrieval.py), which picks candidate
|
| 4 |
+
chunks. Stage 2 — this module — uses a small FlashRank cross-encoder to reorder
|
| 5 |
+
those candidates by true query relevance. The pattern is adapted from the
|
| 6 |
+
hybrid-search RAG examples in github.com/Shubhamsaboo/awesome-llm-apps
|
| 7 |
+
(Apache-2.0), re-authored to fit FinanceFlow's pipeline.
|
| 8 |
+
|
| 9 |
+
Like the LLM layer, it is OPTIONAL and degrades gracefully: if `flashrank` isn't
|
| 10 |
+
installed, the model can't be fetched, or anything fails, we return the original
|
| 11 |
+
TF-IDF order — so the app never breaks and still runs fully offline. Reranking
|
| 12 |
+
only reorders the evidence that gets cited and fed to the LLM; the abstention
|
| 13 |
+
decision still runs on the TF-IDF scores in answering.py.
|
| 14 |
+
"""
|
| 15 |
+
import logging
|
| 16 |
+
from typing import List
|
| 17 |
+
|
| 18 |
+
from . import config
|
| 19 |
+
from .retrieval import ScoredChunk
|
| 20 |
+
|
| 21 |
+
logger = logging.getLogger(__name__)
|
| 22 |
+
|
| 23 |
+
_ranker = None # cached FlashRank Ranker instance
|
| 24 |
+
_unavailable = False # latch: once we know it's unusable, stop retrying
|
| 25 |
+
|
| 26 |
+
|
| 27 |
+
def is_enabled() -> bool:
|
| 28 |
+
return config.RERANK_ENABLED
|
| 29 |
+
|
| 30 |
+
|
| 31 |
+
def _get_ranker():
|
| 32 |
+
"""Lazily build and cache the FlashRank ranker, or return None if unusable."""
|
| 33 |
+
global _ranker, _unavailable
|
| 34 |
+
if _ranker is not None:
|
| 35 |
+
return _ranker
|
| 36 |
+
if _unavailable:
|
| 37 |
+
return None
|
| 38 |
+
try:
|
| 39 |
+
from flashrank import Ranker # optional dependency, imported lazily
|
| 40 |
+
|
| 41 |
+
_ranker = Ranker(model_name=config.RERANK_MODEL)
|
| 42 |
+
logger.info("Reranker ready (%s).", config.RERANK_MODEL)
|
| 43 |
+
return _ranker
|
| 44 |
+
except Exception as exc: # not installed / download failed / offline
|
| 45 |
+
logger.info("Reranker unavailable (%s); using TF-IDF order.", exc)
|
| 46 |
+
_unavailable = True
|
| 47 |
+
return None
|
| 48 |
+
|
| 49 |
+
|
| 50 |
+
def rerank(question: str, scored: List[ScoredChunk], top_k: int) -> List[ScoredChunk]:
|
| 51 |
+
"""Reorder `scored` by cross-encoder relevance and return the top_k.
|
| 52 |
+
|
| 53 |
+
Always safe to call: falls back to the original TF-IDF order (trimmed to
|
| 54 |
+
top_k) when reranking is disabled or unavailable.
|
| 55 |
+
"""
|
| 56 |
+
head = scored[:top_k]
|
| 57 |
+
if not config.RERANK_ENABLED or len(scored) <= 1:
|
| 58 |
+
return head
|
| 59 |
+
ranker = _get_ranker()
|
| 60 |
+
if ranker is None:
|
| 61 |
+
return head
|
| 62 |
+
try:
|
| 63 |
+
from flashrank import RerankRequest
|
| 64 |
+
|
| 65 |
+
passages = [{"id": i, "text": s.chunk.text} for i, s in enumerate(scored)]
|
| 66 |
+
ranked = ranker.rerank(RerankRequest(query=question, passages=passages))
|
| 67 |
+
return [scored[r["id"]] for r in ranked][:top_k]
|
| 68 |
+
except Exception as exc: # pragma: no cover - defensive
|
| 69 |
+
logger.warning("Rerank failed (%s); using TF-IDF order.", exc)
|
| 70 |
+
return head
|
|
@@ -6,6 +6,11 @@ python-multipart==0.0.9
|
|
| 6 |
pypdf==4.3.1
|
| 7 |
python-dotenv==1.0.1
|
| 8 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 9 |
# Optional LLM providers (the app works WITHOUT these installed/configured).
|
| 10 |
# Install only if you want LLM-synthesized answers:
|
| 11 |
# pip install openai anthropic
|
|
|
|
| 6 |
pypdf==4.3.1
|
| 7 |
python-dotenv==1.0.1
|
| 8 |
|
| 9 |
+
# Second-stage reranker (cross-encoder). Pattern adapted from awesome-llm-apps.
|
| 10 |
+
# The app degrades gracefully to pure TF-IDF if this is absent; installing it
|
| 11 |
+
# enables the reranking stage. Small CPU model (~3 MB), no GPU needed.
|
| 12 |
+
flashrank==0.2.10
|
| 13 |
+
|
| 14 |
# Optional LLM providers (the app works WITHOUT these installed/configured).
|
| 15 |
# Install only if you want LLM-synthesized answers:
|
| 16 |
# pip install openai anthropic
|
|
@@ -0,0 +1,54 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""Unit tests for the optional second-stage reranker (reranking.py).
|
| 2 |
+
|
| 3 |
+
These cover the graceful-fallback contract (the app must never break when the
|
| 4 |
+
reranker is disabled or unavailable) and that a working ranker actually reorders.
|
| 5 |
+
"""
|
| 6 |
+
import pytest
|
| 7 |
+
|
| 8 |
+
from app import config, reranking
|
| 9 |
+
from app.retrieval import ScoredChunk
|
| 10 |
+
from tests.conftest import make_chunk
|
| 11 |
+
|
| 12 |
+
try:
|
| 13 |
+
import flashrank # noqa: F401
|
| 14 |
+
|
| 15 |
+
HAS_FLASHRANK = True
|
| 16 |
+
except Exception:
|
| 17 |
+
HAS_FLASHRANK = False
|
| 18 |
+
|
| 19 |
+
|
| 20 |
+
def _scored():
|
| 21 |
+
# TF-IDF order: alpha(0) > bravo(1) > charlie(2)
|
| 22 |
+
return [
|
| 23 |
+
ScoredChunk(make_chunk("alpha", chunk_index=0), 0.9),
|
| 24 |
+
ScoredChunk(make_chunk("bravo", chunk_index=1), 0.5),
|
| 25 |
+
ScoredChunk(make_chunk("charlie", chunk_index=2), 0.1),
|
| 26 |
+
]
|
| 27 |
+
|
| 28 |
+
|
| 29 |
+
def test_rerank_disabled_keeps_tfidf_order(monkeypatch):
|
| 30 |
+
monkeypatch.setattr(config, "RERANK_ENABLED", False)
|
| 31 |
+
out = reranking.rerank("anything", _scored(), top_k=2)
|
| 32 |
+
assert [s.chunk.chunk_index for s in out] == [0, 1] # untouched TF-IDF order, trimmed
|
| 33 |
+
|
| 34 |
+
|
| 35 |
+
def test_rerank_unavailable_falls_back_to_tfidf(monkeypatch):
|
| 36 |
+
# Enabled, but the ranker can't be built -> must fall back to TF-IDF order.
|
| 37 |
+
monkeypatch.setattr(config, "RERANK_ENABLED", True)
|
| 38 |
+
monkeypatch.setattr(reranking, "_get_ranker", lambda: None)
|
| 39 |
+
out = reranking.rerank("anything", _scored(), top_k=2)
|
| 40 |
+
assert [s.chunk.chunk_index for s in out] == [0, 1]
|
| 41 |
+
|
| 42 |
+
|
| 43 |
+
@pytest.mark.skipif(not HAS_FLASHRANK, reason="flashrank not installed")
|
| 44 |
+
def test_rerank_reorders_with_a_working_ranker(monkeypatch):
|
| 45 |
+
# A fake ranker that promotes charlie(2) to the top proves we honour its order.
|
| 46 |
+
monkeypatch.setattr(config, "RERANK_ENABLED", True)
|
| 47 |
+
|
| 48 |
+
class FakeRanker:
|
| 49 |
+
def rerank(self, _req):
|
| 50 |
+
return [{"id": 2}, {"id": 0}, {"id": 1}]
|
| 51 |
+
|
| 52 |
+
monkeypatch.setattr(reranking, "_get_ranker", lambda: FakeRanker())
|
| 53 |
+
out = reranking.rerank("anything", _scored(), top_k=2)
|
| 54 |
+
assert [s.chunk.chunk_index for s in out] == [2, 0]
|