ai / backend /tests /test_retrieval.py
Aissugur Kurganbekov
Claude Opus 4.8 (1M context)
FinanceFlow AI: source-grounded RAG Q&A over finance documents
fc5156b
Raw History Blame Contribute Delete
1.25 kB
"""Unit tests for tokenization and TF-IDF ranking."""
from app.retrieval import rank_chunks, tokenize
from tests.conftest import make_chunk
def test_tokenize_drops_stopwords_and_short_tokens():
tokens = tokenize("What is the TOTAL amount due?")
assert "total" in tokens
assert "amount" in tokens
assert "due" in tokens
# stopwords removed
assert "is" not in tokens
assert "the" not in tokens
assert "what" not in tokens
def test_rank_chunks_orders_by_relevance():
chunks = [
make_chunk("The cat sat on the mat in the garden.", chunk_index=0),
make_chunk("Total amount due is $2,413.98 on this invoice.", chunk_index=1),
make_chunk("Payment terms are net thirty days.", chunk_index=2),
]
ranked = rank_chunks("What is the total amount due?", chunks, top_k=3)
assert ranked[0].chunk.chunk_index == 1 # the invoice-total chunk wins
assert ranked[0].score > 0
def test_rank_chunks_empty_input():
assert rank_chunks("anything", [], top_k=4) == []
def test_rank_chunks_respects_top_k():
chunks = [make_chunk(f"payment number {i} due now", chunk_index=i) for i in range(6)]
ranked = rank_chunks("payment due", chunks, top_k=2)
assert len(ranked) == 2