document-qa-system / tests /test_retrieve.py
Varun2007's picture
Fix list content parsing in query rewriter and improve bullet point extraction for notes
74f3acd
Raw History Blame Contribute Delete
3.95 kB
"""Unit tests for Hybrid Retrieval, Query Rewriting, and Decomposition."""
import unittest
from ingest import DocumentChunk
from retrieve import HybridRetriever, QueryProcessor
class TestRetrieval(unittest.TestCase):
@classmethod
def setUpClass(cls):
cls.retriever = HybridRetriever()
cls.c1 = DocumentChunk(
chunk_id="chunk_01",
doc_id="doc_1",
file_name="cloud.docx",
text="Amazon Aurora PostgreSQL powers the relational database tier with multi-master replication.",
page_number=1,
section_title="Database Tier",
chunk_index=0,
token_count=15
)
cls.c2 = DocumentChunk(
chunk_id="chunk_02",
doc_id="doc_1",
file_name="cloud.docx",
text="Redis Cluster version 7.2 handles user session caching across 6 shards.",
page_number=1,
section_title="Caching Layer",
chunk_index=1,
token_count=13
)
cls.c3 = DocumentChunk(
chunk_id="chunk_03",
doc_id="doc_2",
file_name="security.txt",
text="FIDO2 WebAuthn security keys are mandatory for all production console access.",
page_number=1,
section_title="IAM Policy",
chunk_index=2,
token_count=14
)
cls.retriever.index_documents([cls.c1, cls.c2, cls.c3])
def test_dense_search(self):
res = self.retriever.dense_index.search("relational database", top_k=2)
self.assertGreater(len(res), 0)
self.assertEqual(res[0][0].chunk_id, "chunk_01")
def test_bm25_search(self):
res = self.retriever.sparse_index.search("Redis cluster shards", top_k=2)
self.assertGreater(len(res), 0)
self.assertEqual(res[0][0].chunk_id, "chunk_02")
def test_hybrid_search_with_rerank(self):
res = self.retriever.retrieve(
query="What hardware keys are required for production access?",
initial_top_k=5,
rerank_top_k=2,
mode="hybrid_rerank"
)
self.assertGreater(len(res["chunks"]), 0)
# Top chunk should be chunk_03 (security keys)
self.assertEqual(res["chunks"][0].chunk_id, "chunk_03")
def test_query_rewriting(self):
qp = QueryProcessor()
history = [
{"role": "user", "content": "What is the replication lag for Aurora PostgreSQL?"},
{"role": "assistant", "content": "Average replication lag is less than 12 milliseconds."}
]
# Coreference question should incorporate context from previous turn
rewritten = qp.rewrite_query("How many shards does it use?", history)
self.assertTrue(len(rewritten) > len("How many shards does it use?"))
self.assertIn("Aurora", rewritten)
# Standalone short query should NOT be rewritten or contaminated
standalone = qp.rewrite_query("explain hyperledger", history)
self.assertEqual(standalone, "explain hyperledger")
# List/dict content formats from Gradio should be handled safely without TypeError
list_history = [
{"role": "user", "content": [{"type": "text", "text": "What was total revenue?"}]},
{"role": "assistant", "content": ["$520M"]}
]
res = qp.rewrite_query("hyperledger", list_history)
self.assertEqual(res, "hyperledger")
def test_query_decomposition(self):
qp = QueryProcessor()
# Multi-question test
sub_queries = qp.decompose_query("What database is used? How does caching work?")
self.assertEqual(len(sub_queries), 2)
# Comparative question test
comp_queries = qp.decompose_query("Compare Aurora PostgreSQL and Redis Cluster performance")
self.assertGreaterEqual(len(comp_queries), 2)
if __name__ == "__main__":
unittest.main()