File size: 3,948 Bytes
d3d09d6
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
74f3acd
d3d09d6
 
 
 
74f3acd
 
 
 
 
 
 
 
 
 
 
 
d3d09d6
 
 
 
 
 
 
 
 
 
 
 
 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
"""Unit tests for Hybrid Retrieval, Query Rewriting, and Decomposition."""

import unittest
from ingest import DocumentChunk
from retrieve import HybridRetriever, QueryProcessor


class TestRetrieval(unittest.TestCase):
    @classmethod
    def setUpClass(cls):
        cls.retriever = HybridRetriever()
        cls.c1 = DocumentChunk(
            chunk_id="chunk_01",
            doc_id="doc_1",
            file_name="cloud.docx",
            text="Amazon Aurora PostgreSQL powers the relational database tier with multi-master replication.",
            page_number=1,
            section_title="Database Tier",
            chunk_index=0,
            token_count=15
        )
        cls.c2 = DocumentChunk(
            chunk_id="chunk_02",
            doc_id="doc_1",
            file_name="cloud.docx",
            text="Redis Cluster version 7.2 handles user session caching across 6 shards.",
            page_number=1,
            section_title="Caching Layer",
            chunk_index=1,
            token_count=13
        )
        cls.c3 = DocumentChunk(
            chunk_id="chunk_03",
            doc_id="doc_2",
            file_name="security.txt",
            text="FIDO2 WebAuthn security keys are mandatory for all production console access.",
            page_number=1,
            section_title="IAM Policy",
            chunk_index=2,
            token_count=14
        )
        cls.retriever.index_documents([cls.c1, cls.c2, cls.c3])

    def test_dense_search(self):
        res = self.retriever.dense_index.search("relational database", top_k=2)
        self.assertGreater(len(res), 0)
        self.assertEqual(res[0][0].chunk_id, "chunk_01")

    def test_bm25_search(self):
        res = self.retriever.sparse_index.search("Redis cluster shards", top_k=2)
        self.assertGreater(len(res), 0)
        self.assertEqual(res[0][0].chunk_id, "chunk_02")

    def test_hybrid_search_with_rerank(self):
        res = self.retriever.retrieve(
            query="What hardware keys are required for production access?",
            initial_top_k=5,
            rerank_top_k=2,
            mode="hybrid_rerank"
        )
        self.assertGreater(len(res["chunks"]), 0)
        # Top chunk should be chunk_03 (security keys)
        self.assertEqual(res["chunks"][0].chunk_id, "chunk_03")

    def test_query_rewriting(self):
        qp = QueryProcessor()
        history = [
            {"role": "user", "content": "What is the replication lag for Aurora PostgreSQL?"},
            {"role": "assistant", "content": "Average replication lag is less than 12 milliseconds."}
        ]
        # Coreference question should incorporate context from previous turn
        rewritten = qp.rewrite_query("How many shards does it use?", history)
        self.assertTrue(len(rewritten) > len("How many shards does it use?"))
        self.assertIn("Aurora", rewritten)

        # Standalone short query should NOT be rewritten or contaminated
        standalone = qp.rewrite_query("explain hyperledger", history)
        self.assertEqual(standalone, "explain hyperledger")

        # List/dict content formats from Gradio should be handled safely without TypeError
        list_history = [
            {"role": "user", "content": [{"type": "text", "text": "What was total revenue?"}]},
            {"role": "assistant", "content": ["$520M"]}
        ]
        res = qp.rewrite_query("hyperledger", list_history)
        self.assertEqual(res, "hyperledger")

    def test_query_decomposition(self):
        qp = QueryProcessor()
        # Multi-question test
        sub_queries = qp.decompose_query("What database is used? How does caching work?")
        self.assertEqual(len(sub_queries), 2)

        # Comparative question test
        comp_queries = qp.decompose_query("Compare Aurora PostgreSQL and Redis Cluster performance")
        self.assertGreaterEqual(len(comp_queries), 2)


if __name__ == "__main__":
    unittest.main()