Delete searcher

searcher/dense_retriever.py

@@ -1,79 +0,0 @@
-# searcher/dense_retriever.py
-
-import numpy as np
-import faiss
-import sqlite3
-import yaml
-from indexer.embedder import Embedder
-
-
-class DenseRetriever:
-    """
-    Embeds the (expanded) query and searches the FAISS index for
-    nearest-neighbor chunks by semantic similarity.
-
-    This is the core semantic search — finds chunks that MEAN the same
-    thing as the query, even if they share no keywords.
-    """
-
-    def __init__(self, config_path="config.yaml"):
-        with open(config_path) as f:
-            config = yaml.safe_load(f)
-
-        self.data_dir = config["data_dir"]
-        self.faiss_path = f"{self.data_dir}/index.faiss"
-        self.db_path = f"{self.data_dir}/metadata.db"
-        self.top_k = config.get("top_k", 20)  # fetch more than needed; reranker will trim
-
-        self.embedder = Embedder(config_path)
-        self.index = faiss.read_index(self.faiss_path)
-
-    def retrieve(self, query: str, top_k: int = None) -> list[dict]:
-        """
-        Embed the query and search FAISS for the closest chunk vectors.
-
-        Args:
-            query (str) — expanded query string
-            top_k (int) — number of results
-
-        Returns:
-            list[dict] each containing:
-                chunk_id, chunk_text, filepath, chunk_index,
-                dense_score (float, lower = more similar in L2 space)
-        """
-        k = top_k or self.top_k
-
-        query_vec = self.embedder.embed_single(query)
-        query_vec = np.array([query_vec], dtype="float32")
-
-        distances, ids = self.index.search(query_vec, k)
-
-        results = []
-        conn = sqlite3.connect(self.db_path)
-        cursor = conn.cursor()
-
-        for dist, chunk_id in zip(distances[0], ids[0]):
-            if chunk_id == -1:
-                continue
-            row = cursor.execute(
-                "SELECT chunk_text, filepath, chunk_index FROM chunks WHERE id = ?",
-                (int(chunk_id),)
-            ).fetchone()
-            if row:
-                results.append({
-                    "chunk_id": int(chunk_id),
-                    "chunk_text": row[0],
-                    "filepath": row[1],
-                    "chunk_index": row[2],
-                    "dense_score": float(dist),
-                })
-
-        conn.close()
-        return results
-
-
-if __name__ == "__main__":
-    dr = DenseRetriever()
-    results = dr.retrieve("quarterly budget report", top_k=5)
-    for r in results:
-        print(f"[{r['dense_score']:.4f}] {r['filepath']} → {r['chunk_text'][:80]}")

searcher/facet_filter.py

@@ -1,83 +0,0 @@
-# searcher/facet_filter.py
-
-import os
-from datetime import datetime
-
-
-class FacetFilter:
-    """
-    Post-retrieval filtering by structured metadata.
-    Applied AFTER fusion and reranking to restrict results
-    without hurting semantic quality.
-
-    Supported facets:
-        file_type   — e.g. [".pdf", ".docx"]
-        date_after  — only files modified after this date
-        date_before — only files modified before this date
-        min_size    — minimum file size in bytes
-        max_size    — maximum file size in bytes
-        directory   — only files inside this directory path
-    """
-
-    def filter(
-        self,
-        results: list[dict],
-        file_type: list[str] = None,
-        date_after: datetime = None,
-        date_before: datetime = None,
-        min_size: int = None,
-        max_size: int = None,
-        directory: str = None,
-    ) -> list[dict]:
-        """
-        Apply facet filters to a list of search results.
-
-        Args:
-            results     — list of chunk dicts (must include 'filepath')
-            file_type   — list of extensions e.g. [".pdf", ".txt"]
-            date_after  — datetime object; exclude files older than this
-            date_before — datetime object; exclude files newer than this
-            min_size    — minimum file size in bytes
-            max_size    — maximum file size in bytes
-            directory   — restrict to files under this directory
-
-        Returns:
-            list[dict] — filtered results (same structure, subset)
-        """
-        filtered = []
-
-        for result in results:
-            fp = result["filepath"]
-
-            # File type filter
-            if file_type:
-                ext = os.path.splitext(fp)[1].lower()
-                if ext not in [e.lower() for e in file_type]:
-                    continue
-
-            # Directory scope filter
-            if directory:
-                if not fp.startswith(os.path.abspath(directory)):
-                    continue
-
-            # File system checks (only if file still exists)
-            if os.path.exists(fp):
-                stat = os.stat(fp)
-
-                # Date filters
-                mtime = datetime.fromtimestamp(stat.st_mtime)
-                if date_after and mtime < date_after:
-                    continue
-                if date_before and mtime > date_before:
-                    continue
-
-                # Size filters
-                size = stat.st_size
-                if min_size and size < min_size:
-                    continue
-                if max_size and size > max_size:
-                    continue
-
-            filtered.append(result)
-
-        return filtered

searcher/fusion_ranker.py

@@ -1,80 +0,0 @@
-# searcher/fusion_ranker.py
-
-
-class FusionRanker:
-    """
-    Reciprocal Rank Fusion (RRF) merges dense and sparse result lists
-    into a single ranked list without needing to normalise scores.
-
-    Why RRF instead of score averaging?
-    - Dense scores (L2 distance) and BM25 scores live on different scales
-    - Normalising them requires knowing min/max, which varies per query
-    - RRF uses only the RANK of each result, making it scale-invariant
-
-    RRF formula:
-        score(d) = Σ 1 / (k + rank(d, list_i))
-        where k=60 is a constant that dampens the effect of very high ranks
-    """
-
-    def __init__(self, k: int = 60):
-        self.k = k
-
-    def fuse(
-        self,
-        dense_results: list[dict],
-        sparse_results: list[dict],
-        top_k: int = 10,
-    ) -> list[dict]:
-        """
-        Fuse two ranked result lists using Reciprocal Rank Fusion.
-
-        Args:
-            dense_results  — from DenseRetriever, ordered by dense_score ascending
-                             (lower L2 distance = better)
-            sparse_results — from SparseRetriever, ordered by sparse_score descending
-            top_k          — number of results to return after fusion
-
-        Returns:
-            list[dict] — merged results sorted by rrf_score descending,
-                         each dict includes all original fields + rrf_score
-        """
-        rrf_scores = {}
-        chunk_data = {}
-
-        # Dense list: lower distance = better rank = higher RRF contribution
-        for rank, result in enumerate(dense_results):
-            cid = result["chunk_id"]
-            rrf_scores[cid] = rrf_scores.get(cid, 0) + 1 / (self.k + rank + 1)
-            chunk_data[cid] = result
-
-        # Sparse list: higher BM25 score = better rank = higher RRF contribution
-        for rank, result in enumerate(sparse_results):
-            cid = result["chunk_id"]
-            rrf_scores[cid] = rrf_scores.get(cid, 0) + 1 / (self.k + rank + 1)
-            if cid not in chunk_data:
-                chunk_data[cid] = result
-
-        sorted_ids = sorted(rrf_scores, key=lambda x: rrf_scores[x], reverse=True)[:top_k]
-
-        fused = []
-        for cid in sorted_ids:
-            entry = dict(chunk_data[cid])
-            entry["rrf_score"] = rrf_scores[cid]
-            fused.append(entry)
-
-        return fused
-
-
-if __name__ == "__main__":
-    dense = [
-        {"chunk_id": 1, "chunk_text": "budget summary Q3", "filepath": "/a.pdf", "chunk_index": 0, "dense_score": 0.12},
-        {"chunk_id": 2, "chunk_text": "revenue report",     "filepath": "/b.pdf", "chunk_index": 0, "dense_score": 0.25},
-    ]
-    sparse = [
-        {"chunk_id": 2, "chunk_text": "revenue report",    "filepath": "/b.pdf", "chunk_index": 0, "sparse_score": 8.1},
-        {"chunk_id": 3, "chunk_text": "financial overview", "filepath": "/c.pdf", "chunk_index": 0, "sparse_score": 5.2},
-    ]
-    ranker = FusionRanker()
-    results = ranker.fuse(dense, sparse, top_k=5)
-    for r in results:
-        print(f"[RRF {r['rrf_score']:.5f}] {r['filepath']} → {r['chunk_text'][:60]}")

searcher/highlighter.py

@@ -1,96 +0,0 @@
-# searcher/highlighter.py
-
-
-class Highlighter:
-    """
-    Extracts the most relevant passage from a chunk and highlights
-    query-matching terms for display in search results.
-
-    Why highlight?
-    - The full chunk may be 500 words; the user needs a ~2-sentence preview
-    - Highlighting query terms helps users quickly judge relevance
-    - This is purely display logic — does not affect ranking
-    """
-
-    def __init__(self, preview_words: int = 30):
-        self.preview_words = preview_words
-
-    def extract_preview(self, chunk_text: str, query: str) -> str:
-        """
-        Find the sentence in chunk_text most relevant to the query
-        and return a short preview around it.
-
-        Strategy: find the window of preview_words words that contains
-        the most query term matches.
-
-        Args:
-            chunk_text  — full text of the chunk
-            query       — original user query
-
-        Returns:
-            str — the best preview snippet
-        """
-        words = chunk_text.split()
-        if len(words) <= self.preview_words:
-            return chunk_text
-
-        query_terms = set(query.lower().split())
-        best_score = -1
-        best_start = 0
-
-        for i in range(len(words) - self.preview_words + 1):
-            window = words[i: i + self.preview_words]
-            score = sum(1 for w in window if w.lower().strip(".,;:") in query_terms)
-            if score > best_score:
-                best_score = score
-                best_start = i
-
-        snippet = " ".join(words[best_start: best_start + self.preview_words])
-
-        # Add ellipsis if truncated
-        if best_start > 0:
-            snippet = "..." + snippet
-        if best_start + self.preview_words < len(words):
-            snippet = snippet + "..."
-
-        return snippet
-
-    def highlight_html(self, text: str, query: str) -> str:
-        """
-        Wrap query-matching words in <mark> tags for HTML display.
-
-        Args:
-            text  — preview snippet
-            query — original user query
-
-        Returns:
-            str — HTML string with <mark> tags around matching words
-        """
-        query_terms = set(query.lower().split())
-        highlighted_words = []
-
-        for word in text.split():
-            clean = word.lower().strip(".,;:!?")
-            if clean in query_terms:
-                highlighted_words.append(f"<mark>{word}</mark>")
-            else:
-                highlighted_words.append(word)
-
-        return " ".join(highlighted_words)
-
-    def annotate(self, results: list[dict], query: str) -> list[dict]:
-        """
-        Add preview and highlighted_preview to each result dict.
-
-        Args:
-            results — list of chunk dicts
-            query   — original user query
-
-        Returns:
-            list[dict] — same results with 'preview' and 'preview_html' added
-        """
-        for result in results:
-            preview = self.extract_preview(result["chunk_text"], query)
-            result["preview"] = preview
-            result["preview_html"] = self.highlight_html(preview, query)
-        return results

searcher/query_understanding.py

@@ -1,104 +0,0 @@
-# searcher/query_understanding.py
-
-import yaml
-from nltk.corpus import wordnet
-import nltk
-
-nltk.download("wordnet", quiet=True)
-nltk.download("omw-1.4", quiet=True)
-
-
-class QueryUnderstanding:
-    """
-    Expands and rewrites the raw user query before retrieval.
-
-    Two strategies:
-    1. WordNet synonym expansion — "apples" → ["apple", "malus", "orchard", "fruit"]
-    2. (Optional) T5 query rewriting — rephrase for better embedding alignment
-
-    Why expand?
-    - A search for "apples" should also find "fruit", "orchard", "nutrition"
-    - Embedding models are good but can miss synonyms in different domains
-    - Expansion bridges the vocabulary gap before we even hit FAISS
-    """
-
-    def __init__(self, config_path="config.yaml"):
-        with open(config_path) as f:
-            config = yaml.safe_load(f)
-        self.max_synonyms = config.get("max_synonyms", 5)
-        self.expansion_enabled = config.get("query_expansion", True)
-
-    def expand(self, query: str) -> str:
-        """
-        Expand the query using WordNet synonyms.
-
-        Args:
-            query (str) — raw user query
-
-        Returns:
-            str — expanded query string (original + synonyms appended)
-
-        Example:
-            "apple nutrition" → "apple nutrition fruit malus orchard dietary"
-        """
-        if not self.expansion_enabled:
-            return query
-
-        words = query.lower().split()
-        synonyms = set()
-
-        for word in words:
-            for syn in wordnet.synsets(word):
-                for lemma in syn.lemmas():
-                    cleaned = lemma.name().replace("_", " ")
-                    if cleaned.lower() != word:
-                        synonyms.add(cleaned)
-                if len(synonyms) >= self.max_synonyms:
-                    break
-
-        expanded = query
-        if synonyms:
-            expanded = query + " " + " ".join(list(synonyms)[:self.max_synonyms])
-
-        return expanded
-
-    def rewrite(self, query: str) -> str:
-        """
-        Lightweight query rewriting — normalise and clean the query.
-        (Plug in a T5/FLAN-T5 model here for more powerful rewriting.)
-
-        Args:
-            query (str) — raw user query
-
-        Returns:
-            str — cleaned query
-        """
-        # Basic normalisation: strip extra spaces, lowercase
-        query = " ".join(query.strip().split())
-        return query
-
-    def process(self, query: str) -> dict:
-        """
-        Full query understanding pipeline.
-
-        Returns:
-            dict with:
-                original   — the raw input
-                rewritten  — cleaned version
-                expanded   — synonym-expanded version for dense retrieval
-        """
-        rewritten = self.rewrite(query)
-        expanded = self.expand(rewritten)
-        return {
-            "original": query,
-            "rewritten": rewritten,
-            "expanded": expanded,
-        }
-
-
-if __name__ == "__main__":
-    qu = QueryUnderstanding()
-    result = qu.process("quarterly budget report")
-    print(f"Original : {result['original']}")
-    print(f"Rewritten: {result['rewritten']}")
-    print(f"Expanded : {result['expanded']}")

searcher/reranker.py

@@ -1,66 +0,0 @@
-# searcher/reranker.py
-
-import yaml
-from sentence_transformers import CrossEncoder
-
-
-class Reranker:
-    """
-    Cross-encoder reranking for precision improvement on top-k candidates.
-
-    Why rerank after fusion?
-    - Bi-encoders (used in DenseRetriever) embed query and doc INDEPENDENTLY
-      — fast but they can't model their interaction
-    - Cross-encoders process (query, document) TOGETHER — much more accurate
-      but too slow to run over the whole corpus
-    - Solution: use bi-encoder to shortlist ~20 candidates, then cross-encoder
-      to precisely rank them → speed of FAISS + accuracy of cross-encoder
-
-    Model: ms-marco-MiniLM-L-6-v2 (fast, strong on passage relevance)
-    """
-
-    def __init__(self, config_path="config.yaml"):
-        with open(config_path) as f:
-            config = yaml.safe_load(f)
-
-        model_name = config.get("reranker_model", "cross-encoder/ms-marco-MiniLM-L-6-v2")
-        self.enabled = config.get("reranking_enabled", True)
-        if self.enabled:
-            print(f"Loading reranker model '{model_name}'...")
-            self.model = CrossEncoder(model_name)
-            print("Reranker loaded.")
-
-    def rerank(self, query: str, candidates: list[dict], top_k: int = 5) -> list[dict]:
-        """
-        Rerank candidates using cross-encoder relevance scores.
-
-        Args:
-            query      — original (not expanded) user query
-            candidates — list of chunk dicts from FusionRanker
-            top_k      — number of final results to return
-
-        Returns:
-            list[dict] — reranked results, each with rerank_score added
-        """
-        if not self.enabled or not candidates:
-            return candidates[:top_k]
-
-        pairs = [(query, c["chunk_text"]) for c in candidates]
-        scores = self.model.predict(pairs)
-
-        for i, score in enumerate(scores):
-            candidates[i]["rerank_score"] = float(score)
-
-        reranked = sorted(candidates, key=lambda x: x["rerank_score"], reverse=True)
-        return reranked[:top_k]
-
-
-if __name__ == "__main__":
-    reranker = Reranker()
-    candidates = [
-        {"chunk_id": 1, "chunk_text": "Q3 budget shows 15% revenue increase", "filepath": "/a.pdf", "chunk_index": 0, "rrf_score": 0.03},
-        {"chunk_id": 2, "chunk_text": "The weather today is sunny and warm",   "filepath": "/b.txt", "chunk_index": 0, "rrf_score": 0.02},
-    ]
-    results = reranker.rerank("quarterly revenue", candidates, top_k=2)
-    for r in results:
-        print(f"[rerank {r['rerank_score']:.4f}] {r['chunk_text']}")

searcher/search_engine.py

@@ -1,163 +0,0 @@
-# searcher/search_engine.py
-
-import yaml
-from searcher.query_understanding import QueryUnderstanding
-from searcher.dense_retriever import DenseRetriever
-from searcher.sparse_retriever import SparseRetriever
-from searcher.fusion_ranker import FusionRanker
-from searcher.reranker import Reranker
-from searcher.facet_filter import FacetFilter
-from searcher.highlighter import Highlighter
-
-
-class SearchEngine:
-    """
-    Orchestrates the full search pipeline end-to-end:
-
-        raw query
-            → QueryUnderstanding  (expand + rewrite)
-            → DenseRetriever      (semantic FAISS search)
-            → SparseRetriever     (BM25 lexical search)
-            → FusionRanker        (RRF merge)
-            → Reranker            (cross-encoder precision)
-            → FacetFilter         (type / date / size / directory)
-            → Highlighter         (preview + HTML highlights)
-            → final results
-    """
-
-    def __init__(self, config_path="config.yaml"):
-        self.config_path = config_path
-        with open(config_path) as f:
-            self.config = yaml.safe_load(f)
-
-        self.query_understanding = QueryUnderstanding(config_path)
-        self.dense_retriever = DenseRetriever(config_path)
-        self.sparse_retriever = SparseRetriever(config_path)
-        self.fusion_ranker = FusionRanker(k=60)
-        self.reranker = Reranker(config_path)
-        self.facet_filter = FacetFilter()
-        self.highlighter = Highlighter(preview_words=30)
-
-        self.candidate_k = self.config.get("candidate_k", 20)
-        self.final_k = self.config.get("top_k", 5)
-
-    def search(
-        self,
-        query: str,
-        top_k: int = None,
-        file_type: list[str] = None,
-        date_after=None,
-        date_before=None,
-        min_size: int = None,
-        max_size: int = None,
-        directory: str = None,
-    ) -> dict:
-        """
-        Run the full search pipeline.
-
-        Args:
-            query       — natural language user query
-            top_k       — number of final results (overrides config)
-            file_type   — e.g. [".pdf", ".docx"]
-            date_after  — datetime; exclude older files
-            date_before — datetime; exclude newer files
-            min_size    — min file size in bytes
-            max_size    — max file size in bytes
-            directory   — restrict to this directory
-
-        Returns:
-            dict:
-                query_info  — dict from QueryUnderstanding
-                results     — list of final result dicts, each with:
-                                filepath, chunk_text, chunk_index,
-                                preview, preview_html,
-                                dense_score (if present),
-                                sparse_score (if present),
-                                rrf_score, rerank_score
-        """
-        k = top_k or self.final_k
-
-        # Step 1 — query understanding
-        query_info = self.query_understanding.process(query)
-
-        query_info.setdefault("original", query)
-        query_info.setdefault("expanded", query)
-        query_info.setdefault("rewritten", query)
-
-        # Step 2 — dense retrieval (uses expanded query for better semantic reach)
-        dense_results = self.dense_retriever.retrieve(
-            query_info["expanded"], top_k=self.candidate_k
-        )
-
-        # Step 3 — sparse retrieval (uses rewritten query; expansion hurts BM25)
-        sparse_results = self.sparse_retriever.retrieve(
-            query_info["rewritten"], top_k=self.candidate_k
-        )
-
-        # Step 4 — RRF fusion
-        fused = self.fusion_ranker.fuse(dense_results, sparse_results, top_k=self.candidate_k)
-
-        # Step 5 — cross-encoder reranking
-        reranked = self.reranker.rerank(query_info["original"], fused, top_k=k * 2)
-
-        # Step 6 — facet filtering
-        filtered = self.facet_filter.filter(
-            reranked,
-            file_type=file_type,
-            date_after=date_after,
-            date_before=date_before,
-            min_size=min_size,
-            max_size=max_size,
-            directory=directory,
-        )
-
-        # Trim to top_k after filtering
-        final = filtered[:k]
-
-        # Step 7 — highlight previews
-        final = self.highlighter.annotate(final, query_info["original"])
-        for r in final:
-            if "preview" not in r or not r["preview"]:
-                r["preview"] = r.get("chunk_text", "")[:200]
-
-        return {
-            "query_info": query_info,
-            "results": final or [],
-        }
-
-if __name__ == "__main__":
-    engine = SearchEngine()
-
-    while True:
-        query = input("\n🔍 Enter your search query (or type 'exit'): ")
-
-        if query.lower() == "exit":
-            print("Exiting search engine...")
-            break
-
-        output = engine.search(query, top_k=3)
-
-        print(f"\nQuery     : {output['query_info']['original']}")
-        print(f"Expanded  : {output['query_info']['expanded']}")
-        print(f"Results   : {len(output['results'])}\n")
-
-        for i, r in enumerate(output["results"], 1):
-            print(f"--- Result {i} ---")
-            print(f"File     : {r['filepath']}")
-            print(f"Preview  : {r['preview']}")
-
-            # Handle safe printing of scores
-            rrf = r.get('rrf_score')
-            rerank = r.get('rerank_score')
-
-            if rrf is not None:
-                print(f"RRF      : {rrf:.5f}")
-            else:
-                print("RRF      : n/a")
-
-            if rerank is not None:
-                print(f"Rerank   : {rerank:.4f}")
-            else:
-                print("Rerank   : n/a")
-
-            print()

searcher/sparse_retriever.py

@@ -1,117 +0,0 @@
-# searcher/sparse_retriever.py
-
-import sqlite3
-import math
-import yaml
-from collections import defaultdict
-
-
-class SparseRetriever:
-    """
-    BM25 (Okapi BM25) lexical retrieval over the SQLite chunk store.
-
-    Why BM25 alongside semantic search?
-    - Dense retrieval can miss exact keyword matches (product codes, names, IDs)
-    - BM25 is great for rare/specific terms that embeddings smooth over
-    - Hybrid = best of both worlds
-
-    BM25 formula:
-        score(q, d) = Σ IDF(t) × (tf × (k1+1)) / (tf + k1 × (1 - b + b × dl/avgdl))
-    """
-
-    def __init__(self, config_path="config.yaml"):
-        with open(config_path) as f:
-            config = yaml.safe_load(f)
-
-        self.db_path = f"{config['data_dir']}/metadata.db"
-        self.k1 = 1.5   # term frequency saturation
-        self.b = 0.75   # length normalisation
-
-        # Build in-memory BM25 index from SQLite on startup
-        self._corpus = []       # list of (chunk_id, token_list)
-        self._avgdl = 0.0
-        self._df = defaultdict(int)   # term → doc frequency
-        self._build_index()
-
-    def _build_index(self):
-        """Load all chunks from SQLite and compute BM25 statistics."""
-        conn = sqlite3.connect(self.db_path)
-        rows = conn.execute("SELECT id, chunk_text FROM chunks").fetchall()
-        conn.close()
-
-        total_len = 0
-        for chunk_id, text in rows:
-            tokens = text.lower().split()
-            self._corpus.append((chunk_id, tokens))
-            total_len += len(tokens)
-            for token in set(tokens):
-                self._df[token] += 1
-
-        self._avgdl = total_len / len(rows) if rows else 1.0
-        self._N = len(rows)
-
-    def _idf(self, term: str) -> float:
-        """Inverse document frequency for a term."""
-        df = self._df.get(term, 0)
-        return math.log((self._N - df + 0.5) / (df + 0.5) + 1)
-
-    def retrieve(self, query: str, top_k: int = 20) -> list[dict]:
-        """
-        Run BM25 retrieval over the corpus.
-
-        Args:
-            query (str) — raw or rewritten query (NOT expanded — BM25 is lexical)
-            top_k (int) — number of results to return
-
-        Returns:
-            list[dict] with chunk_id and sparse_score, sorted descending
-        """
-        query_terms = query.lower().split()
-        scores = {}
-
-        for chunk_id, tokens in self._corpus:
-            dl = len(tokens)
-            score = 0.0
-            tf_map = defaultdict(int)
-            for t in tokens:
-                tf_map[t] += 1
-
-            for term in query_terms:
-                if term not in tf_map:
-                    continue
-                tf = tf_map[term]
-                idf = self._idf(term)
-                numerator = tf * (self.k1 + 1)
-                denominator = tf + self.k1 * (1 - self.b + self.b * dl / self._avgdl)
-                score += idf * numerator / denominator
-
-            if score > 0:
-                scores[chunk_id] = score
-
-        sorted_results = sorted(scores.items(), key=lambda x: x[1], reverse=True)[:top_k]
-
-        # Fetch text for top results
-        conn = sqlite3.connect(self.db_path)
-        results = []
-        for chunk_id, score in sorted_results:
-            row = conn.execute(
-                "SELECT chunk_text, filepath, chunk_index FROM chunks WHERE id = ?",
-                (chunk_id,)
-            ).fetchone()
-            if row:
-                results.append({
-                    "chunk_id": chunk_id,
-                    "chunk_text": row[0],
-                    "filepath": row[1],
-                    "chunk_index": row[2],
-                    "sparse_score": score,
-                })
-        conn.close()
-        return results
-
-
-if __name__ == "__main__":
-    sr = SparseRetriever()
-    results = sr.retrieve("quarterly budget", top_k=5)
-    for r in results:
-        print(f"[{r['sparse_score']:.4f}] {r['filepath']} → {r['chunk_text'][:80]}")