Upload 21 files

.dockerignore

@@ -0,0 +1,15 @@
+.git
+.gitignore
+.venv
+__pycache__/
+*.pyc
+*.pyo
+*.pyd
+*.log
+.pytest_cache/
+.mypy_cache/
+.ruff_cache/
+.idea/
+.vscode/
+data/
+results/

.gitignore

@@ -0,0 +1,34 @@
+# Virtual environment
+.venv/
+
+# Vector index and database (large binary files)
+data/
+results/
+
+# Logs
+*.log
+
+# Downloaded ML models (auto-downloaded at runtime)
+models/
+.cache/
+sentence_transformers/
+
+# Python cache
+___pycache__/
+*.pyc
+*.pyo
+*.pyd
+.env
+venv/
+env/
+.venv/
+# Model cache
+.cache/
+
+# OS files
+.DS_Store
+Thumbs.db
+
+# IDE
+.vscode/
+.idea/

Dockerfile

@@ -0,0 +1,20 @@
+FROM python:3.10
+
+ENV PYTHONDONTWRITEBYTECODE=1 \
+    PYTHONUNBUFFERED=1 \
+    PIP_NO_CACHE_DIR=1 \
+    NLTK_DATA=/usr/local/share/nltk_data
+
+WORKDIR /app
+
+COPY requirements.txt .
+
+RUN pip install --upgrade pip && \
+    pip install -r requirements.txt && \
+    python -c "import nltk; nltk.download('wordnet', download_dir='/usr/local/share/nltk_data'); nltk.download('omw-1.4', download_dir='/usr/local/share/nltk_data')"
+
+COPY . .
+
+EXPOSE 7860
+
+CMD ["uvicorn", "main:app", "--host", "0.0.0.0", "--port", "7860"]

check_nfcorpus.py

@@ -0,0 +1,29 @@
+import json
+import sys
+import os
+
+sys.path.append(os.path.abspath("."))
+# Load results
+with open('results/eval_nfcorpus.json') as f:
+    data = json.load(f)
+
+# Load qrels
+from evaluation.dataset_loader import DatasetLoader
+
+loader = DatasetLoader('data/nfcorpus')
+qrels = loader.load_qrels()
+
+# 🔍 Debug prints
+print("Sample RESULT query_id:", list(data.keys())[0])
+
+first_qid = list(qrels.keys())[0]
+print("Sample QREL query_id:", first_qid)
+
+print("Sample QREL doc_id:", list(qrels[first_qid].keys())[0])
+
+print("Total QREL queries:", len(qrels))
+print("Total RESULT queries:", len(data))
+
+# 🔥 Check overlap
+common = set(data.keys()) & set(qrels.keys())
+print("Common query IDs:", len(common))

config.yaml

@@ -0,0 +1,42 @@
+# Directories to index
+watch_paths:
+  - ./data/scifact
+  - ./data/nfcorpus #modify this
+
+# File extensions to include
+include_extensions:
+  - ".pdf"
+  - ".docx"
+  - ".txt"
+  - ".md"
+  - ".pptx"
+  - ".xlsx"
+  - ".py"
+  - ".js"
+  - ".ipynb"
+
+
+# --- Add these ---
+top_k: 5                  # final results returned to user
+candidate_k: 20           # candidates fetched before reranking
+
+query_expansion: true     # WordNet synonym expansion
+max_synonyms: 5           # max synonyms to append
+
+reranking_enabled: true   # cross-encoder reranking
+reranker_model: "cross-encoder/ms-marco-MiniLM-L-6-v2"
+
+# Directories to skip
+skip_directories:
+  - ".git"
+  - "node_modules"
+  - "__pycache__"
+  - ".venv"
+
+# Where to store index data
+data_dir: "./data"
+
+embedding_model: "all-MiniLM-L6-v2"
+# embedding_model: BAAI/bge-small-en-v1.5
+
+debounce_seconds: 5

docker-compose.yml

@@ -0,0 +1,16 @@
+services:
+  semantic-search:
+    build:
+      context: .
+      dockerfile: Dockerfile
+    container_name: semantic-search
+    ports:
+      - "7860:7860"
+    environment:
+      NLTK_DATA: /usr/local/share/nltk_data
+    volumes:
+      - ./config.yaml:/app/config.yaml:ro
+      - ./data:/app/data
+      - ./results:/app/results
+      - ./documents:/documents
+    restart: unless-stopped

main.py

@@ -0,0 +1,269 @@
+# main.py
+
+import json
+import os
+import time
+import yaml
+from fastapi import FastAPI, Request, Form
+from fastapi.responses import HTMLResponse
+from fastapi.staticfiles import StaticFiles
+from fastapi.templating import Jinja2Templates
+
+from searcher.search_engine import SearchEngine
+from evaluation.dataset_loader import DatasetLoader
+
+app = FastAPI(title="Semantic Search Engine")
+
+app.mount("/static", StaticFiles(directory="static"), name="static")
+templates = Jinja2Templates(directory="templates")
+
+# ── load search engine once at startup ──────────────────────────────────────
+def get_engine():
+    from searcher.search_engine import SearchEngine
+    return SearchEngine("config.yaml")
+
+engine = get_engine()
+
+
+# ── load dataset queries at startup ─────────────────────────────────────────
+# These are the actual queries from SciFact and NFCorpus
+# We use them to show "which dataset queries matched your search"
+
+def load_dataset_queries() -> dict:
+    """
+    Load all queries from SciFact and NFCorpus at startup.
+
+    Returns:
+        dict — {
+            "scifact":  {query_id: query_text, ...},
+            "nfcorpus": {query_id: query_text, ...},
+        }
+    """
+    all_queries = {}
+
+    datasets = {
+        "scifact":  "data/scifact",
+        "nfcorpus": "data/nfcorpus",
+    }
+
+    for name, path in datasets.items():
+        if os.path.exists(path):
+            try:
+                loader             = DatasetLoader(path)
+                all_queries[name]  = loader.load_queries()
+                print(f"[Startup] Loaded {len(all_queries[name])} queries from {name}")
+            except Exception as e:
+                print(f"[Startup] Could not load {name} queries: {e}")
+                all_queries[name] = {}
+        else:
+            print(f"[Startup] Dataset path not found: {path}")
+            all_queries[name] = {}
+
+    return all_queries
+
+
+# load once at startup — available globally
+DATASET_QUERIES = load_dataset_queries()
+
+
+# ── helpers ──────────────────────────────────────────────────────────────────
+
+def load_eval_results() -> dict:
+    path = "results/eval_all.json"
+    if os.path.exists(path):
+        with open(path, "r") as f:
+            return json.load(f)
+    return {}
+
+
+def extract_doc_id(filepath: str) -> str:
+    if "://" in filepath:
+        return filepath.split("://", 1)[1]
+    return filepath
+
+
+def get_dataset_from_filepath(filepath: str) -> str:
+    if "scifact://"  in filepath: return "scifact"
+    if "nfcorpus://" in filepath: return "nfcorpus"
+    return "filesystem"
+
+
+def get_file_icon(filepath: str) -> str:
+    if "scifact://"  in filepath: return "🔬"
+    if "nfcorpus://" in filepath: return "🏥"
+    ext   = filepath.lower().split(".")[-1] if "." in filepath else ""
+    icons = {
+        "pdf": "📄", "docx": "📝", "txt": "📃",
+        "pptx": "📊", "xlsx": "📋", "py": "🐍",
+    }
+    return icons.get(ext, "📄")
+
+
+def find_matching_dataset_queries(
+    user_query: str,
+    top_results: list,
+) -> list:
+    """
+    Find which dataset queries are semantically related to what the user typed.
+
+    Strategy — two passes:
+        1. Exact / substring match  — query text contains user words
+        2. Doc-based match          — if a result doc came from dataset X,
+                                      show the queries that reference that doc
+                                      from the qrels (loaded separately)
+
+    We use simple word overlap here (no extra model call needed).
+
+    Returns:
+        list of dicts — [
+            {
+                "query_id":   "1234",
+                "query_text": "Does vitamin D cause cancer?",
+                "dataset":    "scifact",
+                "match_type": "text"   or "doc"
+            },
+            ...
+        ]
+    """
+    matched   = []
+    seen_ids  = set()
+
+    # words from user query — lowercase, skip short words
+    user_words = set(
+        w.lower() for w in user_query.split()
+        if len(w) > 3
+    )
+
+    # Pass 1 — text overlap match
+    # check every dataset query for word overlap with user query
+    for dataset_name, queries in DATASET_QUERIES.items():
+        for qid, qtext in queries.items():
+            q_words = set(w.lower() for w in qtext.split() if len(w) > 3)
+            overlap = user_words & q_words
+
+            # need at least 1 word overlap
+            if overlap and qid not in seen_ids:
+                matched.append({
+                    "query_id":   qid,
+                    "query_text": qtext,
+                    "dataset":    dataset_name,
+                    "match_type": "text",
+                    "overlap":    len(overlap),
+                })
+                seen_ids.add(qid)
+
+    # sort by overlap count — most overlapping queries first
+    matched.sort(key=lambda x: x["overlap"], reverse=True)
+
+    # return top 8 matched queries max
+    return matched[:8]
+
+
+# ── routes ───────────────────────────────────────────────────────────────────
+
+@app.get("/", response_class=HTMLResponse)
+async def home(request: Request):
+    return templates.TemplateResponse("index.html", {
+        "request":          request,
+        "scifact_count":    len(DATASET_QUERIES.get("scifact",  {})),
+        "nfcorpus_count":   len(DATASET_QUERIES.get("nfcorpus", {})),
+    })
+
+
+@app.post("/search", response_class=HTMLResponse)
+async def search(
+    request: Request,
+    query:   str = Form(...),
+    top_k:   int = Form(10),
+    mode:    str = Form("full"),
+):
+    if not query.strip():
+        return templates.TemplateResponse("index.html", {
+            "request": request,
+            "error":   "Please enter a search query.",
+        })
+
+    t0      = time.time()
+    output  = engine.search(query.strip(), top_k=top_k)
+    elapsed = round(time.time() - t0, 3)
+
+    # format search results
+    results = []
+    for r in output.get("results", []):
+        filepath = r.get("filepath", "")
+        doc_id   = extract_doc_id(filepath)
+        score    = r.get("rerank_score", r.get("rrf_score", r.get("dense_score", 0)))
+        snippet  = r.get("chunk_text", r.get("text", "No preview available."))
+
+        if len(snippet) > 200:
+            snippet = snippet[:200].rsplit(" ", 1)[0] + "..."
+
+        dataset = get_dataset_from_filepath(filepath)
+
+        results.append({
+            "doc_id":   doc_id,
+            "filepath": filepath,
+            "score":    round(float(score), 4),
+            "snippet":  snippet,
+            "icon":     get_file_icon(filepath),
+            "dataset":  dataset,
+        })
+
+    # find matching dataset queries
+    matched_queries = find_matching_dataset_queries(query.strip(), results)
+
+    # group matched queries by dataset for display
+    matched_scifact  = [q for q in matched_queries if q["dataset"] == "scifact"]
+    matched_nfcorpus = [q for q in matched_queries if q["dataset"] == "nfcorpus"]
+
+    return templates.TemplateResponse("results.html", {
+        "request":          request,
+        "query":            query,
+        "results":          results,
+        "total":            len(results),
+        "elapsed":          elapsed,
+        "mode":             mode,
+        "top_k":            top_k,
+        "matched_scifact":  matched_scifact,
+        "matched_nfcorpus": matched_nfcorpus,
+        "total_matched":    len(matched_queries),
+    })
+
+
+@app.get("/dashboard", response_class=HTMLResponse)
+async def dashboard(request: Request):
+    eval_data = load_eval_results()
+
+    datasets = []
+    for dataset_name, mode_results in eval_data.items():
+        full = mode_results.get("full", {})
+        datasets.append({
+            "name":      dataset_name,
+            "ndcg":      full.get("NDCG@10",    0.0),
+            "mrr":       full.get("MRR",         0.0),
+            "map":       full.get("MAP@100",     0.0),
+            "recall":    full.get("Recall@100",  0.0),
+            "precision": full.get("P@10",        0.0),
+            "queries":   full.get("num_queries", 0),
+            "modes":     mode_results,
+        })
+
+    return templates.TemplateResponse("dashboard.html", {
+        "request":  request,
+        "datasets": datasets,
+    })
+
+
+@app.get("/health")
+async def health():
+    return {"status": "ok"}
+
+
+if __name__ == "__main__":
+    import uvicorn
+    uvicorn.run("main:app", host="0.0.0.0", port=7860, reload=True)
+
+
+
+
+    # uvicorn main:app --reload --host 0.0.0.0 --port 8000

requirements.txt

@@ -0,0 +1,58 @@
+annotated-doc==0.0.4
+anyio==4.12.1
+certifi==2026.2.25
+click==8.3.1
+colorama==0.4.6
+et_xmlfile==2.0.0
+faiss-cpu==1.13.2
+filelock==3.25.2
+fsspec==2026.2.0
+h11==0.16.0
+hf-xet==1.4.2
+httpcore==1.0.9
+httpx==0.28.1
+huggingface_hub==1.7.1
+idna==3.11
+Jinja2==3.1.6
+joblib==1.5.3
+lxml==6.0.2
+markdown-it-py==4.0.0
+MarkupSafe==3.0.3
+mdurl==0.1.2
+mpmath==1.3.0
+networkx==3.6.1
+numpy==2.4.3
+openpyxl==3.1.5
+packaging==26.0
+pillow==12.1.1
+Pygments==2.19.2
+PyMuPDF==1.27.2
+python-docx==1.2.0
+python-pptx==1.0.2
+PyYAML==6.0.3
+regex==2026.2.28
+rich==14.3.3
+safetensors==0.7.0
+scikit-learn==1.8.0
+scipy==1.17.1
+sentence-transformers==5.3.0
+setuptools==82.0.1
+shellingham==1.5.4
+sympy==1.14.0
+threadpoolctl==3.6.0
+tokenizers==0.22.2
+torch==2.10.0
+tqdm==4.67.3
+transformers==5.3.0
+typer==0.24.1
+typing_extensions==4.15.0
+watchdog==6.0.0
+xlsxwriter==3.2.9
+# add these
+nltk          # WordNet query expansion
+rank-bm25     # optional, but our SparseRetriever is self-contained
+fastapi       # phase 4 API
+uvicorn       # phase 4 API server
+typer         # phase 4 CLI
+jinja2
+python-multipart

searcher/dense_retriever.py

@@ -0,0 +1,79 @@
+# searcher/dense_retriever.py
+
+import numpy as np
+import faiss
+import sqlite3
+import yaml
+from indexer.embedder import Embedder
+
+
+class DenseRetriever:
+    """
+    Embeds the (expanded) query and searches the FAISS index for
+    nearest-neighbor chunks by semantic similarity.
+
+    This is the core semantic search — finds chunks that MEAN the same
+    thing as the query, even if they share no keywords.
+    """
+
+    def __init__(self, config_path="config.yaml"):
+        with open(config_path) as f:
+            config = yaml.safe_load(f)
+
+        self.data_dir = config["data_dir"]
+        self.faiss_path = f"{self.data_dir}/index.faiss"
+        self.db_path = f"{self.data_dir}/metadata.db"
+        self.top_k = config.get("top_k", 20)  # fetch more than needed; reranker will trim
+
+        self.embedder = Embedder(config_path)
+        self.index = faiss.read_index(self.faiss_path)
+
+    def retrieve(self, query: str, top_k: int = None) -> list[dict]:
+        """
+        Embed the query and search FAISS for the closest chunk vectors.
+
+        Args:
+            query (str) — expanded query string
+            top_k (int) — number of results
+
+        Returns:
+            list[dict] each containing:
+                chunk_id, chunk_text, filepath, chunk_index,
+                dense_score (float, lower = more similar in L2 space)
+        """
+        k = top_k or self.top_k
+
+        query_vec = self.embedder.embed_single(query)
+        query_vec = np.array([query_vec], dtype="float32")
+
+        distances, ids = self.index.search(query_vec, k)
+
+        results = []
+        conn = sqlite3.connect(self.db_path)
+        cursor = conn.cursor()
+
+        for dist, chunk_id in zip(distances[0], ids[0]):
+            if chunk_id == -1:
+                continue
+            row = cursor.execute(
+                "SELECT chunk_text, filepath, chunk_index FROM chunks WHERE id = ?",
+                (int(chunk_id),)
+            ).fetchone()
+            if row:
+                results.append({
+                    "chunk_id": int(chunk_id),
+                    "chunk_text": row[0],
+                    "filepath": row[1],
+                    "chunk_index": row[2],
+                    "dense_score": float(dist),
+                })
+
+        conn.close()
+        return results
+
+
+if __name__ == "__main__":
+    dr = DenseRetriever()
+    results = dr.retrieve("quarterly budget report", top_k=5)
+    for r in results:
+        print(f"[{r['dense_score']:.4f}] {r['filepath']} → {r['chunk_text'][:80]}")

searcher/facet_filter.py

@@ -0,0 +1,83 @@
+# searcher/facet_filter.py
+
+import os
+from datetime import datetime
+
+
+class FacetFilter:
+    """
+    Post-retrieval filtering by structured metadata.
+    Applied AFTER fusion and reranking to restrict results
+    without hurting semantic quality.
+
+    Supported facets:
+        file_type   — e.g. [".pdf", ".docx"]
+        date_after  — only files modified after this date
+        date_before — only files modified before this date
+        min_size    — minimum file size in bytes
+        max_size    — maximum file size in bytes
+        directory   — only files inside this directory path
+    """
+
+    def filter(
+        self,
+        results: list[dict],
+        file_type: list[str] = None,
+        date_after: datetime = None,
+        date_before: datetime = None,
+        min_size: int = None,
+        max_size: int = None,
+        directory: str = None,
+    ) -> list[dict]:
+        """
+        Apply facet filters to a list of search results.
+
+        Args:
+            results     — list of chunk dicts (must include 'filepath')
+            file_type   — list of extensions e.g. [".pdf", ".txt"]
+            date_after  — datetime object; exclude files older than this
+            date_before — datetime object; exclude files newer than this
+            min_size    — minimum file size in bytes
+            max_size    — maximum file size in bytes
+            directory   — restrict to files under this directory
+
+        Returns:
+            list[dict] — filtered results (same structure, subset)
+        """
+        filtered = []
+
+        for result in results:
+            fp = result["filepath"]
+
+            # File type filter
+            if file_type:
+                ext = os.path.splitext(fp)[1].lower()
+                if ext not in [e.lower() for e in file_type]:
+                    continue
+
+            # Directory scope filter
+            if directory:
+                if not fp.startswith(os.path.abspath(directory)):
+                    continue
+
+            # File system checks (only if file still exists)
+            if os.path.exists(fp):
+                stat = os.stat(fp)
+
+                # Date filters
+                mtime = datetime.fromtimestamp(stat.st_mtime)
+                if date_after and mtime < date_after:
+                    continue
+                if date_before and mtime > date_before:
+                    continue
+
+                # Size filters
+                size = stat.st_size
+                if min_size and size < min_size:
+                    continue
+                if max_size and size > max_size:
+                    continue
+
+            filtered.append(result)
+
+        return filtered

searcher/fusion_ranker.py

@@ -0,0 +1,80 @@
+# searcher/fusion_ranker.py
+
+
+class FusionRanker:
+    """
+    Reciprocal Rank Fusion (RRF) merges dense and sparse result lists
+    into a single ranked list without needing to normalise scores.
+
+    Why RRF instead of score averaging?
+    - Dense scores (L2 distance) and BM25 scores live on different scales
+    - Normalising them requires knowing min/max, which varies per query
+    - RRF uses only the RANK of each result, making it scale-invariant
+
+    RRF formula:
+        score(d) = Σ 1 / (k + rank(d, list_i))
+        where k=60 is a constant that dampens the effect of very high ranks
+    """
+
+    def __init__(self, k: int = 60):
+        self.k = k
+
+    def fuse(
+        self,
+        dense_results: list[dict],
+        sparse_results: list[dict],
+        top_k: int = 10,
+    ) -> list[dict]:
+        """
+        Fuse two ranked result lists using Reciprocal Rank Fusion.
+
+        Args:
+            dense_results  — from DenseRetriever, ordered by dense_score ascending
+                             (lower L2 distance = better)
+            sparse_results — from SparseRetriever, ordered by sparse_score descending
+            top_k          — number of results to return after fusion
+
+        Returns:
+            list[dict] — merged results sorted by rrf_score descending,
+                         each dict includes all original fields + rrf_score
+        """
+        rrf_scores = {}
+        chunk_data = {}
+
+        # Dense list: lower distance = better rank = higher RRF contribution
+        for rank, result in enumerate(dense_results):
+            cid = result["chunk_id"]
+            rrf_scores[cid] = rrf_scores.get(cid, 0) + 1 / (self.k + rank + 1)
+            chunk_data[cid] = result
+
+        # Sparse list: higher BM25 score = better rank = higher RRF contribution
+        for rank, result in enumerate(sparse_results):
+            cid = result["chunk_id"]
+            rrf_scores[cid] = rrf_scores.get(cid, 0) + 1 / (self.k + rank + 1)
+            if cid not in chunk_data:
+                chunk_data[cid] = result
+
+        sorted_ids = sorted(rrf_scores, key=lambda x: rrf_scores[x], reverse=True)[:top_k]
+
+        fused = []
+        for cid in sorted_ids:
+            entry = dict(chunk_data[cid])
+            entry["rrf_score"] = rrf_scores[cid]
+            fused.append(entry)
+
+        return fused
+
+
+if __name__ == "__main__":
+    dense = [
+        {"chunk_id": 1, "chunk_text": "budget summary Q3", "filepath": "/a.pdf", "chunk_index": 0, "dense_score": 0.12},
+        {"chunk_id": 2, "chunk_text": "revenue report",     "filepath": "/b.pdf", "chunk_index": 0, "dense_score": 0.25},
+    ]
+    sparse = [
+        {"chunk_id": 2, "chunk_text": "revenue report",    "filepath": "/b.pdf", "chunk_index": 0, "sparse_score": 8.1},
+        {"chunk_id": 3, "chunk_text": "financial overview", "filepath": "/c.pdf", "chunk_index": 0, "sparse_score": 5.2},
+    ]
+    ranker = FusionRanker()
+    results = ranker.fuse(dense, sparse, top_k=5)
+    for r in results:
+        print(f"[RRF {r['rrf_score']:.5f}] {r['filepath']} → {r['chunk_text'][:60]}")

searcher/highlighter.py

@@ -0,0 +1,96 @@
+# searcher/highlighter.py
+
+
+class Highlighter:
+    """
+    Extracts the most relevant passage from a chunk and highlights
+    query-matching terms for display in search results.
+
+    Why highlight?
+    - The full chunk may be 500 words; the user needs a ~2-sentence preview
+    - Highlighting query terms helps users quickly judge relevance
+    - This is purely display logic — does not affect ranking
+    """
+
+    def __init__(self, preview_words: int = 30):
+        self.preview_words = preview_words
+
+    def extract_preview(self, chunk_text: str, query: str) -> str:
+        """
+        Find the sentence in chunk_text most relevant to the query
+        and return a short preview around it.
+
+        Strategy: find the window of preview_words words that contains
+        the most query term matches.
+
+        Args:
+            chunk_text  — full text of the chunk
+            query       — original user query
+
+        Returns:
+            str — the best preview snippet
+        """
+        words = chunk_text.split()
+        if len(words) <= self.preview_words:
+            return chunk_text
+
+        query_terms = set(query.lower().split())
+        best_score = -1
+        best_start = 0
+
+        for i in range(len(words) - self.preview_words + 1):
+            window = words[i: i + self.preview_words]
+            score = sum(1 for w in window if w.lower().strip(".,;:") in query_terms)
+            if score > best_score:
+                best_score = score
+                best_start = i
+
+        snippet = " ".join(words[best_start: best_start + self.preview_words])
+
+        # Add ellipsis if truncated
+        if best_start > 0:
+            snippet = "..." + snippet
+        if best_start + self.preview_words < len(words):
+            snippet = snippet + "..."
+
+        return snippet
+
+    def highlight_html(self, text: str, query: str) -> str:
+        """
+        Wrap query-matching words in <mark> tags for HTML display.
+
+        Args:
+            text  — preview snippet
+            query — original user query
+
+        Returns:
+            str — HTML string with <mark> tags around matching words
+        """
+        query_terms = set(query.lower().split())
+        highlighted_words = []
+
+        for word in text.split():
+            clean = word.lower().strip(".,;:!?")
+            if clean in query_terms:
+                highlighted_words.append(f"<mark>{word}</mark>")
+            else:
+                highlighted_words.append(word)
+
+        return " ".join(highlighted_words)
+
+    def annotate(self, results: list[dict], query: str) -> list[dict]:
+        """
+        Add preview and highlighted_preview to each result dict.
+
+        Args:
+            results — list of chunk dicts
+            query   — original user query
+
+        Returns:
+            list[dict] — same results with 'preview' and 'preview_html' added
+        """
+        for result in results:
+            preview = self.extract_preview(result["chunk_text"], query)
+            result["preview"] = preview
+            result["preview_html"] = self.highlight_html(preview, query)
+        return results

searcher/query_understanding.py

@@ -0,0 +1,104 @@
+# searcher/query_understanding.py
+
+import yaml
+from nltk.corpus import wordnet
+import nltk
+
+nltk.download("wordnet", quiet=True)
+nltk.download("omw-1.4", quiet=True)
+
+
+class QueryUnderstanding:
+    """
+    Expands and rewrites the raw user query before retrieval.
+
+    Two strategies:
+    1. WordNet synonym expansion — "apples" → ["apple", "malus", "orchard", "fruit"]
+    2. (Optional) T5 query rewriting — rephrase for better embedding alignment
+
+    Why expand?
+    - A search for "apples" should also find "fruit", "orchard", "nutrition"
+    - Embedding models are good but can miss synonyms in different domains
+    - Expansion bridges the vocabulary gap before we even hit FAISS
+    """
+
+    def __init__(self, config_path="config.yaml"):
+        with open(config_path) as f:
+            config = yaml.safe_load(f)
+        self.max_synonyms = config.get("max_synonyms", 5)
+        self.expansion_enabled = config.get("query_expansion", True)
+
+    def expand(self, query: str) -> str:
+        """
+        Expand the query using WordNet synonyms.
+
+        Args:
+            query (str) — raw user query
+
+        Returns:
+            str — expanded query string (original + synonyms appended)
+
+        Example:
+            "apple nutrition" → "apple nutrition fruit malus orchard dietary"
+        """
+        if not self.expansion_enabled:
+            return query
+
+        words = query.lower().split()
+        synonyms = set()
+
+        for word in words:
+            for syn in wordnet.synsets(word):
+                for lemma in syn.lemmas():
+                    cleaned = lemma.name().replace("_", " ")
+                    if cleaned.lower() != word:
+                        synonyms.add(cleaned)
+                if len(synonyms) >= self.max_synonyms:
+                    break
+
+        expanded = query
+        if synonyms:
+            expanded = query + " " + " ".join(list(synonyms)[:self.max_synonyms])
+
+        return expanded
+
+    def rewrite(self, query: str) -> str:
+        """
+        Lightweight query rewriting — normalise and clean the query.
+        (Plug in a T5/FLAN-T5 model here for more powerful rewriting.)
+
+        Args:
+            query (str) — raw user query
+
+        Returns:
+            str — cleaned query
+        """
+        # Basic normalisation: strip extra spaces, lowercase
+        query = " ".join(query.strip().split())
+        return query
+
+    def process(self, query: str) -> dict:
+        """
+        Full query understanding pipeline.
+
+        Returns:
+            dict with:
+                original   — the raw input
+                rewritten  — cleaned version
+                expanded   — synonym-expanded version for dense retrieval
+        """
+        rewritten = self.rewrite(query)
+        expanded = self.expand(rewritten)
+        return {
+            "original": query,
+            "rewritten": rewritten,
+            "expanded": expanded,
+        }
+
+
+if __name__ == "__main__":
+    qu = QueryUnderstanding()
+    result = qu.process("quarterly budget report")
+    print(f"Original : {result['original']}")
+    print(f"Rewritten: {result['rewritten']}")
+    print(f"Expanded : {result['expanded']}")

searcher/reranker.py

@@ -0,0 +1,66 @@
+# searcher/reranker.py
+
+import yaml
+from sentence_transformers import CrossEncoder
+
+
+class Reranker:
+    """
+    Cross-encoder reranking for precision improvement on top-k candidates.
+
+    Why rerank after fusion?
+    - Bi-encoders (used in DenseRetriever) embed query and doc INDEPENDENTLY
+      — fast but they can't model their interaction
+    - Cross-encoders process (query, document) TOGETHER — much more accurate
+      but too slow to run over the whole corpus
+    - Solution: use bi-encoder to shortlist ~20 candidates, then cross-encoder
+      to precisely rank them → speed of FAISS + accuracy of cross-encoder
+
+    Model: ms-marco-MiniLM-L-6-v2 (fast, strong on passage relevance)
+    """
+
+    def __init__(self, config_path="config.yaml"):
+        with open(config_path) as f:
+            config = yaml.safe_load(f)
+
+        model_name = config.get("reranker_model", "cross-encoder/ms-marco-MiniLM-L-6-v2")
+        self.enabled = config.get("reranking_enabled", True)
+        if self.enabled:
+            print(f"Loading reranker model '{model_name}'...")
+            self.model = CrossEncoder(model_name)
+            print("Reranker loaded.")
+
+    def rerank(self, query: str, candidates: list[dict], top_k: int = 5) -> list[dict]:
+        """
+        Rerank candidates using cross-encoder relevance scores.
+
+        Args:
+            query      — original (not expanded) user query
+            candidates — list of chunk dicts from FusionRanker
+            top_k      — number of final results to return
+
+        Returns:
+            list[dict] — reranked results, each with rerank_score added
+        """
+        if not self.enabled or not candidates:
+            return candidates[:top_k]
+
+        pairs = [(query, c["chunk_text"]) for c in candidates]
+        scores = self.model.predict(pairs)
+
+        for i, score in enumerate(scores):
+            candidates[i]["rerank_score"] = float(score)
+
+        reranked = sorted(candidates, key=lambda x: x["rerank_score"], reverse=True)
+        return reranked[:top_k]
+
+
+if __name__ == "__main__":
+    reranker = Reranker()
+    candidates = [
+        {"chunk_id": 1, "chunk_text": "Q3 budget shows 15% revenue increase", "filepath": "/a.pdf", "chunk_index": 0, "rrf_score": 0.03},
+        {"chunk_id": 2, "chunk_text": "The weather today is sunny and warm",   "filepath": "/b.txt", "chunk_index": 0, "rrf_score": 0.02},
+    ]
+    results = reranker.rerank("quarterly revenue", candidates, top_k=2)
+    for r in results:
+        print(f"[rerank {r['rerank_score']:.4f}] {r['chunk_text']}")

searcher/search_engine.py

@@ -0,0 +1,163 @@
+# searcher/search_engine.py
+
+import yaml
+from searcher.query_understanding import QueryUnderstanding
+from searcher.dense_retriever import DenseRetriever
+from searcher.sparse_retriever import SparseRetriever
+from searcher.fusion_ranker import FusionRanker
+from searcher.reranker import Reranker
+from searcher.facet_filter import FacetFilter
+from searcher.highlighter import Highlighter
+
+
+class SearchEngine:
+    """
+    Orchestrates the full search pipeline end-to-end:
+
+        raw query
+            → QueryUnderstanding  (expand + rewrite)
+            → DenseRetriever      (semantic FAISS search)
+            → SparseRetriever     (BM25 lexical search)
+            → FusionRanker        (RRF merge)
+            → Reranker            (cross-encoder precision)
+            → FacetFilter         (type / date / size / directory)
+            → Highlighter         (preview + HTML highlights)
+            → final results
+    """
+
+    def __init__(self, config_path="config.yaml"):
+        self.config_path = config_path
+        with open(config_path) as f:
+            self.config = yaml.safe_load(f)
+
+        self.query_understanding = QueryUnderstanding(config_path)
+        self.dense_retriever = DenseRetriever(config_path)
+        self.sparse_retriever = SparseRetriever(config_path)
+        self.fusion_ranker = FusionRanker(k=60)
+        self.reranker = Reranker(config_path)
+        self.facet_filter = FacetFilter()
+        self.highlighter = Highlighter(preview_words=30)
+
+        self.candidate_k = self.config.get("candidate_k", 20)
+        self.final_k = self.config.get("top_k", 5)
+
+    def search(
+        self,
+        query: str,
+        top_k: int = None,
+        file_type: list[str] = None,
+        date_after=None,
+        date_before=None,
+        min_size: int = None,
+        max_size: int = None,
+        directory: str = None,
+    ) -> dict:
+        """
+        Run the full search pipeline.
+
+        Args:
+            query       — natural language user query
+            top_k       — number of final results (overrides config)
+            file_type   — e.g. [".pdf", ".docx"]
+            date_after  — datetime; exclude older files
+            date_before — datetime; exclude newer files
+            min_size    — min file size in bytes
+            max_size    — max file size in bytes
+            directory   — restrict to this directory
+
+        Returns:
+            dict:
+                query_info  — dict from QueryUnderstanding
+                results     — list of final result dicts, each with:
+                                filepath, chunk_text, chunk_index,
+                                preview, preview_html,
+                                dense_score (if present),
+                                sparse_score (if present),
+                                rrf_score, rerank_score
+        """
+        k = top_k or self.final_k
+
+        # Step 1 — query understanding
+        query_info = self.query_understanding.process(query)
+
+        query_info.setdefault("original", query)
+        query_info.setdefault("expanded", query)
+        query_info.setdefault("rewritten", query)
+
+        # Step 2 — dense retrieval (uses expanded query for better semantic reach)
+        dense_results = self.dense_retriever.retrieve(
+            query_info["expanded"], top_k=self.candidate_k
+        )
+
+        # Step 3 — sparse retrieval (uses rewritten query; expansion hurts BM25)
+        sparse_results = self.sparse_retriever.retrieve(
+            query_info["rewritten"], top_k=self.candidate_k
+        )
+
+        # Step 4 — RRF fusion
+        fused = self.fusion_ranker.fuse(dense_results, sparse_results, top_k=self.candidate_k)
+
+        # Step 5 — cross-encoder reranking
+        reranked = self.reranker.rerank(query_info["original"], fused, top_k=k * 2)
+
+        # Step 6 — facet filtering
+        filtered = self.facet_filter.filter(
+            reranked,
+            file_type=file_type,
+            date_after=date_after,
+            date_before=date_before,
+            min_size=min_size,
+            max_size=max_size,
+            directory=directory,
+        )
+
+        # Trim to top_k after filtering
+        final = filtered[:k]
+
+        # Step 7 — highlight previews
+        final = self.highlighter.annotate(final, query_info["original"])
+        for r in final:
+            if "preview" not in r or not r["preview"]:
+                r["preview"] = r.get("chunk_text", "")[:200]
+
+        return {
+            "query_info": query_info,
+            "results": final or [],
+        }
+
+if __name__ == "__main__":
+    engine = SearchEngine()
+
+    while True:
+        query = input("\n🔍 Enter your search query (or type 'exit'): ")
+
+        if query.lower() == "exit":
+            print("Exiting search engine...")
+            break
+
+        output = engine.search(query, top_k=3)
+
+        print(f"\nQuery     : {output['query_info']['original']}")
+        print(f"Expanded  : {output['query_info']['expanded']}")
+        print(f"Results   : {len(output['results'])}\n")
+
+        for i, r in enumerate(output["results"], 1):
+            print(f"--- Result {i} ---")
+            print(f"File     : {r['filepath']}")
+            print(f"Preview  : {r['preview']}")
+
+            # Handle safe printing of scores
+            rrf = r.get('rrf_score')
+            rerank = r.get('rerank_score')
+
+            if rrf is not None:
+                print(f"RRF      : {rrf:.5f}")
+            else:
+                print("RRF      : n/a")
+
+            if rerank is not None:
+                print(f"Rerank   : {rerank:.4f}")
+            else:
+                print("Rerank   : n/a")
+
+            print()

searcher/sparse_retriever.py

@@ -0,0 +1,117 @@
+# searcher/sparse_retriever.py
+
+import sqlite3
+import math
+import yaml
+from collections import defaultdict
+
+
+class SparseRetriever:
+    """
+    BM25 (Okapi BM25) lexical retrieval over the SQLite chunk store.
+
+    Why BM25 alongside semantic search?
+    - Dense retrieval can miss exact keyword matches (product codes, names, IDs)
+    - BM25 is great for rare/specific terms that embeddings smooth over
+    - Hybrid = best of both worlds
+
+    BM25 formula:
+        score(q, d) = Σ IDF(t) × (tf × (k1+1)) / (tf + k1 × (1 - b + b × dl/avgdl))
+    """
+
+    def __init__(self, config_path="config.yaml"):
+        with open(config_path) as f:
+            config = yaml.safe_load(f)
+
+        self.db_path = f"{config['data_dir']}/metadata.db"
+        self.k1 = 1.5   # term frequency saturation
+        self.b = 0.75   # length normalisation
+
+        # Build in-memory BM25 index from SQLite on startup
+        self._corpus = []       # list of (chunk_id, token_list)
+        self._avgdl = 0.0
+        self._df = defaultdict(int)   # term → doc frequency
+        self._build_index()
+
+    def _build_index(self):
+        """Load all chunks from SQLite and compute BM25 statistics."""
+        conn = sqlite3.connect(self.db_path)
+        rows = conn.execute("SELECT id, chunk_text FROM chunks").fetchall()
+        conn.close()
+
+        total_len = 0
+        for chunk_id, text in rows:
+            tokens = text.lower().split()
+            self._corpus.append((chunk_id, tokens))
+            total_len += len(tokens)
+            for token in set(tokens):
+                self._df[token] += 1
+
+        self._avgdl = total_len / len(rows) if rows else 1.0
+        self._N = len(rows)
+
+    def _idf(self, term: str) -> float:
+        """Inverse document frequency for a term."""
+        df = self._df.get(term, 0)
+        return math.log((self._N - df + 0.5) / (df + 0.5) + 1)
+
+    def retrieve(self, query: str, top_k: int = 20) -> list[dict]:
+        """
+        Run BM25 retrieval over the corpus.
+
+        Args:
+            query (str) — raw or rewritten query (NOT expanded — BM25 is lexical)
+            top_k (int) — number of results to return
+
+        Returns:
+            list[dict] with chunk_id and sparse_score, sorted descending
+        """
+        query_terms = query.lower().split()
+        scores = {}
+
+        for chunk_id, tokens in self._corpus:
+            dl = len(tokens)
+            score = 0.0
+            tf_map = defaultdict(int)
+            for t in tokens:
+                tf_map[t] += 1
+
+            for term in query_terms:
+                if term not in tf_map:
+                    continue
+                tf = tf_map[term]
+                idf = self._idf(term)
+                numerator = tf * (self.k1 + 1)
+                denominator = tf + self.k1 * (1 - self.b + self.b * dl / self._avgdl)
+                score += idf * numerator / denominator
+
+            if score > 0:
+                scores[chunk_id] = score
+
+        sorted_results = sorted(scores.items(), key=lambda x: x[1], reverse=True)[:top_k]
+
+        # Fetch text for top results
+        conn = sqlite3.connect(self.db_path)
+        results = []
+        for chunk_id, score in sorted_results:
+            row = conn.execute(
+                "SELECT chunk_text, filepath, chunk_index FROM chunks WHERE id = ?",
+                (chunk_id,)
+            ).fetchone()
+            if row:
+                results.append({
+                    "chunk_id": chunk_id,
+                    "chunk_text": row[0],
+                    "filepath": row[1],
+                    "chunk_index": row[2],
+                    "sparse_score": score,
+                })
+        conn.close()
+        return results
+
+
+if __name__ == "__main__":
+    sr = SparseRetriever()
+    results = sr.retrieve("quarterly budget", top_k=5)
+    for r in results:
+        print(f"[{r['sparse_score']:.4f}] {r['filepath']} → {r['chunk_text'][:80]}")

static/style.css

@@ -0,0 +1,405 @@
+/* static/style.css */
+
+* { margin:0; padding:0; box-sizing:border-box; }
+
+body {
+    font-family: "Segoe UI", Arial, sans-serif;
+    background: #f4f6f9;
+    color: #1a1a2e;
+    min-height: 100vh;
+}
+
+/* NAV */
+nav {
+    background: #1a3a5c;
+    padding: 0.8rem 2rem;
+    display: flex;
+    align-items: center;
+    gap: 2rem;
+}
+nav .brand {
+    color: #fff;
+    font-size: 1.05rem;
+    font-weight: 700;
+    text-decoration: none;
+}
+nav a {
+    color: #a8c8e8;
+    text-decoration: none;
+    font-size: 0.9rem;
+}
+nav a:hover { color: #fff; }
+
+/* CONTAINER */
+.container {
+    max-width: 1100px;
+    margin: 0 auto;
+    padding: 2rem 1.5rem;
+}
+
+/* HERO */
+.hero {
+    text-align: center;
+    padding: 3rem 1rem 1.5rem;
+}
+.hero h1 {
+    font-size: 2rem;
+    color: #1a3a5c;
+    margin-bottom: 0.4rem;
+}
+.hero p {
+    color: #777;
+    font-size: 0.98rem;
+    margin-bottom: 1.8rem;
+}
+
+/* STATS PILLS */
+.stats {
+    display: flex;
+    justify-content: center;
+    gap: 1rem;
+    margin-bottom: 2rem;
+    flex-wrap: wrap;
+}
+.stat-pill {
+    background: #e8f0fa;
+    color: #1a3a5c;
+    padding: 0.35rem 1rem;
+    border-radius: 20px;
+    font-size: 0.85rem;
+    font-weight: 600;
+}
+
+/* SEARCH BOX */
+.search-card {
+    background: #fff;
+    border-radius: 10px;
+    box-shadow: 0 2px 16px rgba(0,0,0,0.08);
+    padding: 1.6rem;
+    max-width: 720px;
+    margin: 0 auto 1.5rem;
+}
+.search-row {
+    display: flex;
+    gap: 0.6rem;
+    margin-bottom: 0.85rem;
+}
+.search-input {
+    flex: 1;
+    padding: 0.72rem 1rem;
+    border: 1.5px solid #d0dbe8;
+    border-radius: 7px;
+    font-size: 1rem;
+    outline: none;
+    transition: border 0.2s;
+}
+.search-input:focus { border-color: #1a3a5c; }
+.btn-search {
+    padding: 0.72rem 1.6rem;
+    background: #1a3a5c;
+    color: #fff;
+    border: none;
+    border-radius: 7px;
+    font-size: 0.95rem;
+    font-weight: 700;
+    cursor: pointer;
+    transition: background 0.2s;
+}
+.btn-search:hover { background: #0f2a44; }
+.filters {
+    display: flex;
+    gap: 1.2rem;
+    align-items: center;
+    flex-wrap: wrap;
+}
+.filters label { font-size: 0.85rem; color: #666; }
+.filters select {
+    padding: 0.32rem 0.7rem;
+    border: 1.5px solid #d0dbe8;
+    border-radius: 5px;
+    font-size: 0.85rem;
+    background: #fff;
+    cursor: pointer;
+}
+
+/* SAMPLE QUERIES */
+.sample-queries {
+    text-align: center;
+    margin-top: 0.8rem;
+}
+.sample-queries p {
+    font-size: 0.82rem;
+    color: #999;
+    margin-bottom: 0.5rem;
+}
+.sample-btn {
+    background: #e8f0fa;
+    border: none;
+    border-radius: 14px;
+    padding: 0.28rem 0.85rem;
+    font-size: 0.8rem;
+    color: #1a3a5c;
+    cursor: pointer;
+    margin: 0.2rem;
+    transition: background 0.2s;
+}
+.sample-btn:hover { background: #ccdcf0; }
+
+/* ERROR */
+.error-box {
+    background: #fff0f0;
+    border: 1px solid #f0c0c0;
+    border-radius: 7px;
+    padding: 0.7rem 1rem;
+    color: #a00;
+    font-size: 0.9rem;
+    margin-bottom: 1rem;
+    max-width: 720px;
+    margin-left: auto;
+    margin-right: auto;
+}
+
+/* RESULTS PAGE LAYOUT */
+.results-layout {
+    display: grid;
+    grid-template-columns: 1fr 320px;
+    gap: 1.5rem;
+    align-items: start;
+    margin-top: 1.5rem;
+}
+
+/* RESULTS HEADER */
+.results-header {
+    margin-bottom: 1rem;
+    padding-bottom: 0.6rem;
+    border-bottom: 2px solid #e0e8f0;
+}
+.results-header h2 {
+    font-size: 1.15rem;
+    color: #1a3a5c;
+}
+.results-meta {
+    font-size: 0.83rem;
+    color: #999;
+    margin-top: 0.25rem;
+}
+.expanded-tag {
+    background: #f0f4f8;
+    border-radius: 6px;
+    padding: 0.45rem 0.85rem;
+    font-size: 0.83rem;
+    color: #555;
+    margin-bottom: 0.85rem;
+}
+
+/* RESULT CARD */
+.result-card {
+    background: #fff;
+    border-radius: 8px;
+    padding: 1rem 1.2rem;
+    margin-bottom: 0.8rem;
+    box-shadow: 0 1px 6px rgba(0,0,0,0.06);
+    border-left: 3px solid #1a3a5c;
+    transition: box-shadow 0.15s;
+}
+.result-card:hover { box-shadow: 0 3px 14px rgba(0,0,0,0.1); }
+.result-top {
+    display: flex;
+    justify-content: space-between;
+    align-items: flex-start;
+    margin-bottom: 0.4rem;
+}
+.result-title {
+    font-size: 0.95rem;
+    font-weight: 700;
+    color: #1a3a5c;
+    word-break: break-all;
+}
+.result-score {
+    font-size: 0.8rem;
+    background: #e8f0fa;
+    color: #1a3a5c;
+    padding: 0.18rem 0.55rem;
+    border-radius: 12px;
+    font-weight: 700;
+    white-space: nowrap;
+    margin-left: 0.8rem;
+}
+.result-snippet {
+    font-size: 0.88rem;
+    color: #444;
+    line-height: 1.55;
+}
+.result-footer {
+    margin-top: 0.5rem;
+}
+.badge {
+    font-size: 0.74rem;
+    padding: 0.15rem 0.55rem;
+    border-radius: 10px;
+    font-weight: 600;
+}
+.badge-scifact  { background: #e0f0e8; color: #1a5c2e; }
+.badge-nfcorpus { background: #fce8e8; color: #7a1a1a; }
+.badge-file     { background: #e8eaf0; color: #3a3a6c; }
+
+.no-results {
+    text-align: center;
+    padding: 3rem;
+    color: #999;
+}
+
+/* QUERY PANELS (right side) */
+.query-panel {
+    background: #fff;
+    border-radius: 8px;
+    box-shadow: 0 1px 6px rgba(0,0,0,0.06);
+    overflow: hidden;
+    margin-bottom: 1rem;
+}
+.panel-header {
+    display: flex;
+    justify-content: space-between;
+    align-items: center;
+    padding: 0.6rem 1rem;
+    font-size: 0.88rem;
+    font-weight: 700;
+}
+.panel-header.scifact  { background: #e0f0e8; color: #1a5c2e; }
+.panel-header.nfcorpus { background: #fce8e8; color: #7a1a1a; }
+.panel-count {
+    background: rgba(0,0,0,0.1);
+    border-radius: 10px;
+    padding: 0.1rem 0.5rem;
+    font-size: 0.75rem;
+}
+.query-item {
+    padding: 0.55rem 1rem;
+    border-bottom: 1px solid #f0f4f8;
+}
+.query-item:last-child { border-bottom: none; }
+.query-id   { font-size: 0.72rem; color: #bbb; margin-bottom: 0.15rem; }
+.query-text { font-size: 0.85rem; color: #333; line-height: 1.45; }
+.panel-empty {
+    padding: 0.75rem 1rem;
+    font-size: 0.83rem;
+    color: #bbb;
+    font-style: italic;
+}
+.panel-info {
+    background: #f7fafd;
+    border-radius: 7px;
+    padding: 0.7rem 0.9rem;
+    font-size: 0.78rem;
+    color: #888;
+    line-height: 1.55;
+}
+
+/* DASHBOARD */
+.dash-title {
+    font-size: 1.4rem;
+    color: #1a3a5c;
+    font-weight: 700;
+    margin-bottom: 0.3rem;
+}
+.dash-sub {
+    color: #999;
+    font-size: 0.88rem;
+    margin-bottom: 1.5rem;
+}
+.dash-grid {
+    display: grid;
+    grid-template-columns: 1fr 1fr;
+    gap: 1.2rem;
+    margin-bottom: 2rem;
+}
+.metric-card {
+    background: #fff;
+    border-radius: 8px;
+    padding: 1.2rem 1.4rem;
+    box-shadow: 0 1px 6px rgba(0,0,0,0.06);
+}
+.metric-card h3 {
+    font-size: 1rem;
+    color: #1a3a5c;
+    margin-bottom: 1rem;
+    padding-bottom: 0.5rem;
+    border-bottom: 1.5px solid #e0e8f0;
+}
+.metric-row {
+    display: flex;
+    align-items: center;
+    margin-bottom: 0.65rem;
+}
+.metric-label {
+    width: 95px;
+    font-size: 0.85rem;
+    color: #666;
+}
+.bar-wrap {
+    flex: 1;
+    background: #e8f0fa;
+    border-radius: 4px;
+    height: 8px;
+    overflow: hidden;
+    margin: 0 0.7rem;
+}
+.bar {
+    height: 100%;
+    border-radius: 4px;
+    background: #1a3a5c;
+}
+.bar.green  { background: #2d7a4f; }
+.bar.amber  { background: #c07020; }
+.metric-val {
+    width: 52px;
+    text-align: right;
+    font-size: 0.9rem;
+    font-weight: 700;
+    color: #1a3a5c;
+}
+.section-label {
+    font-size: 1.05rem;
+    font-weight: 700;
+    color: #1a3a5c;
+    margin: 1.5rem 0 0.7rem;
+}
+.table-card {
+    background: #fff;
+    border-radius: 8px;
+    box-shadow: 0 1px 6px rgba(0,0,0,0.06);
+    overflow: hidden;
+    margin-bottom: 1.5rem;
+}
+.table-card table {
+    width: 100%;
+    border-collapse: collapse;
+    font-size: 0.88rem;
+}
+.table-card th {
+    background: #1a3a5c;
+    color: #fff;
+    padding: 0.6rem 1rem;
+    text-align: left;
+    font-weight: 600;
+}
+.table-card td {
+    padding: 0.55rem 1rem;
+    border-bottom: 1px solid #eef2f7;
+    color: #333;
+}
+.table-card tr:last-child td { border-bottom: none; }
+.table-card tr:nth-child(even) td { background: #f7fafd; }
+.table-card tr.best td {
+    font-weight: 700;
+    color: #1a3a5c;
+    background: #eaf4ff;
+}
+
+/* RESPONSIVE */
+@media (max-width: 700px) {
+    .results-layout { grid-template-columns: 1fr; }
+    .dash-grid      { grid-template-columns: 1fr; }
+    .search-row     { flex-direction: column; }
+    .hero h1        { font-size: 1.5rem; }
+}

templates/dashboard.html

@@ -0,0 +1,128 @@
+<!DOCTYPE html>
+<html lang="en">
+<head>
+    <meta charset="UTF-8">
+    <meta name="viewport" content="width=device-width, initial-scale=1.0">
+    <title>Evaluation Dashboard</title>
+    <link rel="stylesheet" href="/static/style.css">
+</head>
+<body>
+
+<nav>
+    <a href="/" class="brand">🔍 Semantic Search</a>
+    <a href="/">Search</a>
+    <a href="/dashboard">Dashboard</a>
+</nav>
+
+<div class="container">
+
+    <div style="margin-top:1.5rem;">
+        <div class="dash-title">Evaluation Dashboard</div>
+        <div class="dash-sub">
+            BEIR Benchmark — Full pipeline (Dense + BM25 + RRF + Cross-Encoder)
+        </div>
+    </div>
+
+    {% if datasets %}
+
+    <!-- metric cards -->
+    <div class="dash-grid">
+        {% for d in datasets %}
+        <div class="metric-card">
+            <h3>
+                {% if d.name == "scifact" %}🔬{% else %}🏥{% endif %}
+                {{ d.name | title }}
+                <span style="font-size:0.76rem;color:#999;font-weight:400;">
+                    — {{ d.queries }} queries
+                </span>
+            </h3>
+
+            <div class="metric-row">
+                <span class="metric-label">NDCG@10</span>
+                <div class="bar-wrap">
+                    <div class="bar green" style="width:{{ (d.ndcg * 100) | round(1) }}%"></div>
+                </div>
+                <span class="metric-val">{{ "%.4f" | format(d.ndcg) }}</span>
+            </div>
+
+            <div class="metric-row">
+                <span class="metric-label">MRR</span>
+                <div class="bar-wrap">
+                    <div class="bar" style="width:{{ (d.mrr * 100) | round(1) }}%"></div>
+                </div>
+                <span class="metric-val">{{ "%.4f" | format(d.mrr) }}</span>
+            </div>
+
+            <div class="metric-row">
+                <span class="metric-label">MAP@100</span>
+                <div class="bar-wrap">
+                    <div class="bar amber" style="width:{{ (d.map * 100) | round(1) }}%"></div>
+                </div>
+                <span class="metric-val">{{ "%.4f" | format(d.map) }}</span>
+            </div>
+
+            <div class="metric-row">
+                <span class="metric-label">Recall@100</span>
+                <div class="bar-wrap">
+                    <div class="bar" style="width:{{ (d.recall * 100) | round(1) }}%"></div>
+                </div>
+                <span class="metric-val">{{ "%.4f" | format(d.recall) }}</span>
+            </div>
+
+            <div class="metric-row">
+                <span class="metric-label">P@10</span>
+                <div class="bar-wrap">
+                    <div class="bar amber" style="width:{{ (d.precision * 100) | round(1) }}%"></div>
+                </div>
+                <span class="metric-val">{{ "%.4f" | format(d.precision) }}</span>
+            </div>
+        </div>
+        {% endfor %}
+    </div>
+
+    <!-- ablation tables -->
+    {% for d in datasets %}
+    <div class="section-label">
+        Ablation Table — {{ d.name | title }}
+    </div>
+    <div class="table-card">
+        <table>
+            <thead>
+                <tr>
+                    <th>Mode</th>
+                    <th>NDCG@10</th>
+                    <th>MAP@100</th>
+                    <th>MRR</th>
+                    <th>Recall@100</th>
+                    <th>P@10</th>
+                </tr>
+            </thead>
+            <tbody>
+                {% for mode_name, m in d.modes.items() %}
+                <tr {% if mode_name == "full" %}class="best"{% endif %}>
+                    <td>{{ mode_name }}</td>
+                    <td>{{ "%.4f" | format(m.get("NDCG@10",    0)) }}</td>
+                    <td>{{ "%.4f" | format(m.get("MAP@100",    0)) }}</td>
+                    <td>{{ "%.4f" | format(m.get("MRR",        0)) }}</td>
+                    <td>{{ "%.4f" | format(m.get("Recall@100", 0)) }}</td>
+                    <td>{{ "%.4f" | format(m.get("P@10",       0)) }}</td>
+                </tr>
+                {% endfor %}
+            </tbody>
+        </table>
+    </div>
+    {% endfor %}
+
+    {% else %}
+    <div class="no-results">
+        <p>No evaluation results found.</p>
+        <p style="margin-top:0.5rem;font-size:0.85rem;">
+            Run:
+            <code>python -m evaluation.run_eval --datasets scifact nfcorpus --mode all</code>
+        </p>
+    </div>
+    {% endif %}
+
+</div>
+</body>
+</html>

templates/index.html

@@ -0,0 +1,75 @@
+<!DOCTYPE html>
+<html lang="en">
+<head>
+    <meta charset="UTF-8">
+    <meta name="viewport" content="width=device-width, initial-scale=1.0">
+    <title>Semantic Search Engine</title>
+    <link rel="stylesheet" href="/static/style.css">
+</head>
+<body>
+
+<nav>
+    <a href="/" class="brand">🔍 Semantic Search</a>
+    <a href="/">Search</a>
+    <a href="/dashboard">Dashboard</a>
+</nav>
+
+<div class="container">
+    <div class="hero">
+        <h1>Semantic File Search Engine</h1>
+        <p>AI-powered search across SciFact and NFCorpus using Dense + BM25 + RRF + Cross-Encoder</p>
+
+        <div class="stats">
+            <span class="stat-pill">🔬 SciFact — {{ scifact_count }} queries</span>
+            <span class="stat-pill">🏥 NFCorpus — {{ nfcorpus_count }} queries</span>
+        </div>
+    </div>
+
+    {% if error %}
+    <div class="error-box">{{ error }}</div>
+    {% endif %}
+
+    <div class="search-card">
+        <form action="/search" method="post">
+            <div class="search-row">
+                <input
+                    class="search-input"
+                    type="text"
+                    name="query"
+                    placeholder="e.g. vitamin D deficiency cancer risk"
+                    autofocus
+                    required
+                />
+                <button class="btn-search" type="submit">Search</button>
+            </div>
+            <div class="filters">
+                <label>Top results:</label>
+                <select name="top_k">
+                    <option value="5">5</option>
+                    <option value="10" selected>10</option>
+                    <option value="20">20</option>
+                </select>
+            </div>
+        </form>
+    </div>
+
+    <div class="sample-queries">
+        <p>Try a sample query:</p>
+        {% for q in [
+            "vitamin D cancer",
+            "BRCA1 breast cancer",
+            "smoking lung disease",
+            "mRNA vaccine immune",
+            "diabetes insulin resistance",
+            "protein folding structure"
+        ] %}
+        <button class="sample-btn"
+            onclick="document.querySelector('.search-input').value='{{ q }}'">
+            {{ q }}
+        </button>
+        {% endfor %}
+    </div>
+</div>
+
+</body>
+</html>

templates/results.html

@@ -0,0 +1,153 @@
+<!DOCTYPE html>
+<html lang="en">
+<head>
+    <meta charset="UTF-8">
+    <meta name="viewport" content="width=device-width, initial-scale=1.0">
+    <title>Results — {{ query }}</title>
+    <link rel="stylesheet" href="/static/style.css">
+</head>
+<body>
+
+<nav>
+    <a href="/" class="brand">🔍 Semantic Search</a>
+    <a href="/">Search</a>
+    <a href="/dashboard">Dashboard</a>
+</nav>
+
+<div class="container">
+
+    <!-- search bar stays on results page -->
+    <div class="search-card" style="margin-top:1.5rem; max-width:100%;">
+        <form action="/search" method="post">
+            <div class="search-row">
+                <input
+                    class="search-input"
+                    type="text"
+                    name="query"
+                    value="{{ query }}"
+                    required
+                />
+                <button class="btn-search" type="submit">Search</button>
+            </div>
+            <div class="filters">
+                <label>Top results:</label>
+                <select name="top_k">
+                    <option value="5"  {% if top_k==5  %}selected{% endif %}>5</option>
+                    <option value="10" {% if top_k==10 %}selected{% endif %}>10</option>
+                    <option value="20" {% if top_k==20 %}selected{% endif %}>20</option>
+                </select>
+            </div>
+        </form>
+    </div>
+
+    <div class="results-layout">
+
+        <!-- LEFT — ranked results -->
+        <div>
+            <div class="results-header">
+                <h2>Results for "{{ query }}"</h2>
+                <div class="results-meta">
+                    {{ total }} result{{ "s" if total != 1 else "" }}
+                    &nbsp;·&nbsp; {{ elapsed }}s
+                    &nbsp;·&nbsp; Full pipeline
+                </div>
+            </div>
+
+            {% if expanded_query and expanded_query != query %}
+            <div class="expanded-tag">
+                <strong>Expanded:</strong> {{ expanded_query }}
+            </div>
+            {% endif %}
+
+            {% if results %}
+                {% for r in results %}
+                <div class="result-card">
+                    <div class="result-top">
+                        <div class="result-title">
+                            {{ r.icon }}&nbsp; {{ r.doc_id }}
+                        </div>
+                        <div class="result-score">{{ r.score }}</div>
+                    </div>
+
+                    <div class="result-snippet">{{ r.snippet }}</div>
+
+                    <div class="result-footer">
+                        {% if r.dataset == "scifact" %}
+                            <span class="badge badge-scifact">🔬 SciFact</span>
+                        {% elif r.dataset == "nfcorpus" %}
+                            <span class="badge badge-nfcorpus">🏥 NFCorpus</span>
+                        {% else %}
+                            <span class="badge badge-file">📁 File</span>
+                        {% endif %}
+                    </div>
+                </div>
+                {% endfor %}
+
+            {% else %}
+                <div class="no-results">
+                    <p>No results found for "<strong>{{ query }}</strong>"</p>
+                    <p style="margin-top:0.5rem; font-size:0.85rem; color:#bbb;">
+                        Try shorter or different keywords.
+                    </p>
+                </div>
+            {% endif %}
+        </div>
+
+        <!-- RIGHT — matched dataset queries -->
+        <div>
+
+            <!-- SciFact panel -->
+            <div class="query-panel">
+                <div class="panel-header scifact">
+                    🔬 SciFact Queries
+                    <span class="panel-count">{{ scifact_matches | length }}</span>
+                </div>
+                {% if scifact_matches %}
+                    {% for q in scifact_matches %}
+                    <div class="query-item">
+                        <div class="query-id">Query #{{ q.query_id }}</div>
+                        <div class="query-text">{{ q.query_text }}</div>
+                    </div>
+                    {% endfor %}
+                {% else %}
+                    <div class="panel-empty">
+                        No SciFact queries matched "{{ query }}"
+                    </div>
+                {% endif %}
+            </div>
+
+            <!-- NFCorpus panel -->
+            <div class="query-panel">
+                <div class="panel-header nfcorpus">
+                    🏥 NFCorpus Queries
+                    <span class="panel-count">{{ nfcorpus_matches | length }}</span>
+                </div>
+                {% if nfcorpus_matches %}
+                    {% for q in nfcorpus_matches %}
+                    <div class="query-item">
+                        <div class="query-id">Query #{{ q.query_id }}</div>
+                        <div class="query-text">{{ q.query_text }}</div>
+                    </div>
+                    {% endfor %}
+                {% else %}
+                    <div class="panel-empty">
+                        No NFCorpus queries matched "{{ query }}"
+                    </div>
+                {% endif %}
+            </div>
+
+            <!-- info box -->
+            <div class="panel-info">
+                These are actual evaluation queries from the BEIR benchmark
+                that share keywords with your search.
+                <br><br>
+                <strong>300</strong> SciFact queries &nbsp;·&nbsp;
+                <strong>323</strong> NFCorpus queries loaded.
+            </div>
+
+        </div>
+    </div>
+</div>
+
+</body>
+</html>