initial

.gitignore

@@ -0,0 +1,5 @@
+__pycache__/
+*.pyc
+qdrant_data/
+results/
+.ipynb_checkpoints/

benchmark/config.py

@@ -0,0 +1,51 @@
+"""Models under test and benchmark-wide settings.
+
+`query_prefix` / `passage_prefix` are plain-text instruction prefixes prepended
+before encoding (the E5 family expects "query: " / "passage: "; other model
+families used here don't need one). This is applied manually rather than via
+SentenceTransformer's `prompt_name=` API so behavior doesn't depend on whether
+a given model repo happens to ship a config_sentence_transformers.json with
+named prompts.
+"""
+
+MODELS = [
+    {
+        "name": "kazalbrur/bangla-embed-e5-small-banglish",
+        "query_prefix": "query: ",
+        "passage_prefix": "passage: ",
+    },
+    {
+        "name": "intfloat/multilingual-e5-small",
+        "query_prefix": "query: ",
+        "passage_prefix": "passage: ",
+    },
+    {
+        "name": "intfloat/multilingual-e5-base",
+        "query_prefix": "query: ",
+        "passage_prefix": "passage: ",
+    },
+    {
+        "name": "BAAI/bge-m3",
+        "query_prefix": "",
+        "passage_prefix": "",
+    },
+    {
+        "name": "sentence-transformers/paraphrase-multilingual-mpnet-base-v2",
+        "query_prefix": "",
+        "passage_prefix": "",
+    },
+]
+
+QUERY_MODES = ["raw", "normalized"]
+
+RETRIEVAL_MODES = ["dense", "hybrid"]  # hybrid = BM25 + dense fused with RRF
+
+TOP_K = 50  # depth retrieved per query; recall@10/@50 and nDCG@10 are sliced from this
+FUSION_DEPTH = 50  # how deep each ranker (BM25, dense) is pulled before RRF fusion
+RRF_K = 60  # RRF's rank-damping constant
+
+BOOKS_PATH = "boi1_sample_books.json"
+QUERIES_PATH = "boi1_eval_queries.json"
+
+QDRANT_PATH = "./qdrant_data"
+RESULTS_DIR = "./results"

benchmark/data.py

@@ -0,0 +1,46 @@
+"""Loading book corpus and evaluation queries."""
+
+import json
+from dataclasses import dataclass, field
+
+
+@dataclass
+class EvalQuery:
+    query: str
+    normalized_query: str
+    relevance: dict  # book_id -> graded relevance (1-3)
+    categories: list = field(default_factory=list)
+
+    def text_for_mode(self, mode):
+        if mode == "normalized":
+            return self.normalized_query
+        return self.query
+
+
+def load_books(path):
+    with open(path, "r", encoding="utf-8") as f:
+        return json.load(f)
+
+
+def load_queries(path):
+    with open(path, "r", encoding="utf-8") as f:
+        raw = json.load(f)
+    return [
+        EvalQuery(
+            query=item["query"],
+            normalized_query=item.get("normalized_query", item["query"]),
+            relevance=item["relevance"],
+            categories=item.get("categories", []),
+        )
+        for item in raw
+    ]
+
+
+def book_to_text(book):
+    return (
+        f"Title: {book['title']}\n"
+        f"Author: {book['author']}\n"
+        f"Genre: {', '.join(book['genres'])}\n"
+        f"Tags: {', '.join(book['tags'])}\n"
+        f"Description: {book['description']}\n"
+    )

benchmark/embedder.py

@@ -0,0 +1,70 @@
+"""Wraps a SentenceTransformer model with prefixing and timing instrumentation."""
+
+import time
+
+import torch
+from sentence_transformers import SentenceTransformer
+
+
+class TimedEmbedder:
+    def __init__(self, model_name, query_prefix="", passage_prefix="", device=None):
+        self.model_name = model_name
+        self.query_prefix = query_prefix
+        self.passage_prefix = passage_prefix
+        self.device = device or ("cuda" if torch.cuda.is_available() else "cpu")
+
+        start = time.perf_counter()
+        self.model = SentenceTransformer(model_name, device=self.device)
+        self.load_time_sec = time.perf_counter() - start
+
+        self.param_count = sum(p.numel() for p in self.model.parameters())
+        self.model_size_mb = sum(
+            p.numel() * p.element_size() for p in self.model.parameters()
+        ) / (1024 ** 2)
+
+        if hasattr(self.model, "get_embedding_dimension"):
+            self.embedding_dim = self.model.get_embedding_dimension()
+        else:
+            self.embedding_dim = self.model.get_sentence_embedding_dimension()
+
+    def encode_passages(self, texts, batch_size=32):
+        """Batch-encode documents; returns (embeddings, elapsed_seconds)."""
+        prefixed = [self.passage_prefix + t for t in texts]
+        start = time.perf_counter()
+        embeddings = self.model.encode(
+            prefixed,
+            batch_size=batch_size,
+            normalize_embeddings=True,
+            convert_to_numpy=True,
+            show_progress_bar=False,
+        )
+        elapsed = time.perf_counter() - start
+        return embeddings, elapsed
+
+    def encode_query(self, text):
+        """Single-query encode; returns (embedding, elapsed_seconds)."""
+        prefixed = self.query_prefix + text
+        start = time.perf_counter()
+        embedding = self.model.encode(
+            prefixed,
+            normalize_embeddings=True,
+            convert_to_numpy=True,
+            show_progress_bar=False,
+        )
+        elapsed = time.perf_counter() - start
+        return embedding, elapsed
+
+    def peak_encode_memory_mb(self, sample_text):
+        """Peak accelerator memory used while encoding one query (CUDA only, else None)."""
+        if self.device != "cuda":
+            return None
+        torch.cuda.synchronize()
+        torch.cuda.reset_peak_memory_stats(self.device)
+        self.encode_query(sample_text)
+        torch.cuda.synchronize()
+        return torch.cuda.max_memory_allocated(self.device) / (1024 ** 2)
+
+    def unload(self):
+        del self.model
+        if self.device == "cuda":
+            torch.cuda.empty_cache()

benchmark/fusion.py

@@ -0,0 +1,10 @@
+"""Reciprocal Rank Fusion for combining multiple ranked-id lists."""
+
+
+def reciprocal_rank_fusion(rankings, k=60):
+    """rankings: list of ranked-id lists (best first). Returns a single fused ranking."""
+    scores = {}
+    for ranking in rankings:
+        for rank, doc_id in enumerate(ranking, start=1):
+            scores[doc_id] = scores.get(doc_id, 0.0) + 1.0 / (k + rank)
+    return sorted(scores, key=scores.get, reverse=True)

benchmark/lexical.py

@@ -0,0 +1,16 @@
+"""BM25 lexical index, shared across all embedding models (it doesn't depend on any of them)."""
+
+import numpy as np
+from rank_bm25 import BM25Okapi
+
+
+class BM25Index:
+    def __init__(self, documents, book_ids):
+        self.book_ids = book_ids
+        tokenized = [doc.lower().split() for doc in documents]
+        self.bm25 = BM25Okapi(tokenized)
+
+    def search(self, query_text, top_k):
+        scores = self.bm25.get_scores(query_text.lower().split())
+        ranked_indices = np.argsort(scores)[::-1][:top_k]
+        return [self.book_ids[i] for i in ranked_indices]

benchmark/metrics.py

@@ -0,0 +1,54 @@
+"""Retrieval quality metrics for graded relevance judgments."""
+
+import math
+
+
+def recall_at_k(retrieved_ids, relevance, k):
+    """Fraction of all known-relevant items that appear in the top-k results."""
+    relevant = {doc_id for doc_id, rel in relevance.items() if rel > 0}
+    if not relevant:
+        return None
+    top_k = set(retrieved_ids[:k])
+    return len(top_k & relevant) / len(relevant)
+
+
+def reciprocal_rank(retrieved_ids, relevance):
+    """1 / rank of the first retrieved item with relevance > 0, else 0."""
+    relevant = {doc_id for doc_id, rel in relevance.items() if rel > 0}
+    for rank, doc_id in enumerate(retrieved_ids, start=1):
+        if doc_id in relevant:
+            return 1.0 / rank
+    return 0.0
+
+
+def _dcg(gains):
+    return sum(gain / math.log2(pos + 1) for pos, gain in enumerate(gains, start=1))
+
+
+def ndcg_at_k(retrieved_ids, relevance, k):
+    """Normalized DCG@k using the standard 2^rel - 1 gain function."""
+    gains = [
+        (2 ** relevance.get(doc_id, 0)) - 1
+        for doc_id in retrieved_ids[:k]
+    ]
+    dcg = _dcg(gains)
+
+    ideal_gains = sorted(
+        ((2 ** rel) - 1 for rel in relevance.values()),
+        reverse=True,
+    )[:k]
+    idcg = _dcg(ideal_gains)
+
+    if idcg == 0:
+        return 0.0
+    return dcg / idcg
+
+
+def evaluate_ranking(retrieved_ids, relevance):
+    """Compute all retrieval-quality metrics for a single query's ranking."""
+    return {
+        "recall@10": recall_at_k(retrieved_ids, relevance, 10),
+        "recall@50": recall_at_k(retrieved_ids, relevance, 50),
+        "mrr": reciprocal_rank(retrieved_ids, relevance),
+        "ndcg@10": ndcg_at_k(retrieved_ids, relevance, 10),
+    }

benchmark/report.py

@@ -0,0 +1,121 @@
+"""Flattens raw benchmark results into JSON + CSV artifacts."""
+
+import csv
+import json
+import os
+
+MODEL_FIELDS = [
+    "model",
+    "embedding_dim",
+    "model_size_mb",
+    "param_count",
+    "load_time_sec",
+    "doc_embed_time_sec",
+    "doc_throughput_docs_per_sec",
+    "raw_vector_storage_mb",
+    "qdrant_collection_disk_mb",
+    "peak_query_encode_memory_mb",
+]
+
+MODE_FIELDS = [
+    "mean_embedding_latency_ms",
+    "mean_retrieval_latency_ms",
+    "queries_per_sec",
+    "recall@10",
+    "recall@50",
+    "mrr",
+    "ndcg@10",
+]
+
+SUMMARY_FIELDS = MODEL_FIELDS[:1] + ["retrieval_mode", "query_mode"] + MODEL_FIELDS[1:] + MODE_FIELDS
+
+CATEGORY_FIELDS = [
+    "model",
+    "retrieval_mode",
+    "query_mode",
+    "category",
+    "n",
+    "recall@10",
+    "recall@50",
+    "mrr",
+    "ndcg@10",
+]
+
+
+def _summary_rows(results):
+    rows = []
+    for model_result in results:
+        for retrieval_mode, query_mode_results in model_result["retrieval_modes"].items():
+            for query_mode, mode_result in query_mode_results.items():
+                row = {k: model_result.get(k) for k in MODEL_FIELDS}
+                row["retrieval_mode"] = retrieval_mode
+                row["query_mode"] = query_mode
+                for metric in MODE_FIELDS:
+                    row[metric] = mode_result.get(metric)
+                rows.append(row)
+    return rows
+
+
+def _category_rows(results):
+    rows = []
+    for model_result in results:
+        for retrieval_mode, query_mode_results in model_result["retrieval_modes"].items():
+            for query_mode, mode_result in query_mode_results.items():
+                for category, stats in mode_result["per_category"].items():
+                    rows.append(
+                        {
+                            "model": model_result["model"],
+                            "retrieval_mode": retrieval_mode,
+                            "query_mode": query_mode,
+                            "category": category,
+                            **stats,
+                        }
+                    )
+    return rows
+
+
+def _write_csv(path, fieldnames, rows):
+    with open(path, "w", newline="", encoding="utf-8") as f:
+        writer = csv.DictWriter(f, fieldnames=fieldnames)
+        writer.writeheader()
+        for row in rows:
+            writer.writerow(row)
+
+
+def save_results(results, results_dir):
+    os.makedirs(results_dir, exist_ok=True)
+
+    raw_path = os.path.join(results_dir, "raw_results.json")
+    with open(raw_path, "w", encoding="utf-8") as f:
+        json.dump(results, f, ensure_ascii=False, indent=2)
+
+    summary_rows = _summary_rows(results)
+    summary_path = os.path.join(results_dir, "summary.csv")
+    _write_csv(summary_path, SUMMARY_FIELDS, summary_rows)
+
+    category_rows = _category_rows(results)
+    category_path = os.path.join(results_dir, "summary_by_category.csv")
+    _write_csv(category_path, CATEGORY_FIELDS, category_rows)
+
+    return {
+        "raw_results": raw_path,
+        "summary": summary_path,
+        "summary_by_category": category_path,
+    }
+
+
+def print_summary_table(results):
+    header = (
+        f"{'model':<55} {'retrieval':<8} {'mode':<11} {'dim':>5} {'size(MB)':>9} "
+        f"{'R@10':>6} {'R@50':>6} {'MRR':>6} {'nDCG@10':>8} {'lat(ms)':>8} {'q/s':>7}"
+    )
+    print(header)
+    print("-" * len(header))
+    for row in _summary_rows(results):
+        print(
+            f"{row['model']:<55} {row['retrieval_mode']:<8} {row['query_mode']:<11} "
+            f"{row['embedding_dim']:>5} {row['model_size_mb']:>9.1f} "
+            f"{row['recall@10']:>6.3f} {row['recall@50']:>6.3f} {row['mrr']:>6.3f} "
+            f"{row['ndcg@10']:>8.3f} {row['mean_retrieval_latency_ms']:>8.1f} "
+            f"{row['queries_per_sec']:>7.1f}"
+        )

benchmark/runner.py

@@ -0,0 +1,195 @@
+"""Orchestrates the per-model, per-retrieval-mode, per-query-mode benchmark run."""
+
+import statistics
+import time
+
+from . import config
+from .data import book_to_text, load_books, load_queries
+from .embedder import TimedEmbedder
+from .fusion import reciprocal_rank_fusion
+from .lexical import BM25Index
+from .metrics import evaluate_ranking
+from .vectorstore import BookVectorStore
+
+
+def _mean(values):
+    values = [v for v in values if v is not None]
+    return statistics.mean(values) if values else None
+
+
+def _retrieve(retrieval_mode, embedder, store, bm25_index, model_name, query_embedding, query_text, top_k, fusion_depth, rrf_k):
+    """Returns (retrieved_book_ids, extra_retrieval_seconds) where extra excludes embedding time."""
+    start = time.perf_counter()
+    if retrieval_mode == "dense":
+        retrieved = store.search(model_name, query_embedding, top_k)
+    elif retrieval_mode == "hybrid":
+        bm25_ranked = bm25_index.search(query_text, fusion_depth)
+        dense_ranked = store.search(model_name, query_embedding, fusion_depth)
+        fused = reciprocal_rank_fusion([bm25_ranked, dense_ranked], k=rrf_k)
+        retrieved = fused[:top_k]
+    else:
+        raise ValueError(f"Unknown retrieval_mode: {retrieval_mode}")
+    elapsed = time.perf_counter() - start
+    return retrieved, elapsed
+
+
+def _evaluate(embedder, store, bm25_index, model_name, queries, retrieval_mode, query_mode, top_k, fusion_depth, rrf_k):
+    per_query = []
+    embed_latencies_sec = []
+    retrieval_latencies_sec = []
+
+    for q in queries:
+        text = q.text_for_mode(query_mode)
+        embedding, embed_latency = embedder.encode_query(text)
+        retrieved, extra_latency = _retrieve(
+            retrieval_mode, embedder, store, bm25_index, model_name,
+            embedding, text, top_k, fusion_depth, rrf_k,
+        )
+        total_latency = embed_latency + extra_latency
+
+        embed_latencies_sec.append(embed_latency)
+        retrieval_latencies_sec.append(total_latency)
+
+        metrics = evaluate_ranking(retrieved, q.relevance)
+
+        per_query.append(
+            {
+                "query": text,
+                "categories": q.categories,
+                "embedding_latency_ms": embed_latency * 1000,
+                "retrieval_latency_ms": total_latency * 1000,
+                **metrics,
+            }
+        )
+
+    per_category = {}
+    for entry in per_query:
+        for cat in entry["categories"]:
+            bucket = per_category.setdefault(
+                cat, {"n": 0, "recall@10": [], "recall@50": [], "mrr": [], "ndcg@10": []}
+            )
+            bucket["n"] += 1
+            for metric in ("recall@10", "recall@50", "mrr", "ndcg@10"):
+                bucket[metric].append(entry[metric])
+
+    per_category_summary = {
+        cat: {
+            "n": bucket["n"],
+            "recall@10": _mean(bucket["recall@10"]),
+            "recall@50": _mean(bucket["recall@50"]),
+            "mrr": _mean(bucket["mrr"]),
+            "ndcg@10": _mean(bucket["ndcg@10"]),
+        }
+        for cat, bucket in per_category.items()
+    }
+
+    mean_embed_latency_sec = _mean(embed_latencies_sec)
+    mean_retrieval_latency_sec = _mean(retrieval_latencies_sec)
+
+    return {
+        "mean_embedding_latency_ms": mean_embed_latency_sec * 1000 if mean_embed_latency_sec else None,
+        "mean_retrieval_latency_ms": mean_retrieval_latency_sec * 1000 if mean_retrieval_latency_sec else None,
+        "queries_per_sec": (1.0 / mean_retrieval_latency_sec) if mean_retrieval_latency_sec else None,
+        "recall@10": _mean(e["recall@10"] for e in per_query),
+        "recall@50": _mean(e["recall@50"] for e in per_query),
+        "mrr": _mean(e["mrr"] for e in per_query),
+        "ndcg@10": _mean(e["ndcg@10"] for e in per_query),
+        "per_query": per_query,
+        "per_category": per_category_summary,
+    }
+
+
+def run_benchmark(
+    models=None,
+    query_modes=None,
+    retrieval_modes=None,
+    top_k=None,
+    fusion_depth=None,
+    rrf_k=None,
+    books_path=None,
+    queries_path=None,
+    qdrant_path=None,
+):
+    models = models if models is not None else config.MODELS
+    query_modes = query_modes if query_modes is not None else config.QUERY_MODES
+    retrieval_modes = retrieval_modes if retrieval_modes is not None else config.RETRIEVAL_MODES
+    top_k = top_k or config.TOP_K
+    fusion_depth = fusion_depth or config.FUSION_DEPTH
+    rrf_k = rrf_k if rrf_k is not None else config.RRF_K
+
+    books = load_books(books_path or config.BOOKS_PATH)
+    queries = load_queries(queries_path or config.QUERIES_PATH)
+    store = BookVectorStore(qdrant_path or config.QDRANT_PATH)
+
+    documents = [book_to_text(b) for b in books]
+    book_ids = [b["book_id"] for b in books]
+    num_docs = len(books)
+
+    bm25_index = BM25Index(documents, book_ids) if "hybrid" in retrieval_modes else None
+
+    results = []
+
+    for model_cfg in models:
+        model_name = model_cfg["name"]
+        print(f"\n=== {model_name} ===")
+
+        embedder = TimedEmbedder(
+            model_name,
+            query_prefix=model_cfg.get("query_prefix", ""),
+            passage_prefix=model_cfg.get("passage_prefix", ""),
+        )
+        print(
+            f"  loaded in {embedder.load_time_sec:.2f}s | "
+            f"dim={embedder.embedding_dim} | size={embedder.model_size_mb:.1f} MB "
+            f"| device={embedder.device}"
+        )
+
+        doc_embeddings, doc_embed_time = embedder.encode_passages(documents)
+        doc_throughput = num_docs / doc_embed_time if doc_embed_time else None
+        print(
+            f"  embedded {num_docs} docs in {doc_embed_time:.2f}s "
+            f"({doc_throughput:.1f} docs/sec)"
+        )
+
+        store.index(model_name, doc_embeddings, books)
+
+        raw_vector_bytes = embedder.embedding_dim * 4 * num_docs
+        qdrant_disk_mb = store.collection_disk_size_mb(model_name)
+        peak_mem_mb = embedder.peak_encode_memory_mb(queries[0].query)
+
+        model_result = {
+            "model": model_name,
+            "device": embedder.device,
+            "embedding_dim": embedder.embedding_dim,
+            "param_count": embedder.param_count,
+            "model_size_mb": embedder.model_size_mb,
+            "load_time_sec": embedder.load_time_sec,
+            "doc_embed_time_sec": doc_embed_time,
+            "doc_throughput_docs_per_sec": doc_throughput,
+            "raw_vector_storage_mb": raw_vector_bytes / (1024 ** 2),
+            "qdrant_collection_disk_mb": qdrant_disk_mb,
+            "peak_query_encode_memory_mb": peak_mem_mb,
+            "retrieval_modes": {},
+        }
+
+        for retrieval_mode in retrieval_modes:
+            model_result["retrieval_modes"][retrieval_mode] = {}
+            for query_mode in query_modes:
+                print(f"  evaluating retrieval_mode={retrieval_mode} query_mode={query_mode} ...")
+                mode_result = _evaluate(
+                    embedder, store, bm25_index, model_name, queries,
+                    retrieval_mode, query_mode, top_k, fusion_depth, rrf_k,
+                )
+                model_result["retrieval_modes"][retrieval_mode][query_mode] = mode_result
+                print(
+                    f"    recall@10={mode_result['recall@10']:.3f} "
+                    f"recall@50={mode_result['recall@50']:.3f} "
+                    f"mrr={mode_result['mrr']:.3f} "
+                    f"ndcg@10={mode_result['ndcg@10']:.3f} "
+                    f"retrieval_latency={mode_result['mean_retrieval_latency_ms']:.1f}ms"
+                )
+
+        results.append(model_result)
+        embedder.unload()
+
+    return results

benchmark/vectorstore.py

@@ -0,0 +1,69 @@
+"""Thin Qdrant wrapper: one collection per embedding model under test."""
+
+import os
+import re
+from uuid import uuid4
+
+from qdrant_client import QdrantClient
+from qdrant_client.models import Distance, PointStruct, VectorParams
+
+
+def collection_name_for(model_name):
+    return "books__" + re.sub(r"[^a-zA-Z0-9]+", "_", model_name).strip("_")
+
+
+def _dir_size_bytes(path):
+    total = 0
+    for root, _dirs, files in os.walk(path):
+        for name in files:
+            fp = os.path.join(root, name)
+            if os.path.isfile(fp):
+                total += os.path.getsize(fp)
+    return total
+
+
+class BookVectorStore:
+    def __init__(self, qdrant_path):
+        self.qdrant_path = qdrant_path
+        self.client = QdrantClient(path=qdrant_path)
+
+    def index(self, model_name, embeddings, books):
+        name = collection_name_for(model_name)
+        if self.client.collection_exists(name):
+            self.client.delete_collection(name)
+
+        self.client.create_collection(
+            collection_name=name,
+            vectors_config=VectorParams(
+                size=embeddings.shape[1],
+                distance=Distance.COSINE,
+            ),
+        )
+
+        points = [
+            PointStruct(
+                id=str(uuid4()),
+                vector=embedding.tolist(),
+                payload={"book_id": book["book_id"]},
+            )
+            for book, embedding in zip(books, embeddings)
+        ]
+        self.client.upsert(collection_name=name, points=points)
+        return name
+
+    def search(self, model_name, query_embedding, top_k):
+        name = collection_name_for(model_name)
+        results = self.client.query_points(
+            collection_name=name,
+            query=query_embedding.tolist(),
+            limit=top_k,
+            with_payload=True,
+        ).points
+        return [point.payload["book_id"] for point in results]
+
+    def collection_disk_size_mb(self, model_name):
+        name = collection_name_for(model_name)
+        collection_dir = os.path.join(self.qdrant_path, "collection", name)
+        if not os.path.isdir(collection_dir):
+            return None
+        return _dir_size_bytes(collection_dir) / (1024 ** 2)

boi1_eval_queries.json

@@ -10,7 +10,8 @@
       "pure_bangla",
       "factual",
       "literature_history"
-    ]
+    ],
+    "normalized_query": "বাংলা সাহিত্যের প্রথম ঐতিহাসিক উপন্যাস কোনটি?"
   },
   {
     "query": "রবীন্দ্রনাথ ঠাকুরের বিখ্যাত কাব্যগ্রন্থ কী কী?",
@@ -22,7 +23,8 @@
       "pure_bangla",
       "author_specific",
       "poetry"
-    ]
+    ],
+    "normalized_query": "রবীন্দ্রনাথ ঠাকুরের বিখ্যাত কাব্যগ্রন্থ কী কী?"
   },
   {
     "query": "মুক্তিযুদ্ধ নিয়ে লেখা ভালো উপন্যাস",
@@ -37,7 +39,8 @@
       "pure_bangla",
       "intent_based",
       "theme"
-    ]
+    ],
+    "normalized_query": "মুক্তিযুদ্ধ নিয়ে লেখা ভালো উপন্যাস"
   },
   {
     "query": "শরৎচন্দ্র চট্টোপাধ্যায়ের প্রেমের উপন্যাস",
@@ -50,7 +53,8 @@
       "pure_bangla",
       "author_specific",
       "genre"
-    ]
+    ],
+    "normalized_query": "শরৎচন্দ্র চট্টোপাধ্যায়ের প্রেমের উপন্যাস"
   },
   {
     "query": "ভালো গোয়েন্দা কাহিনী সুপারিশ করুন",
@@ -64,7 +68,8 @@
       "pure_bangla",
       "recommendation",
       "genre"
-    ]
+    ],
+    "normalized_query": "ভালো গোয়েন্দা কাহিনী সুপারিশ করুন"
   },
   {
     "query": "misir ali series er shera boi gula ki ki",
@@ -77,7 +82,8 @@
       "banglish",
       "series",
       "recommendation"
-    ]
+    ],
+    "normalized_query": "মিসির আলি সিরিজের সেরা বই গুলো কি কি"
   },
   {
     "query": "rabindranath er kabbo grontho gula",
@@ -89,7 +95,8 @@
       "banglish",
       "author_variation",
       "poetry"
-    ]
+    ],
+    "normalized_query": "রবীন্দ্রনাথের কাব্যগ্রন্থ গুলো"
   },
   {
     "query": "himu series porte chai kon boi diye shuru korbo",
@@ -101,7 +108,8 @@
       "banglish",
       "series",
       "recommendation"
-    ]
+    ],
+    "normalized_query": "হিমু সিরিজ পড়তে চাই কোন বই দিয়ে শুরু করবো"
   },
   {
     "query": "sunil gangopadhyay er itihas bhittik uponnash",
@@ -114,7 +122,8 @@
       "banglish",
       "author_variation",
       "genre"
-    ]
+    ],
+    "normalized_query": "সুনীল গঙ্গোপাধ্যায়ের ইতিহাস ভিত্তিক উপন্যাস"
   },
   {
     "query": "muktijudder upore lekha kishor uponnash",
@@ -127,7 +136,8 @@
       "banglish",
       "theme",
       "recommendation"
-    ]
+    ],
+    "normalized_query": "মুক্তিযুদ্ধের উপরে লেখা কিশোর উপন্যাস"
   },
   {
     "query": "What is considered the first novel in Bengali literature?",
@@ -138,7 +148,8 @@
     "categories": [
       "english_to_bangla",
       "factual"
-    ]
+    ],
+    "normalized_query": "বাংলা সাহিত্যের প্রথম উপন্যাস কোনটি বলে বিবেচিত হয়?"
   },
   {
     "query": "Best science fiction books by Muhammed Zafar Iqbal",
@@ -152,7 +163,8 @@
       "english_to_bangla",
       "author_specific",
       "genre"
-    ]
+    ],
+    "normalized_query": "মুহম্মদ জাফর ইকবালের সেরা বিজ্ঞান কল্পকাহিনী বই"
   },
   {
     "query": "Autobiography written by Taslima Nasrin",
@@ -164,7 +176,8 @@
       "english_to_bangla",
       "author_specific",
       "genre"
-    ]
+    ],
+    "normalized_query": "তসলিমা নাসরিনের লেখা আত্মজীবনী"
   },
   {
     "query": "Detective novels featuring Feluda",
@@ -175,7 +188,8 @@
       "english_to_bangla",
       "character_specific",
       "genre"
-    ]
+    ],
+    "normalized_query": "ফেলুদা চরিত্রের গোয়েন্দা উপন্যাস"
   },
   {
     "query": "Rabindranath er nationalism niye lekha boi",
@@ -187,7 +201,8 @@
       "mixed_script",
       "theme",
       "author_variation"
-    ]
+    ],
+    "normalized_query": "রবীন্দ্রনাথের জাতীয়তাবাদ নিয়ে লেখা বই"
   },
   {
     "query": "Zafar Iqbal er muktijuddho niye kishor uponnash",
@@ -199,7 +214,8 @@
       "mixed_script",
       "author_variation",
       "theme"
-    ]
+    ],
+    "normalized_query": "জাফর ইকবালের মুক্তিযুদ্ধ নিয়ে কিশোর উপন্যাস"
   },
   {
     "query": "Bibhutibhushan er প্রকৃতি বিষয়ক লেখা",
@@ -211,7 +227,8 @@
       "mixed_script",
       "author_variation",
       "theme"
-    ]
+    ],
+    "normalized_query": "বিভূতিভূষণের প্রকৃতি বিষয়ক লেখা"
   },
   {
     "query": "Sarat Chandra র সামাজিক উপন্যাস",
@@ -224,7 +241,8 @@
       "mixed_script",
       "author_variation",
       "genre"
-    ]
+    ],
+    "normalized_query": "শরৎচন্দ্রের সামাজিক উপন্যাস"
   },
   {
     "query": "Humayoon Ahmed er lekha jonopriyo boi somuho",
@@ -239,7 +257,8 @@
       "author_variation",
       "banglish",
       "bibliography"
-    ]
+    ],
+    "normalized_query": "হুমায়ূন আহমেদের লেখা জনপ্রিয় বই সমূহ"
   },
   {
     "query": "Zafor Iqbal science fiction boi",
@@ -252,7 +271,8 @@
       "author_variation",
       "banglish",
       "genre"
-    ]
+    ],
+    "normalized_query": "জাফর ইকবালের বিজ্ঞান কল্পকাহিনী বই"
   },
   {
     "query": "Bongkim Chandra historical novels",
@@ -265,7 +285,8 @@
       "author_variation",
       "english_to_bangla",
       "genre"
-    ]
+    ],
+    "normalized_query": "বঙ্কিমচন্দ্রের ঐতিহাসিক উপন্যাস"
   },
   {
     "query": "Shordindu Bandopadhyay Byomkesh series",
@@ -276,7 +297,8 @@
       "author_variation",
       "banglish",
       "series"
-    ]
+    ],
+    "normalized_query": "শরদিন্দু বন্দ্যোপাধ্যায়ের ব্যোমকেশ সিরিজ"
   },
   {
     "query": "nondito noroke",
@@ -286,7 +308,8 @@
     "categories": [
       "typo",
       "title_variation"
-    ]
+    ],
+    "normalized_query": "নন্দিত নরকে"
   },
   {
     "query": "nondito norokey",
@@ -296,7 +319,8 @@
     "categories": [
       "typo",
       "title_variation"
-    ]
+    ],
+    "normalized_query": "নন্দিত নরকে"
   },
   {
     "query": "debdash bangla uponnash",
@@ -306,7 +330,8 @@
     "categories": [
       "typo",
       "title_variation"
-    ]
+    ],
+    "normalized_query": "দেবদাস বাংলা উপন্যাস"
   },
   {
     "query": "pother panchaly bivutibhushon",
@@ -317,7 +342,8 @@
       "typo",
       "title_variation",
       "author_variation"
-    ]
+    ],
+    "normalized_query": "পথের পাঁচালী বিভূতিভূষণ"
   },
   {
     "query": "thriller boi suggestion",
@@ -332,7 +358,8 @@
       "intent_based",
       "recommendation",
       "genre"
-    ]
+    ],
+    "normalized_query": "থ্রিলার বই সাজেশন"
   },
   {
     "query": "science fiction bangla boi",
@@ -347,7 +374,8 @@
       "intent_based",
       "recommendation",
       "genre"
-    ]
+    ],
+    "normalized_query": "বিজ্ঞান কল্পকাহিনী বাংলা বই"
   },
   {
     "query": "romantic bangla novel recommendation",
@@ -361,7 +389,8 @@
       "intent_based",
       "recommendation",
       "genre"
-    ]
+    ],
+    "normalized_query": "রোমান্টিক বাংলা উপন্যাস সুপারিশ"
   },
   {
     "query": "bengali poetry books for beginners",
@@ -375,6 +404,7 @@
       "intent_based",
       "recommendation",
       "genre"
-    ]
+    ],
+    "normalized_query": "শুরুর জন্য বাংলা কবিতার বই"
   }
-]
+]

run_benchmark.py

@@ -0,0 +1,78 @@
+#!/usr/bin/env python3
+"""CLI entry point for the embedding-model retrieval benchmark.
+
+Usage:
+    python run_benchmark.py                       # run the full model set from benchmark/config.py
+    python run_benchmark.py --models kazalbrur/bangla-embed-e5-small-banglish
+    python run_benchmark.py --query-modes raw      # skip the normalized-query experiment
+"""
+
+import argparse
+
+from benchmark import config, report
+from benchmark.runner import run_benchmark
+
+
+def parse_args():
+    parser = argparse.ArgumentParser(description=__doc__)
+    parser.add_argument(
+        "--models",
+        nargs="+",
+        default=None,
+        help="HF model names to benchmark (default: full set in benchmark/config.py)",
+    )
+    parser.add_argument(
+        "--query-modes",
+        nargs="+",
+        choices=["raw", "normalized"],
+        default=None,
+        help="Which query variants to evaluate (default: both)",
+    )
+    parser.add_argument(
+        "--retrieval-modes",
+        nargs="+",
+        choices=["dense", "hybrid"],
+        default=None,
+        help="dense (embedding-only) and/or hybrid (BM25 + dense RRF) (default: both)",
+    )
+    parser.add_argument("--top-k", type=int, default=None)
+    parser.add_argument("--results-dir", default=config.RESULTS_DIR)
+    return parser.parse_args()
+
+
+def main():
+    args = parse_args()
+
+    models = None
+    if args.models:
+        by_name = {m["name"]: m for m in config.MODELS}
+        models = []
+        for name in args.models:
+            if name not in by_name:
+                raise SystemExit(
+                    f"Unknown model '{name}'. Add it to benchmark/config.py MODELS first "
+                    f"(with its query/passage prefix)."
+                )
+            models.append(by_name[name])
+
+    results = run_benchmark(
+        models=models,
+        query_modes=args.query_modes,
+        retrieval_modes=args.retrieval_modes,
+        top_k=args.top_k,
+    )
+
+    paths = report.save_results(results, args.results_dir)
+
+    print("\n" + "=" * 100)
+    print("SUMMARY")
+    print("=" * 100)
+    report.print_summary_table(results)
+
+    print("\nSaved:")
+    for label, path in paths.items():
+        print(f"  {label}: {path}")
+
+
+if __name__ == "__main__":
+    main()