benchmark/config.py

"""Models under test and benchmark-wide settings.

`query_prefix` / `passage_prefix` are plain-text instruction prefixes prepended
before encoding (the E5 family expects "query: " / "passage: "; other model
families used here don't need one). This is applied manually rather than via
SentenceTransformer's `prompt_name=` API so behavior doesn't depend on whether
a given model repo happens to ship a config_sentence_transformers.json with
named prompts.
"""

MODELS = [
    {
        "name": "kazalbrur/bangla-embed-e5-small-banglish",
        "query_prefix": "query: ",
        "passage_prefix": "passage: ",
    },
    {
        "name": "BAAI/bge-m3",
        "query_prefix": "",
        "passage_prefix": "",
    },
    {
        # 4B/8B variants are skipped: their fp32 weights (~16GB / ~32GB) don't fit
        # this machine's 8GB VRAM or its ~15GB system RAM.
        "name": "Qwen/Qwen3-Embedding-0.6B",
        "query_prefix": (
            "Instruct: Given a web search query, retrieve relevant passages "
            "that answer the query\nQuery:"
        ),
        "passage_prefix": "",
    },
    {
        "name": "intfloat/multilingual-e5-large",
        "query_prefix": "query: ",
        "passage_prefix": "passage: ",
    },
    {
        # trust_remote_code is required to load Jina's custom modeling code; the
        # model uses task-specific LoRA adapters selected via an encode()-time
        # `task` kwarg rather than a text prefix.
        "name": "jinaai/jina-embeddings-v3",
        "query_prefix": "",
        "passage_prefix": "",
        "trust_remote_code": True,
        "query_encode_kwargs": {"task": "retrieval.query"},
        "passage_encode_kwargs": {"task": "retrieval.passage"},
    },
    {
        "name": "sentence-transformers/LaBSE",
        "query_prefix": "",
        "passage_prefix": "",
    },
    {
        # trust_remote_code is required to load Alibaba's custom modeling code.
        "name": "Alibaba-NLP/gte-multilingual-base",
        "query_prefix": "",
        "passage_prefix": "",
        "trust_remote_code": True,
    },
]

QUERY_MODES = ["raw", "normalized"]

RETRIEVAL_MODES = ["dense", "hybrid"]  # hybrid = BM25 + dense fused with RRF

TOP_K = 50  # depth retrieved per query; recall@10/@50 and nDCG@10 are sliced from this
FUSION_DEPTH = 50  # how deep each ranker (BM25, dense) is pulled before RRF fusion
RRF_K = 60  # RRF's rank-damping constant

BOOKS_PATH = "boi1_sample_books.json"
QUERIES_PATH = "boi1_eval_queries.json"

QDRANT_PATH = "./qdrant_data"
RESULTS_DIR = "./results"