Blame for embed_models.py

"""Registry of embedding models for the SQ1 model screen.

Each entry: HF id, max sequence length the model supports, the prefix (or prompt) it expects on passages
and on queries, and the folder its shards live in under data/.  Shared by the Colab notebook, build_index.py
and search.py so every model is embedded and queried the way its authors intend.

    from embed_models import MODELS
    MODELS["bge"]  -> {"hf": "BAAI/bge-m3", "max_seq": 1024, ...}
"""
MODELS = {
    # key      HF model id                                    max_seq  passage prefix   query prefix                                   index dir     notes
    "bge":     {"hf": "BAAI/bge-m3",                              "max_seq": 1024, "passage": "",            "query": "",
                "dir": "index",      "note": "8k context; 100+ languages; current baseline"},
    "sealion": {"hf": "aisingapore/SEA-LION-E5-Embedding-600M",   "max_seq": 512,  "passage": "passage: ",  "query": "query: ",
                "dir": "index_sealion", "note": "mE5-large fine-tuned on 11 SEA languages incl. Burmese; 512-token limit"},
    "me5i":    {"hf": "intfloat/multilingual-e5-large-instruct",  "max_seq": 512,  "passage": "",
                "query": "Instruct: Given a question about Myanmar parliamentary debates, retrieve the passages that answer it\nQuery: ",
                "dir": "index_me5i", "note": "best Burmese score on SEA-BED; instruction on the query side only; 512-token limit"},
    "me5":     {"hf": "intfloat/multilingual-e5-large",           "max_seq": 512,  "passage": "passage: ",  "query": "query: ",
                "dir": "index_me5",  "note": "plain mE5; 512-token limit"},
    "modernbert": {"hf": "aisingapore/SEA-LION-ModernBERT-Embedding-600M", "max_seq": 1024, "passage": "", "query": "",
                "dir": "index_modernbert", "note": "8k context, SEA languages incl. Burmese; new, not yet on the leaderboard"},
    "myx":     {"hf": "DatarrX/myX-Semantic",                     "max_seq": 512,  "passage": "",            "query": "",
                "dir": "index_myx",  "note": "Burmese-only MiniLM distillation, 768-d; no published benchmarks; cheap to run"},
}
DEFAULT = "bge"

def spec(key):
    if key not in MODELS: raise KeyError(f"unknown embedding model '{key}'; choose from {', '.join(MODELS)}")
    return MODELS[key]