Spaces:
Sleeping
Sleeping
File size: 2,584 Bytes
3694da1 f070f64 3694da1 c27330d 3694da1 c27330d 3694da1 c27330d 3694da1 c27330d f070f64 3694da1 c27330d f070f64 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 | """Multilingual embeddings via fastembed (ONNX-based, no torch dependency).
We use `sentence-transformers/paraphrase-multilingual-MiniLM-L12-v2`:
• 120MB ONNX model — small enough for the free HF Space
• 384-dim output, well-supported in fastembed
• Covers ~50 languages including French/Spanish/Arabic → close enough
to Mauritian Kreol for retrieval to work (Creole shares heavy
French-derived vocabulary)
• Comparable retrieval quality to e5-small at similar size
To see fastembed's full supported-model list:
from fastembed import TextEmbedding
TextEmbedding.list_supported_models()
"""
from __future__ import annotations
import os
from functools import lru_cache
from typing import Iterable
import numpy as np
# Already MULTILINGUAL by default (~50 languages incl. French → covers most of
# Kreol's French-derived vocabulary). Swap to a stronger multilingual model
# without code changes, e.g.:
# EMBED_MODEL=intfloat/multilingual-e5-large (1024-d, 100 languages, heavier)
# EMBED_MODEL=intfloat/multilingual-e5-small (384-d, 100 languages)
# After changing it, delete data/knowledge.sqlite so the base re-embeds.
DEFAULT_MODEL = os.environ.get(
"EMBED_MODEL", "sentence-transformers/paraphrase-multilingual-MiniLM-L12-v2")
EMBED_DIM = int(os.environ.get("EMBED_DIM", "384"))
def _is_e5(name: str) -> bool:
return "e5" in (name or "").lower()
@lru_cache(maxsize=2)
def _model(model_name: str):
from fastembed import TextEmbedding
print(f"[knowledge] Loading embedder: {model_name}")
return TextEmbedding(model_name=model_name)
def embed_texts(
texts: Iterable[str], model_name: str = DEFAULT_MODEL
) -> np.ndarray:
"""Embed a list of strings → (N, dim) float32, L2-normalised."""
texts = list(texts)
if not texts:
return np.zeros((0, EMBED_DIM), dtype=np.float32)
model = _model(model_name)
embeddings = list(model.embed(texts))
arr = np.array(embeddings, dtype=np.float32)
norms = np.linalg.norm(arr, axis=1, keepdims=True)
norms[norms == 0] = 1.0
return arr / norms
def embed_passages(texts: Iterable[str]) -> np.ndarray:
"""Embed text chunks for storage (e5 models need the 'passage:' prefix)."""
texts = list(texts)
if _is_e5(DEFAULT_MODEL):
texts = [f"passage: {t}" for t in texts]
return embed_texts(texts)
def embed_query(text: str) -> np.ndarray:
"""Embed a search query (e5 models need the 'query:' prefix)."""
if _is_e5(DEFAULT_MODEL):
text = f"query: {text}"
return embed_texts([text])[0]
|