Buckets:
AnonymousSpottedToad/human_verified_bucket / codebase /python /ctrl-gaurav_effGen /effgen /api /embeddings.py
| """OpenAI-compatible local embeddings API. | |
| Provides a ``/v1/embeddings`` endpoint that works out of the box without | |
| paid APIs. Uses ``sentence-transformers`` when available, otherwise falls | |
| back to an in-process TF-IDF hashing embedder so the endpoint remains | |
| functional in minimal installs. | |
| Caching: | |
| * In-memory LRU cache for repeated texts. | |
| * Optional SQLite-backed persistent cache keyed by (model, text_hash). | |
| Usage: | |
| >>> from fastapi import FastAPI | |
| >>> from effgen.api.embeddings import create_embeddings_router | |
| >>> app = FastAPI() | |
| >>> app.include_router(create_embeddings_router()) | |
| """ | |
| from __future__ import annotations | |
| import hashlib | |
| import logging | |
| import math | |
| import os | |
| import re | |
| import sqlite3 | |
| import threading | |
| from collections import OrderedDict | |
| from typing import Any | |
| logger = logging.getLogger(__name__) | |
| # Model aliases — OpenAI-style names mapped to local models. | |
| MODEL_ALIASES = { | |
| "text-embedding-small": "sentence-transformers/all-MiniLM-L6-v2", | |
| "text-embedding-ada-002": "sentence-transformers/all-MiniLM-L6-v2", | |
| "text-embedding-3-small": "sentence-transformers/all-MiniLM-L6-v2", | |
| "text-embedding-3-large": "sentence-transformers/all-mpnet-base-v2", | |
| } | |
| DEFAULT_DIM = 384 # TF-IDF fallback dimension | |
| # Provider prefixes an OpenAI-compatible client may send (the same ``provider:`` | |
| # prefix the chat endpoint accepts). They are stripped before the alias lookup so | |
| # that, e.g., ``openai:text-embedding-3-small`` resolves to the real neural model | |
| # exactly as the bare ``text-embedding-3-small`` does — instead of falling | |
| # through to the lexical fallback because the prefixed id matched no alias. | |
| _PROVIDER_PREFIXES = ( | |
| "openai", | |
| "hf", | |
| "huggingface", | |
| "sentence-transformers", | |
| "together", | |
| "fireworks", | |
| "cohere", | |
| "voyage", | |
| ) | |
| # Identifier reported on the response (and ``x-effgen-embedding-backend`` header) | |
| # when the requested neural backend could not load and the lexical TF-IDF | |
| # embedder served the request instead. Never silently substituted. | |
| TFIDF_FALLBACK_ID = "tfidf-fallback" | |
| def _strict_mode() -> bool: | |
| """Whether to fail closed (503) rather than serve degraded TF-IDF vectors.""" | |
| return os.getenv("EFFGEN_EMBEDDINGS_STRICT", "0").strip().lower() in ( | |
| "1", "true", "yes", "on", | |
| ) | |
| class EmbeddingBackendUnavailable(RuntimeError): | |
| """The requested neural embedding backend could not be loaded (strict mode).""" | |
| # --------------------------------------------------------------------------- | |
| # Caches | |
| # --------------------------------------------------------------------------- | |
| class LRUCache: | |
| """Thread-safe LRU cache for embeddings keyed by (model, text).""" | |
| def __init__(self, max_size: int = 1024) -> None: | |
| self.max_size = max_size | |
| self._data: "OrderedDict[str, list[float]]" = OrderedDict() | |
| self._lock = threading.Lock() | |
| def key(model: str, text: str) -> str: | |
| """Return the cache key for *model* and *text* (SHA-256 of the text).""" | |
| h = hashlib.sha256(text.encode("utf-8")).hexdigest() | |
| return f"{model}:{h}" | |
| def get(self, model: str, text: str) -> list[float] | None: | |
| """Return the cached vector for (*model*, *text*), or ``None`` on a miss.""" | |
| k = self.key(model, text) | |
| with self._lock: | |
| val = self._data.get(k) | |
| if val is not None: | |
| self._data.move_to_end(k) | |
| return val | |
| def put(self, model: str, text: str, vec: list[float]) -> None: | |
| """Store *vec* for (*model*, *text*), evicting the oldest entries past capacity.""" | |
| k = self.key(model, text) | |
| with self._lock: | |
| self._data[k] = vec | |
| self._data.move_to_end(k) | |
| while len(self._data) > self.max_size: | |
| self._data.popitem(last=False) | |
| class SQLiteCache: | |
| """Persistent embedding cache backed by SQLite.""" | |
| def __init__(self, path: str | None = None) -> None: | |
| self.path = path or os.path.expanduser("~/.effgen/embeddings_cache.db") | |
| os.makedirs(os.path.dirname(self.path), exist_ok=True) | |
| self._lock = threading.Lock() | |
| self._conn = sqlite3.connect(self.path, check_same_thread=False) | |
| self._conn.execute( | |
| "CREATE TABLE IF NOT EXISTS embeddings (" | |
| " model TEXT NOT NULL," | |
| " text_hash TEXT NOT NULL," | |
| " vector BLOB NOT NULL," | |
| " dim INTEGER NOT NULL," | |
| " PRIMARY KEY (model, text_hash)" | |
| ")" | |
| ) | |
| self._conn.commit() | |
| def _hash(text: str) -> str: | |
| return hashlib.sha256(text.encode("utf-8")).hexdigest() | |
| def get(self, model: str, text: str) -> list[float] | None: | |
| """Return the stored vector for (*model*, *text*), or ``None`` on a miss.""" | |
| with self._lock: | |
| row = self._conn.execute( | |
| "SELECT vector, dim FROM embeddings WHERE model=? AND text_hash=?", | |
| (model, self._hash(text)), | |
| ).fetchone() | |
| if not row: | |
| return None | |
| blob, dim = row | |
| import struct | |
| return list(struct.unpack(f"{dim}f", blob)) | |
| def put(self, model: str, text: str, vec: list[float]) -> None: | |
| """Persist *vec* for (*model*, *text*), replacing any existing row.""" | |
| import struct | |
| blob = struct.pack(f"{len(vec)}f", *vec) | |
| with self._lock: | |
| self._conn.execute( | |
| "INSERT OR REPLACE INTO embeddings (model, text_hash, vector, dim)" | |
| " VALUES (?, ?, ?, ?)", | |
| (model, self._hash(text), blob, len(vec)), | |
| ) | |
| self._conn.commit() | |
| # --------------------------------------------------------------------------- | |
| # Embedder backends | |
| # --------------------------------------------------------------------------- | |
| _WORD_RE = re.compile(r"[a-zA-Z0-9_]+") | |
| class TFIDFEmbedder: | |
| """Hashing TF-IDF embedder — dependency-free fallback. | |
| Produces fixed-dimension L2-normalized sparse-to-dense vectors using a | |
| hashing trick. Not as good as a neural embedder, but deterministic and | |
| works without any additional packages. | |
| """ | |
| def __init__(self, dim: int = DEFAULT_DIM) -> None: | |
| self.dim = dim | |
| def _tokenize(self, text: str) -> list[str]: | |
| return [t.lower() for t in _WORD_RE.findall(text)] | |
| def embed(self, texts: list[str]) -> list[list[float]]: | |
| """Return one L2-normalized hashed TF vector per input text.""" | |
| out: list[list[float]] = [] | |
| for text in texts: | |
| vec = [0.0] * self.dim | |
| tokens = self._tokenize(text) | |
| if not tokens: | |
| out.append(vec) | |
| continue | |
| # term frequency with hashing | |
| for tok in tokens: | |
| h = int(hashlib.md5(tok.encode("utf-8")).hexdigest(), 16) | |
| idx = h % self.dim | |
| sign = 1.0 if (h >> 32) & 1 else -1.0 | |
| vec[idx] += sign | |
| # IDF-ish log dampening | |
| norm = math.sqrt(sum(v * v for v in vec)) | |
| if norm > 0: | |
| vec = [v / norm for v in vec] | |
| out.append(vec) | |
| return out | |
| class SentenceTransformerEmbedder: | |
| """Wrapper around ``sentence-transformers`` when installed.""" | |
| def __init__(self, model_name: str) -> None: | |
| from effgen.utils.embedding_backend import load_sentence_transformer | |
| self.model_name = model_name | |
| self.model = load_sentence_transformer(model_name) | |
| def embed(self, texts: list[str]) -> list[list[float]]: | |
| """Return one normalized sentence-transformer vector per input text.""" | |
| vecs = self.model.encode(texts, normalize_embeddings=True) | |
| return [list(map(float, v)) for v in vecs] | |
| # --------------------------------------------------------------------------- | |
| # Engine | |
| # --------------------------------------------------------------------------- | |
| class EmbeddingEngine: | |
| """Main embedding engine: resolves models, caches, dispatches.""" | |
| def __init__( | |
| self, | |
| lru_size: int = 1024, | |
| persistent_cache: bool = True, | |
| cache_path: str | None = None, | |
| ) -> None: | |
| self.lru = LRUCache(max_size=lru_size) | |
| self.sqlite: SQLiteCache | None | |
| try: | |
| self.sqlite = SQLiteCache(cache_path) if persistent_cache else None | |
| except Exception: | |
| self.sqlite = None | |
| self._backends: dict = {} | |
| self._backend_lock = threading.Lock() | |
| # Resolved model ids that fell back to the lexical TF-IDF embedder | |
| # because their neural backend could not load — surfaced to callers. | |
| self._fallback_models: set[str] = set() | |
| def _resolve_model(self, model: str) -> str: | |
| # Strip a known ``provider:`` prefix so a prefixed id resolves like the | |
| # bare alias (``openai:text-embedding-3-small`` -> the MiniLM model). | |
| bare = model | |
| prefix, sep, rest = model.partition(":") | |
| if sep and prefix.lower() in _PROVIDER_PREFIXES and rest: | |
| bare = rest | |
| return MODEL_ALIASES.get(bare, bare) | |
| def is_fallback(self, model: str) -> bool: | |
| """True when *model* is being served by the lexical TF-IDF fallback.""" | |
| return self._resolve_model(model) in self._fallback_models | |
| def _get_backend(self, model: str) -> Any: | |
| resolved = self._resolve_model(model) | |
| with self._backend_lock: | |
| if resolved in self._backends: | |
| return self._backends[resolved] | |
| backend: Any | |
| if isinstance(resolved, str) and resolved.lower().startswith("tfidf"): | |
| # The caller explicitly asked for the lexical embedder. | |
| backend = TFIDFEmbedder() | |
| else: | |
| try: | |
| backend = SentenceTransformerEmbedder(resolved) | |
| except Exception as exc: # noqa: BLE001 - neural backend optional | |
| if _strict_mode(): | |
| # Fail closed instead of quietly degrading the vectors. | |
| raise EmbeddingBackendUnavailable( | |
| f"embedding backend for {resolved!r} could not load " | |
| f"({exc}). Install the neural backend " | |
| "(`pip install 'effgen[rag]'`) and check the model " | |
| "id, or set EFFGEN_EMBEDDINGS_STRICT=0 to allow the " | |
| "lexical TF-IDF fallback." | |
| ) from exc | |
| # Non-spammy fallback: warn once per model, record it | |
| # so the response can tell the caller the vectors are lexical. | |
| self._fallback_models.add(resolved) | |
| logger.warning( | |
| "Embedding backend for %r could not load (%s); serving " | |
| "lexical TF-IDF fallback vectors. Install/repair " | |
| "sentence-transformers for neural embeddings, or set " | |
| "EFFGEN_EMBEDDINGS_STRICT=1 to fail closed instead.", | |
| resolved, exc, | |
| ) | |
| backend = TFIDFEmbedder() | |
| self._backends[resolved] = backend | |
| return backend | |
| def embed(self, texts: list[str], model: str = "text-embedding-small") -> list[list[float]]: | |
| """Embed *texts* with *model*, serving repeats from the LRU/SQLite caches.""" | |
| if not texts: | |
| return [] | |
| resolved = self._resolve_model(model) | |
| # Lookup caches | |
| results: list[list[float] | None] = [None] * len(texts) | |
| missing_idx: list[int] = [] | |
| missing_texts: list[str] = [] | |
| for i, t in enumerate(texts): | |
| cached = self.lru.get(resolved, t) | |
| if cached is None and self.sqlite is not None: | |
| cached = self.sqlite.get(resolved, t) | |
| if cached is not None: | |
| self.lru.put(resolved, t, cached) | |
| if cached is not None: | |
| results[i] = cached | |
| else: | |
| missing_idx.append(i) | |
| missing_texts.append(t) | |
| if missing_texts: | |
| backend = self._get_backend(model) | |
| new_vecs = backend.embed(missing_texts) | |
| for idx, text, vec in zip(missing_idx, missing_texts, new_vecs): | |
| results[idx] = vec | |
| self.lru.put(resolved, text, vec) | |
| if self.sqlite is not None: | |
| try: | |
| self.sqlite.put(resolved, text, vec) | |
| except Exception: | |
| pass | |
| return [r for r in results if r is not None] # type: ignore[misc] | |
| # --------------------------------------------------------------------------- | |
| # FastAPI router (optional dependency) | |
| # --------------------------------------------------------------------------- | |
| # Imported at module level so the ``response: Response`` route annotation | |
| # resolves under ``from __future__ import annotations`` (FastAPI reads type hints | |
| # from module globals; a function-local import would not be found and the param | |
| # would be misread as a query field). | |
| try: | |
| from fastapi import Response # type: ignore | |
| except Exception: # pragma: no cover - fastapi optional | |
| Response = None # type: ignore | |
| try: | |
| from pydantic import BaseModel as _PydBaseModel # type: ignore | |
| from pydantic import Field as _PydField | |
| class EmbeddingRequest(_PydBaseModel): | |
| """Request body for ``POST /v1/embeddings``.""" | |
| model: str = _PydField(default="text-embedding-small") | |
| input: str | list[str] | |
| class EmbeddingItem(_PydBaseModel): | |
| """One embedding vector in the response ``data`` list.""" | |
| object: str = "embedding" | |
| index: int | |
| embedding: list[float] | |
| class EmbeddingUsage(_PydBaseModel): | |
| """Token usage counters for an embeddings request.""" | |
| prompt_tokens: int = 0 | |
| total_tokens: int = 0 | |
| class EmbeddingResponse(_PydBaseModel): | |
| """Response body for ``POST /v1/embeddings``.""" | |
| object: str = "list" | |
| data: list[EmbeddingItem] | |
| model: str | |
| usage: EmbeddingUsage | |
| # Non-standard extension (OpenAI clients ignore unknown keys): records | |
| # the resolved model and flags when the lexical TF-IDF fallback served | |
| # the request, so a caller building semantic search/RAG can tell the | |
| # vectors are degraded rather than neural. | |
| effgen: dict[str, Any] | None = None | |
| except Exception: # pragma: no cover - pydantic optional | |
| EmbeddingRequest = None # type: ignore | |
| EmbeddingItem = None # type: ignore | |
| EmbeddingUsage = None # type: ignore | |
| EmbeddingResponse = None # type: ignore | |
| def create_embeddings_router(engine: EmbeddingEngine | None = None) -> Any: | |
| """Create a FastAPI router exposing ``/v1/embeddings``. | |
| Requires ``fastapi`` and ``pydantic`` to be installed. Otherwise import | |
| of this function will raise. | |
| """ | |
| from fastapi import APIRouter, Body # type: ignore | |
| from fastapi.responses import JSONResponse # type: ignore | |
| from effgen.api.openai_compat import error_envelope | |
| eng = engine or EmbeddingEngine() | |
| router = APIRouter() | |
| def create_embeddings( | |
| response: Response, req: EmbeddingRequest = Body(...) | |
| ) -> Any: | |
| # A JSONResponse built from the shared `{"error": {message, type, param, | |
| # code}}` envelope (the same one every other /v1 route uses) bypasses | |
| # the declared `response_model` and is returned as-is, matching every | |
| # other error path on this server instead of FastAPI's default | |
| # `{"detail": ...}` shape. | |
| texts = [req.input] if isinstance(req.input, str) else list(req.input) | |
| # An empty list — or one holding nothing but blank strings — has no text | |
| # to embed. Reject it with a 400 instead of returning vectors for | |
| # nothing, the same rule /v1/chat/completions applies to content-free | |
| # messages. A blank entry alongside real text is still embedded, so a | |
| # batch caller keeps index alignment. | |
| if not texts or all(not str(t).strip() for t in texts): | |
| return JSONResponse( | |
| status_code=400, | |
| content=error_envelope(400, "input must not be empty", code="empty_input"), | |
| ) | |
| try: | |
| # Resolve the backend first so the fallback status is known even when | |
| # every vector comes from the cache (a cache hit skips backend | |
| # creation, so without this the accuracy flag could go stale). | |
| eng._get_backend(req.model) | |
| vecs = eng.embed(texts, model=req.model) | |
| except EmbeddingBackendUnavailable as e: | |
| # Strict mode: refuse to serve degraded vectors under a neural id. | |
| return JSONResponse( | |
| status_code=503, | |
| content=error_envelope(503, str(e), code="embedding_backend_unavailable"), | |
| ) | |
| except Exception as e: | |
| return JSONResponse( | |
| status_code=500, | |
| content=error_envelope(500, f"embedding failed: {e}"), | |
| ) | |
| total_tokens = sum(len(t.split()) for t in texts) | |
| resolved = eng._resolve_model(req.model) | |
| degraded = eng.is_fallback(req.model) | |
| backend = TFIDF_FALLBACK_ID if degraded else "sentence-transformers" | |
| # Reflect the real backend on a header *and* the body extension so the | |
| # caller is never fooled into thinking lexical hash vectors are neural. | |
| response.headers["x-effgen-embedding-backend"] = backend | |
| return EmbeddingResponse( | |
| data=[EmbeddingItem(index=i, embedding=v) for i, v in enumerate(vecs)], | |
| model=req.model, | |
| usage=EmbeddingUsage(prompt_tokens=total_tokens, total_tokens=total_tokens), | |
| effgen={ | |
| "requested_model": req.model, | |
| "resolved_model": resolved, | |
| "backend": backend, | |
| "degraded": degraded, | |
| }, | |
| ) | |
| return router | |
| __all__ = [ | |
| "MODEL_ALIASES", | |
| "TFIDF_FALLBACK_ID", | |
| "LRUCache", | |
| "SQLiteCache", | |
| "TFIDFEmbedder", | |
| "SentenceTransformerEmbedder", | |
| "EmbeddingBackendUnavailable", | |
| "EmbeddingEngine", | |
| "create_embeddings_router", | |
| ] | |
Xet Storage Details
- Size:
- 18.5 kB
- Xet hash:
- 41b5a587f30bac1e5d77ddbb2107a98b7de83e32f43e0f3d22e7d69d5a9db354
·
Xet efficiently stores files, intelligently splitting them into unique chunks and accelerating uploads and downloads. More info.