Deploy vectorization service from GitHub be8a53f8cee433cb3de44a139be2fab8e47b1ccb

README.md

@@ -42,6 +42,55 @@ curl -sS -X POST 'http://localhost:7860/colbert/encode' \
 
 E5 keeps `query: ` / `passage: ` prefixes, attention-mask-aware mean pooling, **no extra L2 normalization**. Qwen uses its official query instruction + last-token pooling + L2 normalization; documents have no instruction. PyLate mLateOn handles query and passage separately, retaining token vectors and their model-native normalization. Token limits are checked *including model markers and special tokens*, without silent truncation. Exceeding limits or using invalid input returns HTTP 422. When the shared inference gate is occupied, requests return HTTP 429 with `Retry-After`.
 
+## OpenAI-compatible API (separate from Voyage)
+
+**OpenAI SDK-compatible base URL:** \`https://YOUR-SPACE.hf.space/openai/v1\`.
+This separate namespace is necessary because the existing Voyage-compatible
+\`/v1/embeddings\` endpoint is already in production and must retain its default
+\`input_type\` handling. Point OpenAI-compatible custom-provider integrations
+to the base URL ending in \`/openai/v1\`, not to the Voyage \`/v1\` URL.
+
+- \`POST /openai/v1/embeddings\`: accepts \`model\`, \`input\` (string or string array),
+  optional \`dimensions\`, \`encoding_format\` (\`float\` or \`base64\`) and \`user\`.
+  Responds in the OpenAI \`data[].embedding\` / \`usage.prompt_tokens\` /
+  \`usage.total_tokens\` format. Only fixed native dimensions are supported.
+- \`GET /openai/v1/models\`: dense model aliases for provider selection.
+- Bearer authentication uses the **same** \`API_KEY\` as Voyage; no new secrets.
+- Dense E5 and Qwen3 only. ColBERT retains \`/colbert/encode\` and Voyage extension.
+- **Crucial:** this endpoint forwards raw input to the tokenizer **without adding any
+  query/document instructions**. Supply prefixes in the client configuration.
+  It never silently truncates, and unsupported dimensions return an error.
+
+For E5 configure:
+- Document prefix: \`passage: \`
+- Query prefix: \`query: \`
+
+For Qwen3:
+- Document prefix: *empty*
+- Query prefix: \`Instruct: Given a question, retrieve relevant passages that answer the question\nQuery:\`
+
+The Qwen query prefix contains an actual newline before \`Query:\`; ensure the
+client inserts a newline character rather than the two characters backslash+n.
+Use \`Query:\` immediately before the question, matching the model's original
+instruction template.
+
+Example with the standard Python OpenAI SDK:
+
+\`\`\`python
+from openai import OpenAI
+
+client = OpenAI(
+    base_url="https://YOUR-SPACE.hf.space/openai/v1",
+    api_key="YOUR_API_KEY",
+)
+# The calling application has to apply the configured prefix.
+embedding = client.embeddings.create(
+    model="e5",
+    input=["passage: Первое предложение", "passage: Второе предложение"],
+)
+print(len(embedding.data[0].embedding))  # 384
+\`\`\`
+
 ## Official Voyage AI SDK compatibility
 
 The **unmodified Voyage Python client** can talk directly to our service.

VERSION

@@ -1 +1 @@
-6f9bd192211bf40966f92934e10f54f002c3c28b
+be8a53f8cee433cb3de44a139be2fab8e47b1ccb

app/batching.py

@@ -33,8 +33,12 @@ def _token_count(tokenizer, text):
     return len(encoded["input_ids"])
 
 
-def _dense_batch(engine, texts, kind, is_qwen):
-    if is_qwen:
+def _dense_batch(engine, texts, kind, is_qwen, raw=False):
+    if raw:
+        # OpenAI-compatible callers provide preformatted text; never add prompts.
+        strings = list(texts)
+        budget, max_size = (4096, 16) if is_qwen else (8192, 64)
+    elif is_qwen:
         strings = [
             f"Instruct: {INSTRUCTION}\nQuery:{text}" if kind == "query" else text
             for text in texts
@@ -122,7 +126,7 @@ def _colbert_batch(engine, texts, kind):
     return results
 
 
-def batch_encode(engine, texts, kind, is_qwen=False, is_colbert=False):
+def batch_encode(engine, texts, kind, is_qwen=False, is_colbert=False, raw=False):
     """Native tensor batch for real models, single-item fallback for test doubles."""
     if is_colbert:
         if not hasattr(engine, "model") or not hasattr(engine.model, "tokenizer"):
@@ -130,4 +134,6 @@ def batch_encode(engine, texts, kind, is_qwen=False, is_colbert=False):
         return _colbert_batch(engine, texts, kind)
     if not hasattr(engine, "tokenizer") or not hasattr(engine, "model"):
         return [engine.encode(text, kind) for text in texts]
-    return _dense_batch(engine, texts, kind, is_qwen)
+    if raw and (not hasattr(engine, "tokenizer") or not hasattr(engine, "model")):
+        raise RuntimeError("Raw embeddings require the loaded model and tokenizer")
+    return _dense_batch(engine, texts, kind, is_qwen, raw=raw)

app/main.py

@@ -41,6 +41,7 @@ async def lifespan(application: FastAPI):
 
 
 from app.voyage_api import router as voyage_router
+from app.openai_api import router as openai_router
 
 app = FastAPI(
     title="Vectorization API",
@@ -51,6 +52,7 @@ app = FastAPI(
 
 
 app.include_router(voyage_router)
+app.include_router(openai_router)
 
 
 class TextRequest(BaseModel):

app/models.py

@@ -165,6 +165,19 @@ class ModelRegistry:
         result = self._run(lambda: engine.encode(text, kind))
         return (QWEN_ID if engine is self.qwen else E5_ID), engine.dimensions, result
 
+    def openai_encode(self, texts, canonical_model):
+        """Raw OpenAI-compatible embeddings: client supplies any query/document prefix."""
+        from app.batching import batch_encode
+        if canonical_model == E5_ID:
+            engine, is_qwen = self.e5, False
+        elif canonical_model == QWEN_ID:
+            engine, is_qwen = self.qwen, True
+        else:
+            raise ValueError("OpenAI embeddings only support dense E5 and Qwen models")
+        return self._run(lambda: batch_encode(
+            engine, texts, "passage", is_qwen=is_qwen, raw=True,
+        ))
+
     def colbert_encode(self, text, kind):
         return self._run(lambda: self.colbert.encode(text, kind))
 

app/openai_api.py

@@ -0,0 +1,111 @@
+"""OpenAI SDK-compatible embeddings API, intentionally separate from Voyage routes.
+
+Configure OpenAI(base_url="https://HOST/openai/v1", api_key="...").
+The client is responsible for model-specific query/document prefixes.
+"""
+from __future__ import annotations
+
+import base64
+import logging
+import struct
+from typing import Literal
+
+from fastapi import APIRouter, Depends, HTTPException
+from fastapi.responses import JSONResponse
+from pydantic import BaseModel, ConfigDict, Field, field_validator
+
+from app.embedding import MODEL_ID as E5_ID, TooManyTokens
+from app.models import ModelRegistry, QWEN_ID, ServiceBusy
+from app.voyage_api import authenticate, resident_models
+
+router = APIRouter(prefix="/openai/v1", tags=["OpenAI-compatible"])
+log = logging.getLogger(__name__)
+DenseModelName = Literal[
+    "e5", "qwen", "intfloat/multilingual-e5-small", "Qwen/Qwen3-Embedding-0.6B"
+]
+
+
+class OpenAIEmbeddingRequest(BaseModel):
+    model_config = ConfigDict(extra="forbid")
+    model: DenseModelName
+    input: str | list[str] = Field(
+        description="Raw input; client must add model-specific query/document prefixes"
+    )
+    dimensions: int | None = Field(default=None, ge=1)
+    encoding_format: Literal["float", "base64"] = "float"
+    user: str | None = None
+
+    @field_validator("input")
+    @classmethod
+    def validate_input(cls, value):
+        texts = [value] if isinstance(value, str) else value
+        if not texts or len(texts) > 1000:
+            raise ValueError("input must contain between 1 and 1000 texts")
+        if any(not text.strip() for text in texts):
+            raise ValueError("input cannot contain blank strings")
+        return value
+
+
+def api_error(status: int, message: str, error_type="invalid_request_error", code=None):
+    return JSONResponse(status_code=status, content={
+        "error": {"message": message, "type": error_type, "param": None, "code": code}
+    })
+
+
+@router.post("/embeddings", dependencies=[Depends(authenticate)])
+def create_embedding(
+    request: OpenAIEmbeddingRequest,
+    registry: ModelRegistry = Depends(resident_models),
+):
+    model = request.model
+    canonical = QWEN_ID if model in ("qwen", QWEN_ID) else E5_ID
+    engine = registry.qwen if canonical == QWEN_ID else registry.e5
+    if request.dimensions is not None and request.dimensions != engine.dimensions:
+        return api_error(
+            400,
+            f"Model {model} supports exactly {engine.dimensions} dimensions",
+            code="unsupported_dimensions",
+        )
+
+    texts = [request.input] if isinstance(request.input, str) else request.input
+    try:
+        results = registry.openai_encode(texts, canonical)
+    except TooManyTokens as exc:
+        return api_error(400, str(exc), code="context_length_exceeded")
+    except ValueError as exc:
+        return api_error(400, str(exc))
+    except ServiceBusy as exc:
+        response = api_error(429, str(exc), "rate_limit_error", "rate_limit_exceeded")
+        response.headers["Retry-After"] = "2"
+        return response
+    except Exception:
+        log.exception("OpenAI-compatible embedding failed")
+        return api_error(500, "Embedding generation failed", "server_error")
+
+    def encode(vector):
+        if request.encoding_format == "float":
+            return vector
+        raw = struct.pack(f"<{len(vector)}f", *vector)
+        return base64.b64encode(raw).decode("ascii")
+
+    total_tokens = sum(result.tokens_count for result in results)
+    return {
+        "object": "list",
+        "data": [
+            {"object": "embedding", "index": index, "embedding": encode(item.embedding)}
+            for index, item in enumerate(results)
+        ],
+        "model": model,
+        "usage": {"prompt_tokens": total_tokens, "total_tokens": total_tokens},
+    }
+
+
+@router.get("/models", dependencies=[Depends(authenticate)])
+def list_models():
+    return {
+        "object": "list",
+        "data": [
+            {"id": mid, "object": "model", "created": 0, "owned_by": "local"}
+            for mid in ("e5", "qwen", E5_ID, QWEN_ID)
+        ],
+    }