Deploy vectorization service from GitHub da30902c39d4577af0609f4da9aa94e293a64033

README.md

@@ -40,7 +40,9 @@ curl -sS -X POST 'http://localhost:7860/colbert/encode' \
   -d '{"text":"שלום עולם","type":"passage"}'
 ```
 
-E5 keeps `query: ` / `passage: ` prefixes, attention-mask-aware mean pooling, **no extra L2 normalization**. Qwen uses its official query instruction + last-token pooling + L2 normalization; documents have no instruction. PyLate mLateOn handles query and passage separately, retaining token vectors and their model-native normalization. Token limits are checked *including model markers and special tokens*, without silent truncation. Exceeding limits or using invalid input returns HTTP 422. A single CPU inference runs at a time; up to 16 additional HTTP requests wait
+E5 keeps `query: ` / `passage: ` prefixes, attention-mask-aware mean pooling, **no extra L2 normalization**. Qwen uses its official query instruction + last-token pooling + L2 normalization; documents have no instruction. PyLate mLateOn handles query and passage separately, retaining token vectors and their model-native normalization. Token limits include model markers and special tokens. The legacy and Voyage
+APIs reject overlong inputs (HTTP 422); only the OpenAI-compatible E5
+endpoint truncates inputs exceeding 512 tokens, logging affected batch indices. A single CPU inference runs at a time; up to 16 additional HTTP requests wait
 for a free inference slot (40 seconds by default). A full queue or expired wait
 returns HTTP 429 with `Retry-After`. Configure `INFERENCE_MAX_WAITERS` and
 `INFERENCE_WAIT_SECONDS` as needed. Dify's OpenAI-compatible embedding provider
@@ -63,7 +65,10 @@ to the base URL ending in `/openai/v1`, not to the Voyage `/v1` URL.
 - Dense E5 and Qwen3 only. ColBERT retains `/colbert/encode` and Voyage extension.
 - **Crucial:** this endpoint forwards raw input to the tokenizer **without adding any
   query/document instructions**. Supply prefixes in the client configuration.
-  It never silently truncates, and unsupported dimensions return an error.
+  For **E5 only**, inputs exceeding 512 tokens are truncated from the right
+  to 512 tokens, including the prefix and special tokens. Short inputs are
+  unchanged; truncation is logged with the input index and original token count.
+  Qwen inputs are not truncated. Unsupported dimensions return an error.
 
 For E5 configure in Dify:
 - Document prefix: `passage:`
@@ -72,7 +77,8 @@ For E5 configure in Dify:
   which is not this server's limit). Dify inserts a space after the prefix.
 
 The OpenAI endpoint accepts up to 1000 strings per request, subject to
-65,536 total input tokens and internal padding-aware microbatches. Large
+65,536 total **processed** input tokens (after E5 truncation) and internal
+padding-aware microbatches. Large
 requests may still exceed the client's timeout; prefer batches of 8–16 on CPU.
 
 For Qwen3:

VERSION

@@ -1 +1 @@
-3807fb18ead86e70413d1081aa3bedb1935f6e61
+da30902c39d4577af0609f4da9aa94e293a64033

app/batching.py

@@ -5,9 +5,13 @@ No model initialization, truncation, or extra E5 normalization occurs here.
 """
 from __future__ import annotations
 
+import logging
+
 from app.embedding import EmbeddingResult, TooManyTokens
 from app.models import ColbertResult, validate_vector, INSTRUCTION
 
+log = logging.getLogger(__name__)
+
 
 def _split_by_padding_cost(counts, token_budget, max_size):
     """Partition ordered inputs, accounting for batch padding to longest text."""
@@ -33,7 +37,7 @@ def _token_count(tokenizer, text):
     return len(encoded["input_ids"])
 
 
-def _dense_batch(engine, texts, kind, is_qwen, raw=False):
+def _dense_batch(engine, texts, kind, is_qwen, raw=False, truncate_long=False):
     if raw:
         # OpenAI-compatible callers provide preformatted text; never add prompts.
         strings = list(texts)
@@ -53,7 +57,14 @@ def _dense_batch(engine, texts, kind, is_qwen, raw=False):
     counts = [_token_count(engine.tokenizer, text) for text in strings]
     for index, count in enumerate(counts):
         if count > engine.max_tokens:
-            raise TooManyTokens(count, engine.max_tokens, input_index=index)
+            if not truncate_long:
+                raise TooManyTokens(count, engine.max_tokens, input_index=index)
+            log.warning(
+                "OpenAI embeddings truncated: model=e5 input_index=%d "
+                "original_tokens=%d used_tokens=%d",
+                index, count, engine.max_tokens,
+            )
+            counts[index] = engine.max_tokens
     # Bound total work in one HTTP request (not the model context limit).
     if sum(counts) > 65536:
         raise ValueError("Batch exceeds 65536 total input tokens")
@@ -61,10 +72,17 @@ def _dense_batch(engine, texts, kind, is_qwen, raw=False):
     results = [None] * len(texts)
     for group in _split_by_padding_cost(counts, budget, max_size):
         inputs = [strings[i] for i in group]
-        batch = engine.tokenizer(
-            inputs, return_tensors="pt", padding=True,
-            truncation=False, add_special_tokens=True,
-        )
+        tokenizer_options = {
+            "return_tensors": "pt",
+            "padding": True,
+            "truncation": truncate_long,
+            "add_special_tokens": True,
+        }
+        if truncate_long:
+            # Hugging Face preserves special tokens inside max_length and
+            # truncates from the right, keeping the leading query:/passage: prefix.
+            tokenizer_options["max_length"] = engine.max_tokens
+        batch = engine.tokenizer(inputs, **tokenizer_options)
         with engine.torch.inference_mode():
             hidden = engine.model(**batch).last_hidden_state
             if is_qwen:
@@ -126,8 +144,11 @@ def _colbert_batch(engine, texts, kind):
     return results
 
 
-def batch_encode(engine, texts, kind, is_qwen=False, is_colbert=False, raw=False):
+def batch_encode(engine, texts, kind, is_qwen=False, is_colbert=False,
+                 raw=False, truncate_long=False):
     """Native tensor batch for real models, single-item fallback for test doubles."""
+    if truncate_long and (not raw or is_qwen or is_colbert):
+        raise ValueError("Automatic truncation is only supported for raw E5 input")
     if is_colbert:
         if not hasattr(engine, "model") or not hasattr(engine.model, "tokenizer"):
             return [engine.encode(text, kind) for text in texts]
@@ -136,4 +157,5 @@ def batch_encode(engine, texts, kind, is_qwen=False, is_colbert=False, raw=False
         if raw:
             raise RuntimeError("Raw embeddings require the loaded model and tokenizer")
         return [engine.encode(text, kind) for text in texts]
-    return _dense_batch(engine, texts, kind, is_qwen, raw=raw)
+    return _dense_batch(engine, texts, kind, is_qwen, raw=raw,
+                        truncate_long=truncate_long)

app/models.py

@@ -190,6 +190,7 @@ class ModelRegistry:
             raise ValueError("OpenAI embeddings only support dense E5 and Qwen models")
         return self._run(lambda: batch_encode(
             engine, texts, "passage", is_qwen=is_qwen, raw=True,
+            truncate_long=(canonical_model == E5_ID),
         ))
 
     def colbert_encode(self, text, kind):