thong7d's picture
Pin E5 model and add verified encoder metadata and normalized endpoint
d8f6ca9 verified
Raw History Blame Contribute Delete
5.54 kB
"""Pinned E5 inference with a legacy endpoint and a verified release protocol."""
# Import Spaces before Torch; ZeroGPU owns CUDA emulation and allocation.
import spaces
import gradio as gr
import torch
from huggingface_hub import snapshot_download
from transformers import AutoModel, AutoTokenizer
from pathlib import Path
from importlib.metadata import version
import platform
import subprocess
import sys
from protocol import (
MAX_LENGTH, MODEL, MODEL_REVISION, contract_for_snapshot, input_fingerprint, prepare_input,
)
ROOT = Path(__file__).parent
EXPECTED_PACKAGES = {
"gradio": "6.30.0", "spaces": "0.51.3", "torch": "2.8.0",
"transformers": "5.22.2", "numpy": "2.4.6", "huggingface-hub": "1.33.0", "datasets": "5.0.1",
}
PACKAGE_VERSIONS = {name: version(name) for name in EXPECTED_PACKAGES}
if any(PACKAGE_VERSIONS[name].split("+", 1)[0] != expected
for name, expected in EXPECTED_PACKAGES.items()):
raise RuntimeError("Installed encoder dependencies differ from the reviewed lock")
DEPENDENCY_CHECK = subprocess.run(
[sys.executable, "-m", "pip", "check"], capture_output=True, text=True, timeout=15,
)
if DEPENDENCY_CHECK.returncode:
raise RuntimeError("Installed Space dependency graph is inconsistent")
torch.manual_seed(0)
torch.backends.cuda.matmul.allow_tf32 = False
torch.backends.cudnn.allow_tf32 = False
# Official full-revision cache snapshots bind both model and tokenizer files.
# https://huggingface.co/docs/huggingface_hub/guides/download
snapshot = Path(snapshot_download(
MODEL, revision=MODEL_REVISION, token=False,
allow_patterns=["*.json", "*.safetensors", "sentencepiece.bpe.model"], max_workers=4,
))
if snapshot.name != MODEL_REVISION:
raise RuntimeError("Model snapshot resolution failed")
tokenizer = AutoTokenizer.from_pretrained(
snapshot, local_files_only=True, trust_remote_code=False, token=False,
)
# ZeroGPU recommends placing the model on CUDA at module startup.
# https://huggingface.co/docs/hub/spaces-zerogpu
model = AutoModel.from_pretrained(
snapshot, local_files_only=True, trust_remote_code=False, token=False,
use_safetensors=True, dtype=torch.float32,
).to("cuda").eval()
if model.dtype != torch.float32:
raise RuntimeError("Loaded encoder weights must be float32")
CONTRACT = contract_for_snapshot(snapshot.name, dimensions=model.config.hidden_size, root=ROOT)
@spaces.GPU(duration=10)
def pooled_vector(batch, normalized):
batch = {name: value.to("cuda") for name, value in batch.items()}
# Official E5 attention-mask mean pooling, with FP32 weights and no autocast.
# https://huggingface.co/intfloat/multilingual-e5-large
with torch.inference_mode(), torch.autocast("cuda", enabled=False):
hidden = model(**batch).last_hidden_state
hidden = hidden.masked_fill(~batch["attention_mask"][..., None].bool(), 0.0)
values = hidden.sum(dim=1) / batch["attention_mask"].sum(dim=1)[..., None]
if normalized:
values = torch.nn.functional.normalize(values, p=2, dim=1)
return values[0].cpu().float().tolist()
def metadata():
return dict(CONTRACT)
def runtime_info():
return {
"python": platform.python_version(), "packages": dict(PACKAGE_VERSIONS),
"dependency_check_passed": DEPENDENCY_CHECK.returncode == 0,
"torch_cuda_build": torch.version.cuda, "torch_dtype": str(model.dtype),
"snapshot_revision": snapshot.name,
"implementation_sha256": CONTRACT["implementation_sha256"],
"dependency_lock_sha256": CONTRACT["dependency_lock_sha256"],
}
def encode_pinned(text: str, kind: str):
prefixed = prepare_input(text, kind)
batch = tokenizer(prefixed, return_tensors="pt", padding=True, truncation=False)
if batch["input_ids"].shape[1] > MAX_LENGTH:
raise ValueError("Encoder input exceeds token limit")
return {
"contract": dict(CONTRACT), "input_sha256": input_fingerprint(text, kind),
"vector": pooled_vector(batch, True),
}
def encode_text(text: str):
# SentenceTransformer's original modules.json includes Normalize, even when
# encode(normalize_embeddings=False) is passed. Preserve that unit output.
if not text or not text.strip():
return []
cleaned = text.strip()
if not cleaned.startswith(("query: ", "passage: ")):
cleaned = "query: " + cleaned
batch = tokenizer(cleaned, return_tensors="pt", padding=True,
truncation=True, max_length=MAX_LENGTH)
return pooled_vector(batch, True)
with gr.Blocks(analytics_enabled=False, title="Pinned multilingual E5") as demo:
gr.Markdown("Multilingual E5 embedding API")
text = gr.Textbox(label="Text")
kind = gr.Dropdown(["query", "passage"], value="query", label="Input kind")
output = gr.JSON(label="Embedding")
gr.Button("Encode").click(
encode_text, inputs=[text], outputs=output, api_name="encode",
concurrency_id="encoder", concurrency_limit=1,
)
gr.Button("Encode pinned").click(
encode_pinned, inputs=[text, kind], outputs=output, api_name="encode_pinned",
concurrency_id="encoder", concurrency_limit=1,
)
gr.Button("Metadata").click(
metadata, inputs=[], outputs=gr.JSON(label="Contract"), api_name="metadata", queue=False,
)
gr.Button("Runtime").click(
runtime_info, inputs=[], outputs=gr.JSON(label="Runtime"), api_name="runtime", queue=False,
)
demo.queue(max_size=16)
if __name__ == "__main__":
demo.launch(show_error=False)