File size: 5,536 Bytes
d704ace
 
14479b0
d704ace
 
 
 
 
 
 
665ddab
 
d704ace
 
 
 
 
 
 
 
665ddab
d704ace
 
 
 
 
665ddab
 
 
 
 
d704ace
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
7bb019c
 
 
d704ace
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
665ddab
d704ace
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
7bb019c
d8f6ca9
 
7bb019c
 
 
d704ace
 
 
 
d8f6ca9
d704ace
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
7bb019c
 
d704ace
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
"""Pinned E5 inference with a legacy endpoint and a verified release protocol."""
# Import Spaces before Torch; ZeroGPU owns CUDA emulation and allocation.
import spaces
import gradio as gr
import torch
from huggingface_hub import snapshot_download
from transformers import AutoModel, AutoTokenizer
from pathlib import Path
from importlib.metadata import version
import platform
import subprocess
import sys

from protocol import (
    MAX_LENGTH, MODEL, MODEL_REVISION, contract_for_snapshot, input_fingerprint, prepare_input,
)

ROOT = Path(__file__).parent
EXPECTED_PACKAGES = {
    "gradio": "6.30.0", "spaces": "0.51.3", "torch": "2.8.0",
    "transformers": "5.22.2", "numpy": "2.4.6", "huggingface-hub": "1.33.0", "datasets": "5.0.1",
}
PACKAGE_VERSIONS = {name: version(name) for name in EXPECTED_PACKAGES}
if any(PACKAGE_VERSIONS[name].split("+", 1)[0] != expected
       for name, expected in EXPECTED_PACKAGES.items()):
    raise RuntimeError("Installed encoder dependencies differ from the reviewed lock")
DEPENDENCY_CHECK = subprocess.run(
    [sys.executable, "-m", "pip", "check"], capture_output=True, text=True, timeout=15,
)
if DEPENDENCY_CHECK.returncode:
    raise RuntimeError("Installed Space dependency graph is inconsistent")
torch.manual_seed(0)
torch.backends.cuda.matmul.allow_tf32 = False
torch.backends.cudnn.allow_tf32 = False

# Official full-revision cache snapshots bind both model and tokenizer files.
# https://huggingface.co/docs/huggingface_hub/guides/download
snapshot = Path(snapshot_download(
    MODEL, revision=MODEL_REVISION, token=False,
    allow_patterns=["*.json", "*.safetensors", "sentencepiece.bpe.model"], max_workers=4,
))
if snapshot.name != MODEL_REVISION:
    raise RuntimeError("Model snapshot resolution failed")
tokenizer = AutoTokenizer.from_pretrained(
    snapshot, local_files_only=True, trust_remote_code=False, token=False,
)
# ZeroGPU recommends placing the model on CUDA at module startup.
# https://huggingface.co/docs/hub/spaces-zerogpu
model = AutoModel.from_pretrained(
    snapshot, local_files_only=True, trust_remote_code=False, token=False,
    use_safetensors=True, dtype=torch.float32,
).to("cuda").eval()
if model.dtype != torch.float32:
    raise RuntimeError("Loaded encoder weights must be float32")
CONTRACT = contract_for_snapshot(snapshot.name, dimensions=model.config.hidden_size, root=ROOT)


@spaces.GPU(duration=10)
def pooled_vector(batch, normalized):
    batch = {name: value.to("cuda") for name, value in batch.items()}
    # Official E5 attention-mask mean pooling, with FP32 weights and no autocast.
    # https://huggingface.co/intfloat/multilingual-e5-large
    with torch.inference_mode(), torch.autocast("cuda", enabled=False):
        hidden = model(**batch).last_hidden_state
        hidden = hidden.masked_fill(~batch["attention_mask"][..., None].bool(), 0.0)
        values = hidden.sum(dim=1) / batch["attention_mask"].sum(dim=1)[..., None]
        if normalized:
            values = torch.nn.functional.normalize(values, p=2, dim=1)
    return values[0].cpu().float().tolist()


def metadata():
    return dict(CONTRACT)


def runtime_info():
    return {
        "python": platform.python_version(), "packages": dict(PACKAGE_VERSIONS),
        "dependency_check_passed": DEPENDENCY_CHECK.returncode == 0,
        "torch_cuda_build": torch.version.cuda, "torch_dtype": str(model.dtype),
        "snapshot_revision": snapshot.name,
        "implementation_sha256": CONTRACT["implementation_sha256"],
        "dependency_lock_sha256": CONTRACT["dependency_lock_sha256"],
    }


def encode_pinned(text: str, kind: str):
    prefixed = prepare_input(text, kind)
    batch = tokenizer(prefixed, return_tensors="pt", padding=True, truncation=False)
    if batch["input_ids"].shape[1] > MAX_LENGTH:
        raise ValueError("Encoder input exceeds token limit")
    return {
        "contract": dict(CONTRACT), "input_sha256": input_fingerprint(text, kind),
        "vector": pooled_vector(batch, True),
    }


def encode_text(text: str):
    # SentenceTransformer's original modules.json includes Normalize, even when
    # encode(normalize_embeddings=False) is passed. Preserve that unit output.
    if not text or not text.strip():
        return []
    cleaned = text.strip()
    if not cleaned.startswith(("query: ", "passage: ")):
        cleaned = "query: " + cleaned
    batch = tokenizer(cleaned, return_tensors="pt", padding=True,
                      truncation=True, max_length=MAX_LENGTH)
    return pooled_vector(batch, True)


with gr.Blocks(analytics_enabled=False, title="Pinned multilingual E5") as demo:
    gr.Markdown("Multilingual E5 embedding API")
    text = gr.Textbox(label="Text")
    kind = gr.Dropdown(["query", "passage"], value="query", label="Input kind")
    output = gr.JSON(label="Embedding")
    gr.Button("Encode").click(
        encode_text, inputs=[text], outputs=output, api_name="encode",
        concurrency_id="encoder", concurrency_limit=1,
    )
    gr.Button("Encode pinned").click(
        encode_pinned, inputs=[text, kind], outputs=output, api_name="encode_pinned",
        concurrency_id="encoder", concurrency_limit=1,
    )
    gr.Button("Metadata").click(
        metadata, inputs=[], outputs=gr.JSON(label="Contract"), api_name="metadata", queue=False,
    )
    gr.Button("Runtime").click(
        runtime_info, inputs=[], outputs=gr.JSON(label="Runtime"), api_name="runtime", queue=False,
    )
demo.queue(max_size=16)

if __name__ == "__main__":
    demo.launch(show_error=False)