Spaces:
Running on Zero
Running on Zero
File size: 1,804 Bytes
d704ace | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 | """CPU-only input and identity boundary for the pinned E5 Space."""
import hashlib
import json
from pathlib import Path
MODEL = "intfloat/multilingual-e5-large"
MODEL_REVISION = "3d7cfbdacd47fdda877c5cd8a79fbcc4f2a574f3"
MAX_LENGTH = 512
def _canonical(value):
return json.dumps(value, ensure_ascii=False, sort_keys=True, separators=(",", ":"),
allow_nan=False).encode("utf-8")
def input_fingerprint(text, kind):
return hashlib.sha256(_canonical({"kind": kind, "text": text})).hexdigest()
def prepare_input(text, kind):
if (kind not in ("query", "passage") or not isinstance(text, str) or not text.strip()
or len(text) > (4000 if kind == "query" else 16000)
or len(text.encode("utf-8")) > 24000):
raise ValueError("Invalid encoder input")
return ("query: " if kind == "query" else "passage: ") + text
def contract_for_snapshot(snapshot_revision, *, dimensions, root):
if snapshot_revision != MODEL_REVISION or dimensions != 1024:
raise ValueError("Loaded encoder snapshot differs from the pinned release")
root = Path(root)
hashes = {
name: hashlib.sha256((root / name).read_bytes()).hexdigest()
for name in ["app.py", "protocol.py"]
}
return {
"schema_version": 1, "model": MODEL, "revision": snapshot_revision,
"tokenizer": MODEL, "tokenizer_revision": snapshot_revision,
"pooling": "masked_mean", "normalized": True, "dtype": "float32",
"dimensions": dimensions, "max_length": MAX_LENGTH,
"query_prefix": "query: ", "passage_prefix": "passage: ",
"implementation_sha256": hashlib.sha256(_canonical(hashes)).hexdigest(),
"dependency_lock_sha256": hashlib.sha256((root / "requirements.txt").read_bytes()).hexdigest(),
}
|