File size: 1,804 Bytes
d704ace
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
"""CPU-only input and identity boundary for the pinned E5 Space."""
import hashlib
import json
from pathlib import Path

MODEL = "intfloat/multilingual-e5-large"
MODEL_REVISION = "3d7cfbdacd47fdda877c5cd8a79fbcc4f2a574f3"
MAX_LENGTH = 512


def _canonical(value):
    return json.dumps(value, ensure_ascii=False, sort_keys=True, separators=(",", ":"),
                      allow_nan=False).encode("utf-8")


def input_fingerprint(text, kind):
    return hashlib.sha256(_canonical({"kind": kind, "text": text})).hexdigest()


def prepare_input(text, kind):
    if (kind not in ("query", "passage") or not isinstance(text, str) or not text.strip()
            or len(text) > (4000 if kind == "query" else 16000)
            or len(text.encode("utf-8")) > 24000):
        raise ValueError("Invalid encoder input")
    return ("query: " if kind == "query" else "passage: ") + text


def contract_for_snapshot(snapshot_revision, *, dimensions, root):
    if snapshot_revision != MODEL_REVISION or dimensions != 1024:
        raise ValueError("Loaded encoder snapshot differs from the pinned release")
    root = Path(root)
    hashes = {
        name: hashlib.sha256((root / name).read_bytes()).hexdigest()
        for name in ["app.py", "protocol.py"]
    }
    return {
        "schema_version": 1, "model": MODEL, "revision": snapshot_revision,
        "tokenizer": MODEL, "tokenizer_revision": snapshot_revision,
        "pooling": "masked_mean", "normalized": True, "dtype": "float32",
        "dimensions": dimensions, "max_length": MAX_LENGTH,
        "query_prefix": "query: ", "passage_prefix": "passage: ",
        "implementation_sha256": hashlib.sha256(_canonical(hashes)).hexdigest(),
        "dependency_lock_sha256": hashlib.sha256((root / "requirements.txt").read_bytes()).hexdigest(),
    }