import json, numpy as np, onnxruntime as ort from PIL import Image from tokenizers import Tokenizer D = '/tmp/work/clip/' _v = _t = _tok = None MEAN = np.array([0.48145466, 0.4578275, 0.40821073], np.float32); STD = np.array([0.26862954, 0.26130258, 0.27577711], np.float32) def _load(): global _v, _t, _tok if _v is None: o = ort.SessionOptions(); o.intra_op_num_threads = 2 _v = ort.InferenceSession(D + 'vision_model_quantized.onnx', o) _t = ort.InferenceSession(D + 'text_model_quantized.onnx', o) _tok = Tokenizer.from_file(D + 'tokenizer.json') def image_emb(img): _load() img = img.convert('RGB'); w, h = img.size; s = 224 / min(w, h) img = img.resize((max(224, round(w * s)), max(224, round(h * s))), Image.BICUBIC) w, h = img.size; l, t = (w - 224) // 2, (h - 224) // 2 a = (np.asarray(img.crop((l, t, l + 224, t + 224)), np.float32) / 255 - MEAN) / STD e = _v.run(None, {'pixel_values': a.transpose(2, 0, 1)[None]})[0][0] return e / np.linalg.norm(e) def text_emb(texts): _load() out = [] for s in texts: ids = _tok.encode(s).ids[:77] e = _t.run(None, {'input_ids': np.array([ids], np.int64)})[0][0] out.append(e / np.linalg.norm(e)) return np.array(out)