Download plugin/plugins/VibeSearch/vibemodel.py from Divi19/vibe-embed: direct link, hf CLI and curl.
- Browser
- Download file 9.07 kB
-
https://huggingface.co/Divi19/vibe-embed/resolve/main/plugin/plugins/VibeSearch/vibemodel.py
- Command line
-
hf download hf://Divi19/vibe-embed/plugin/plugins/VibeSearch/vibemodel.py
-
curl -L -o vibemodel.py https://huggingface.co/Divi19/vibe-embed/resolve/main/plugin/plugins/VibeSearch/vibemodel.py
9.07 kB
| """Runtime for the vibe models: ONNX audio and text encoders, with the exact preprocessing they were trained with. | |
| This module has no AudioMuse-AI imports, so it can be used and tested on its own. | |
| Audio: 32 kHz mono float32. A song is split into non-overlapping 10 s windows (a shorter last window is kept if it | |
| is at least 2 s long), each window is embedded, the window vectors are L2-normalised and averaged, and the average is | |
| L2-normalised again. Text: the prompt from config.json is put in front of the query, the result is tokenised (left | |
| padding, at most 256 tokens) and embedded. Both give 512 numbers in the same space, so cosine similarity compares a | |
| search with a song, or a song with a song. | |
| """ | |
| import hashlib | |
| import json | |
| import os | |
| import threading | |
| import numpy as np | |
| SR = 32000 | |
| WIN = 10 * SR | |
| MIN_LAST = 2 * SR | |
| MAX_TOKENS = 256 | |
| AUDIO_FILE = {"int8": "audio_encoder_int8.onnx", "fp32": "audio_encoder.onnx"} | |
| TEXT_FILE = {"int8": "text/text_encoder_int8.onnx", "fp32": "text/text_encoder.onnx"} | |
| TOKENIZER_FILES = ["text/tokenizer/tokenizer.json", "text/tokenizer/tokenizer_config.json"] | |
| COMMON_FILES = ["config.json", "parity.json"] | |
| PARITY_MIN = {"fp32": 0.995, "int8": 0.98} | |
| def l2n(x): | |
| x = np.asarray(x, dtype=np.float32) | |
| return x / np.maximum(np.linalg.norm(x, axis=-1, keepdims=True), 1e-12) | |
| def split_windows(wave): | |
| """Non-overlapping 10 s windows; a shorter last window is kept if it is at least 2 s (or if it is the only one).""" | |
| n = len(wave) | |
| out = [] | |
| for a in range(0, max(n, 1), WIN): | |
| b = min(a + WIN, n) | |
| if b - a >= MIN_LAST or a == 0: | |
| out.append(wave[a:b]) | |
| return [w for w in out if len(w) > 0] | |
| def files_needed(student, precision, kinds): | |
| need = list(COMMON_FILES) | |
| if "audio" in kinds: | |
| need.append(f"{student}/{AUDIO_FILE[precision]}") | |
| if "text" in kinds: | |
| need += [TEXT_FILE[precision]] + TOKENIZER_FILES | |
| return need | |
| def download_files(model_dir, base_url, repo, revision, rel_paths, log=None, timeout=(10, 120)): | |
| """Fetch missing files from a Hugging Face style repository (<base>/<repo>/resolve/<revision>/<path>). | |
| A large ONNX model may come with a companion .onnx.data file; it is fetched when the repository has one.""" | |
| import requests | |
| base = f"{base_url.rstrip('/')}/{repo}/resolve/{revision}/" | |
| wanted = [] | |
| for rel in rel_paths: | |
| wanted.append((rel, False)) | |
| if rel.endswith(".onnx"): | |
| wanted.append((rel + ".data", True)) | |
| for rel, optional in wanted: | |
| dst = os.path.join(model_dir, rel) | |
| if os.path.exists(dst): | |
| continue | |
| os.makedirs(os.path.dirname(dst), exist_ok=True) | |
| with requests.get(base + rel, stream=True, timeout=timeout) as r: | |
| if optional and r.status_code == 404: | |
| continue | |
| r.raise_for_status() | |
| with open(dst + ".part", "wb") as f: | |
| for chunk in r.iter_content(1 << 22): | |
| f.write(chunk) | |
| os.replace(dst + ".part", dst) | |
| if log: | |
| log(f"vibe_search: downloaded {rel}") | |
| class VibeModel: | |
| """Loads one side (audio or text) of the vibe model and checks it against the reference values in parity.json.""" | |
| def __init__(self, model_dir, student="mn20_as", precision="int8", threads=4): | |
| self.model_dir, self.student, self.precision = model_dir, student, precision | |
| self.threads = max(1, int(threads)) | |
| self.config = json.load(open(os.path.join(model_dir, "config.json"))) | |
| self.parity = json.load(open(os.path.join(model_dir, "parity.json"))) | |
| self.prompt = self.config["text"]["prompt"] | |
| self.audio = self.text = self.tok = None | |
| self._lock = threading.Lock() | |
| def tag(self): | |
| """Identifies the model version the stored song vectors belong to (int8 and fp32 give the same vectors).""" | |
| h = hashlib.md5(json.dumps(self.config, sort_keys=True).encode()).hexdigest()[:12] | |
| return f"{self.student}:{h}" | |
| def _session(self, rel): | |
| import onnxruntime as ort | |
| so = ort.SessionOptions() | |
| so.intra_op_num_threads = self.threads | |
| so.inter_op_num_threads = 1 | |
| return ort.InferenceSession(os.path.join(self.model_dir, rel), so, providers=["CPUExecutionProvider"]) | |
| # ---------- audio ---------- | |
| def load_audio(self): | |
| self.audio = self._session(f"{self.student}/{AUDIO_FILE[self.precision]}") | |
| self.audio_input = self.audio.get_inputs()[0].name | |
| ref = np.asarray(self.parity["audio"][self.student], np.float32) | |
| t = np.arange(WIN, dtype=np.float64) / SR | |
| wave = (0.1 * np.sin(2 * np.pi * (220.0 * t + 50.0 * t ** 2))).astype(np.float32) | |
| cos = float((self._run_audio(wave[None])[0] * l2n(ref)).sum()) | |
| if cos < PARITY_MIN[self.precision]: | |
| raise RuntimeError(f"audio model check failed: cosine {cos:.4f} to the reference (expected >= {PARITY_MIN[self.precision]})") | |
| return cos | |
| def _run_audio(self, batch): | |
| with self._lock: | |
| return l2n(self.audio.run(None, {self.audio_input: np.ascontiguousarray(batch, dtype=np.float32)})[0]) | |
| def embed_audio(self, wave, batch=8): | |
| """wave: 32 kHz mono float32. Returns (512-number song vector, number of windows).""" | |
| wins = split_windows(np.asarray(wave, np.float32)) | |
| if not wins: | |
| raise ValueError("empty audio") | |
| full = [w for w in wins if len(w) == WIN] | |
| rest = [w for w in wins if len(w) != WIN] | |
| vecs = [self._run_audio(np.stack(full[i:i + batch])) for i in range(0, len(full), batch)] | |
| vecs += [self._run_audio(w[None]) for w in rest] | |
| return l2n(np.concatenate(vecs).mean(0)), len(wins) | |
| # ---------- text ---------- | |
| def _load_tokenizer(self, expected): | |
| """The tokenizers library first; transformers as a fallback. Either must reproduce the reference token ids.""" | |
| tdir = os.path.join(self.model_dir, "text", "tokenizer") | |
| errors = [] | |
| try: | |
| from tokenizers import Tokenizer | |
| tok = Tokenizer.from_file(os.path.join(tdir, "tokenizer.json")) | |
| cfg = json.load(open(os.path.join(tdir, "tokenizer_config.json"))) | |
| pad = cfg.get("pad_token") | |
| pad = pad.get("content") if isinstance(pad, dict) else pad | |
| pad_id = tok.token_to_id(pad) if pad else None | |
| if pad_id is None: | |
| pad_id = tok.token_to_id("<|endoftext|>") or 0 | |
| pad = tok.id_to_token(pad_id) | |
| tok.enable_truncation(MAX_TOKENS) | |
| tok.enable_padding(direction="left", pad_id=pad_id, pad_token=pad) | |
| def enc_tok(texts): | |
| e = tok.encode_batch([self.prompt + t for t in texts]) | |
| return np.array([x.ids for x in e], np.int64), np.array([x.attention_mask for x in e], np.int64) | |
| if all(ids is None or list(enc_tok([q])[0][0]) == ids for q, ids in expected): | |
| return enc_tok, "tokenizers" | |
| errors.append("tokenizers: token ids differ from the reference") | |
| except Exception as e: | |
| errors.append(f"tokenizers: {e}") | |
| try: | |
| from transformers import AutoTokenizer | |
| hf = AutoTokenizer.from_pretrained(tdir, padding_side="left", local_files_only=True) | |
| def enc_hf(texts): | |
| e = hf([self.prompt + t for t in texts], padding=True, truncation=True, max_length=MAX_TOKENS, return_tensors="np") | |
| return e["input_ids"].astype(np.int64), e["attention_mask"].astype(np.int64) | |
| if all(ids is None or list(enc_hf([q])[0][0]) == ids for q, ids in expected): | |
| return enc_hf, "transformers" | |
| errors.append("transformers: token ids differ from the reference") | |
| except Exception as e: | |
| errors.append(f"transformers: {e}") | |
| raise RuntimeError("could not reproduce the model's tokenisation: " + "; ".join(errors)) | |
| def load_text(self): | |
| # token ids are compared when parity.json has them; the vectors below are always compared | |
| expected = [(t["query"], t.get("input_ids")) for t in self.parity["texts"]] | |
| self.tok, self.tokenizer_backend = self._load_tokenizer(expected) | |
| self.text = self._session(TEXT_FILE[self.precision]) | |
| got = self.embed_texts([q for q, _ in expected]) | |
| ref = l2n(np.asarray([t["vector"] for t in self.parity["texts"]], np.float32)) | |
| cos = float((got * ref).sum(1).min()) | |
| if cos < PARITY_MIN[self.precision]: | |
| raise RuntimeError(f"text model check failed: cosine {cos:.4f} to the reference (expected >= {PARITY_MIN[self.precision]})") | |
| return cos | |
| def embed_texts(self, texts, batch=16): | |
| out = [] | |
| for i in range(0, len(texts), batch): | |
| ids, mask = self.tok(texts[i:i + batch]) | |
| with self._lock: | |
| out.append(self.text.run(None, {"input_ids": ids, "attention_mask": mask})[0]) | |
| return l2n(np.concatenate(out)) | |