Download plugin/plugins/VibeSearch/__init__.py from Divi19/vibe-embed: direct link, hf CLI and curl.
- Browser
- Download file 18.2 kB
-
https://huggingface.co/Divi19/vibe-embed/resolve/main/plugin/plugins/VibeSearch/__init__.py
- Command line
-
hf download hf://Divi19/vibe-embed/plugin/plugins/VibeSearch/__init__.py
-
curl -L -o __init__.py https://huggingface.co/Divi19/vibe-embed/resolve/main/plugin/plugins/VibeSearch/__init__.py
18.2 kB
| """Vibe Search: search your library by how music sounds and feels ("rainy Sunday coffee shop", "upbeat but | |
| melancholic, no vocals"), or find songs with the same vibe as one you pick, and save the results as a playlist. | |
| It runs next to AudioMuse-AI's own models and changes nothing in them: | |
| * after each song is analysed, a small audio model (EfficientAT mn20_as, distilled from a teacher that joins | |
| LCO-Embedding-Omni-7B and e5-omni-7B) gives it a 512-number vibe vector, stored in this plugin's own table; | |
| * "Index my library" fills in the songs that were analysed before the plugin was installed; | |
| * searches are turned into the same kind of vector by a text model (Qwen3-Embedding-0.6B plus a small trained layer) | |
| and compared with every song. | |
| The models are downloaded on first use from the Hugging Face repository set on the settings page. | |
| """ | |
| import html | |
| import os | |
| import threading | |
| import time | |
| import numpy as np | |
| from flask import Blueprint, redirect, request, url_for | |
| from plugin.api import (config, enqueue, get_db, get_score_data_by_ids, get_setting, logger, manage_plugins_url, | |
| render_page, set_setting, table) | |
| from . import vibemodel | |
| bp = Blueprint("vibe_search", __name__) | |
| DEFAULTS = { | |
| "model_repo": "Divi19/vibe-embed", | |
| "revision": "main", | |
| "base_url": "https://huggingface.co", | |
| "precision": "int8", | |
| "threads": 4, | |
| "embed_on_analysis": True, | |
| "result_limit": 50, | |
| } | |
| STUDENT = "mn20_as" # the only audio model size the release ships | |
| INT8_TEXT_MIN_ORT = (1, 28) # the 8-bit text model uses an operator setting that older onnxruntime builds lack | |
| def setting(key): | |
| return get_setting(key, DEFAULTS[key]) | |
| def ort_version(): | |
| try: | |
| import onnxruntime as ort | |
| return tuple(int(x) for x in ort.__version__.split(".")[:2]) | |
| except Exception: | |
| return (0, 0) | |
| def model_dir(): | |
| base = getattr(config, "PLUGINS_DIR", "") or "/tmp" | |
| return os.path.join(base, "_data", "vibe_search", "models", setting("model_repo").replace("/", "__"), setting("revision")) | |
| # ---------------------------------------------------------------- models (loaded once per container, on first use) | |
| _models, _model_errors, _model_notes, _model_lock = {}, {}, {}, threading.Lock() | |
| def get_model(kind): | |
| """kind: 'audio' (worker: indexing) or 'text' (web: searching). Returns the loaded model or None (error logged).""" | |
| wanted = setting("precision") | |
| key = (kind, setting("model_repo"), setting("revision"), STUDENT, wanted) | |
| with _model_lock: | |
| if key in _models: | |
| return _models[key] | |
| tries = [wanted] if wanted == "fp32" else ["int8", "fp32"] | |
| if kind == "text" and wanted == "int8" and ort_version() < INT8_TEXT_MIN_ORT: | |
| tries = ["fp32"] # e.g. AudioMuse-AI's no-AVX2 image (onnxruntime 1.19): skip the 600 MB download that cannot run | |
| _model_notes[kind] = (f"The search model runs at full precision: this onnxruntime ({'.'.join(map(str, ort_version()))}) " | |
| f"is older than {'.'.join(map(str, INT8_TEXT_MIN_ORT))}, which the 8-bit file needs.") | |
| errors = [] | |
| for precision in tries: | |
| try: | |
| d = model_dir() | |
| vibemodel.download_files(d, setting("base_url"), setting("model_repo"), setting("revision"), | |
| vibemodel.files_needed(STUDENT, precision, [kind]), log=logger.info) | |
| m = vibemodel.VibeModel(d, STUDENT, precision, setting("threads")) | |
| cos = m.load_audio() if kind == "audio" else m.load_text() | |
| logger.info(f"vibe_search: {kind} model ready ({m.tag}, {precision}, check {cos:.4f})") | |
| if precision != wanted and kind not in _model_notes: | |
| _model_notes[kind] = f"The {kind} model runs at full precision because the 8-bit file failed: {errors[-1]}" | |
| _models[key] = m | |
| _model_errors.pop(kind, None) | |
| return m | |
| except Exception as e: | |
| errors.append(f"{precision}: {type(e).__name__}: {e}") | |
| logger.exception(f"vibe_search: the {kind} model ({precision}) could not be loaded") | |
| _model_errors[kind] = "; ".join(errors) | |
| return None | |
| def current_tag(): | |
| """The model version stored vectors must match. Read from config.json without loading a model.""" | |
| try: | |
| import json, hashlib | |
| cfg = json.load(open(os.path.join(model_dir(), "config.json"))) | |
| return f"{STUDENT}:{hashlib.md5(json.dumps(cfg, sort_keys=True).encode()).hexdigest()[:12]}" | |
| except Exception: | |
| return None | |
| # ---------------------------------------------------------------- storage | |
| def vec_table(): | |
| return table("vectors") | |
| def migrate(db): | |
| cur = db.cursor() | |
| cur.execute("CREATE TABLE IF NOT EXISTS " + vec_table() + " (item_id TEXT PRIMARY KEY, model_tag TEXT NOT NULL, " | |
| "vec BYTEA NOT NULL, n_windows INTEGER, updated_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP)") | |
| cur.execute("CREATE INDEX IF NOT EXISTS " + vec_table() + "_tag ON " + vec_table() + " (model_tag)") | |
| db.commit() | |
| cur.close() | |
| def save_vector(item_id, tag, vec, n_windows): | |
| db = get_db() | |
| cur = db.cursor() | |
| cur.execute("INSERT INTO " + vec_table() + " (item_id, model_tag, vec, n_windows) VALUES (%s, %s, %s, %s) " | |
| "ON CONFLICT (item_id) DO UPDATE SET model_tag = EXCLUDED.model_tag, vec = EXCLUDED.vec, " | |
| "n_windows = EXCLUDED.n_windows, updated_at = CURRENT_TIMESTAMP", | |
| (str(item_id), tag, np.asarray(vec, np.float16).tobytes(), int(n_windows))) | |
| db.commit() | |
| cur.close() | |
| _matrix_cache.clear() | |
| def indexed_ids(tag): | |
| cur = get_db().cursor() | |
| cur.execute("SELECT item_id FROM " + vec_table() + " WHERE model_tag = %s", (tag,)) | |
| out = {r[0] for r in cur.fetchall()} | |
| cur.close() | |
| return out | |
| _matrix_cache = {} | |
| def song_matrix(tag): | |
| """All stored song vectors of this model version, cached until a vector is added.""" | |
| cur = get_db().cursor() | |
| cur.execute("SELECT COUNT(*), MAX(updated_at) FROM " + vec_table() + " WHERE model_tag = %s", (tag,)) | |
| stamp = tuple(str(x) for x in cur.fetchone()) | |
| if _matrix_cache.get("stamp") == (tag,) + stamp: | |
| cur.close() | |
| return _matrix_cache["ids"], _matrix_cache["mat"] | |
| cur.execute("SELECT item_id, vec FROM " + vec_table() + " WHERE model_tag = %s", (tag,)) | |
| rows = cur.fetchall() | |
| cur.close() | |
| ids = [r[0] for r in rows] | |
| mat = np.stack([np.frombuffer(bytes(r[1]), np.float16) for r in rows]) if rows else np.zeros((0, 512), np.float16) | |
| _matrix_cache.update(stamp=(tag,) + stamp, ids=ids, mat=mat) | |
| return ids, mat | |
| def top_k(query_vec, tag, k, exclude=None): | |
| ids, mat = song_matrix(tag) | |
| if not ids: | |
| return [] | |
| scores = np.concatenate([mat[i:i + 50000].astype(np.float32) @ query_vec for i in range(0, len(ids), 50000)]) | |
| order = np.argsort(-scores) | |
| out = [] | |
| for j in order: | |
| if ids[j] != exclude: | |
| out.append((ids[j], float(scores[j]))) | |
| if len(out) >= k: | |
| break | |
| return out | |
| # ---------------------------------------------------------------- audio | |
| def load_wave(path): | |
| """Decode to 32 kHz mono with AudioMuse-AI's own loader (the same one its analysis uses).""" | |
| try: | |
| from tasks.analysis.song import robust_load_audio_with_fallback | |
| audio, sr = robust_load_audio_with_fallback(path, target_sr=vibemodel.SR) | |
| if audio is not None and sr == vibemodel.SR: | |
| return np.asarray(audio, np.float32) | |
| except Exception: | |
| logger.exception("vibe_search: AudioMuse-AI's audio loader failed; trying soundfile") | |
| import soundfile as sf | |
| y, sr = sf.read(path, dtype="float32", always_2d=True) | |
| y = y.mean(1) | |
| if sr != vibemodel.SR: | |
| import soxr | |
| y = soxr.resample(y, sr, vibemodel.SR, quality="HQ").astype(np.float32) | |
| return y | |
| def canonical_id(item_id, server_id=None): | |
| try: | |
| from tasks.mediaserver.registry import canonical_input_ids | |
| return canonical_input_ids([item_id], server_id).get(str(item_id), str(item_id)) | |
| except Exception: | |
| return str(item_id) | |
| def embed_file(path, item_id): | |
| m = get_model("audio") | |
| if m is None: | |
| return False | |
| t0 = time.time() | |
| vec, n = m.embed_audio(load_wave(path)) | |
| save_vector(item_id, m.tag, vec, n) | |
| logger.info(f"vibe_search: {item_id} embedded ({n} windows, {time.time() - t0:.1f} s)") | |
| return True | |
| def on_song_analyzed(song): | |
| """Runs on the worker right after AudioMuse-AI finishes a song; the audio file is deleted once this returns.""" | |
| if not setting("embed_on_analysis") or not song.get("audio_path"): | |
| return | |
| embed_file(song["audio_path"], canonical_id(song["item_id"], song.get("server_id"))) | |
| def index_library(): | |
| """Worker task: give a vibe vector to every song that does not have one yet (for songs analysed before install).""" | |
| from tasks.mediaserver import download_track, get_all_songs | |
| m = get_model("audio") | |
| if m is None: | |
| raise RuntimeError("the audio model could not be loaded: " + _model_errors.get("audio", "unknown error")) | |
| songs = get_all_songs() or [] | |
| done = indexed_ids(m.tag) | |
| todo = [s for s in songs if canonical_id(s.get("Id")) not in done] | |
| status = {"started": time.strftime("%Y-%m-%d %H:%M"), "total": len(todo), "done": 0, "failed": 0, "finished": False} | |
| set_setting("index_status", status) | |
| logger.info(f"vibe_search: indexing {len(todo)} of {len(songs)} songs") | |
| temp_dir = getattr(config, "TEMP_DIR", "/tmp") | |
| os.makedirs(temp_dir, exist_ok=True) | |
| for i, item in enumerate(todo): | |
| path = None | |
| try: | |
| path = download_track(temp_dir, item) | |
| if not path or not embed_file(path, canonical_id(item.get("Id"))): | |
| status["failed"] += 1 | |
| else: | |
| status["done"] += 1 | |
| except Exception: | |
| status["failed"] += 1 | |
| logger.exception(f"vibe_search: could not index {item.get('Id')}") | |
| finally: | |
| if path and os.path.exists(path): | |
| os.remove(path) | |
| if (i + 1) % 25 == 0: | |
| set_setting("index_status", status) | |
| status["finished"] = True | |
| set_setting("index_status", status) | |
| # ---------------------------------------------------------------- pages | |
| CSS = """<style> | |
| .vs { max-width: 980px; display: grid; gap: 16px; } | |
| .vs form.search { display: flex; gap: 8px; flex-wrap: wrap; } | |
| .vs input[type=text] { flex: 1 1 360px; padding: 8px 10px; font-size: 1rem; } | |
| .vs table { border-collapse: collapse; width: 100%; } | |
| .vs td, .vs th { padding: 6px 8px; border-bottom: 1px solid rgba(128,128,128,.25); text-align: left; vertical-align: top; } | |
| .vs .score { font-variant-numeric: tabular-nums; opacity: .7; } | |
| .vs .note { opacity: .75; font-size: .92rem; } | |
| .vs .err { color: #b3261e; } | |
| </style>""" | |
| def page(body, title="Vibe Search"): | |
| return render_page(CSS + '<div class="vs">' + body + "</div>", title=title) | |
| def status_html(): | |
| tag = current_tag() | |
| n_songs = n_vec = 0 | |
| try: | |
| cur = get_db().cursor() | |
| cur.execute("SELECT COUNT(*) FROM score") | |
| n_songs = cur.fetchone()[0] | |
| if tag: | |
| cur.execute("SELECT COUNT(*) FROM " + vec_table() + " WHERE model_tag = %s", (tag,)) | |
| n_vec = cur.fetchone()[0] | |
| cur.close() | |
| except Exception: | |
| logger.exception("vibe_search: status query failed") | |
| st = get_setting("index_status", None) | |
| msg = f"{n_vec:,} of {n_songs:,} analysed songs have a vibe vector." | |
| if st and not st.get("finished"): | |
| msg += f" Indexing: {st['done']:,} of {st['total']:,} done" + (f", {st['failed']} failed" if st["failed"] else "") + "." | |
| err = "".join(f'<p class="err">The {k} model is not available: {html.escape(v)}</p>' for k, v in _model_errors.items()) | |
| notes = "".join(f'<p class="note">{html.escape(v)}</p>' for v in _model_notes.values()) | |
| return f'<p class="note">{msg}</p>{notes}{err}' | |
| def results_html(rows, heading, playlist_name): | |
| if not rows: | |
| return "<p>No songs with a vibe vector yet. Analyse your library, or use Index my library below.</p>" | |
| info = {r["item_id"]: r for r in (get_score_data_by_ids([i for i, _ in rows]) or [])} | |
| trs = [] | |
| for item_id, score in rows: | |
| r = info.get(item_id, {}) | |
| trs.append( | |
| f'<tr><td><input type="checkbox" name="ids" value="{html.escape(item_id)}" checked></td>' | |
| f'<td>{html.escape(str(r.get("title") or item_id))}</td><td>{html.escape(str(r.get("author") or ""))}</td>' | |
| f'<td>{html.escape(str(r.get("album") or ""))}</td><td class="score">{score:.3f}</td>' | |
| f'<td><a href="{url_for("vibe_search.similar", item_id=item_id)}">same vibe</a></td></tr>') | |
| return (f"<h3>{html.escape(heading)}</h3>" | |
| f'<form method="post" action="{url_for("vibe_search.playlist")}">' | |
| f"<table><tr><th></th><th>Title</th><th>Artist</th><th>Album</th><th>Match</th><th></th></tr>{''.join(trs)}</table>" | |
| f'<p><input type="text" name="name" value="{html.escape(playlist_name)}" aria-label="Playlist name"> ' | |
| f'<button type="submit">Save ticked songs as a playlist</button></p></form>') | |
| def search_form(q=""): | |
| return (f'<form class="search" method="get" action="{url_for("vibe_search.home")}">' | |
| f'<input type="text" name="q" value="{html.escape(q)}" placeholder="late night drive, rainy Sunday, upbeat but melancholic..." ' | |
| f'aria-label="Describe the music"><button type="submit">Search</button></form>') | |
| def home(): | |
| q = (request.args.get("q") or "").strip() | |
| body = "<h2>Vibe Search</h2>" + search_form(q) + status_html() | |
| if q: | |
| m = get_model("text") | |
| if m is None: | |
| body += '<p class="err">The search model could not be loaded; see the message above.</p>' | |
| else: | |
| tag = current_tag() or m.tag | |
| rows = top_k(m.embed_texts([q])[0], tag, int(setting("result_limit"))) | |
| body += results_html(rows, f"Songs that match “{q}”", f"Vibe: {q[:60]}") | |
| body += (f'<form method="post" action="{url_for("vibe_search.index")}"><button type="submit">Index my library</button> ' | |
| '<span class="note">Gives a vibe vector to analysed songs that do not have one yet (runs in the background).</span></form>') | |
| return page(body) | |
| def similar(item_id): | |
| tag = current_tag() | |
| ids, mat = song_matrix(tag) if tag else ([], None) | |
| body = "<h2>Vibe Search</h2>" + search_form() + status_html() | |
| if item_id not in ids: | |
| body += "<p>This song has no vibe vector yet.</p>" | |
| else: | |
| seed = mat[ids.index(item_id)].astype(np.float32) | |
| r = (get_score_data_by_ids([item_id]) or [{}])[0] | |
| name = f'{r.get("title") or item_id}' + (f' by {r.get("author")}' if r.get("author") else "") | |
| body += results_html(top_k(seed, tag, int(setting("result_limit")), exclude=item_id), | |
| f"Same vibe as {name}", f"Same vibe: {name[:60]}") | |
| return page(body) | |
| def playlist(): | |
| ids = request.form.getlist("ids") | |
| name = (request.form.get("name") or "Vibe Search").strip()[:120] | |
| if not ids: | |
| return page("<h2>Vibe Search</h2><p>No songs were ticked.</p>" + search_form()) | |
| try: | |
| from tasks.mediaserver import create_or_replace_playlist | |
| create_or_replace_playlist(name, ids) | |
| msg = f"<p>Saved {len(ids)} songs as the playlist “{html.escape(name)}” on your media server.</p>" | |
| except Exception as e: | |
| logger.exception("vibe_search: playlist creation failed") | |
| msg = f'<p class="err">The playlist could not be created: {html.escape(str(e))}</p>' | |
| return page("<h2>Vibe Search</h2>" + msg + search_form()) | |
| def index(): | |
| try: | |
| enqueue(index_library) | |
| msg = "<p>Indexing started in the background. This page shows the progress.</p>" | |
| except Exception as e: | |
| msg = f'<p class="err">Indexing could not start now: {html.escape(str(e))}</p>' | |
| return page("<h2>Vibe Search</h2>" + msg + search_form() + status_html()) | |
| def settings(): | |
| if request.method == "POST": | |
| for k, v in DEFAULTS.items(): | |
| if k not in request.form and not isinstance(v, bool): | |
| continue | |
| raw = request.form.get(k) | |
| val = (raw == "on") if isinstance(v, bool) else (int(raw) if isinstance(v, int) else raw.strip()) | |
| set_setting(k, val) | |
| _models.clear(); _model_errors.clear(); _model_notes.clear(); _matrix_cache.clear() | |
| return redirect(manage_plugins_url()) | |
| rows = [] | |
| for k, v in DEFAULTS.items(): | |
| cur = setting(k) | |
| if isinstance(v, bool): | |
| field = f'<input type="checkbox" name="{k}" id="{k}"{" checked" if cur else ""}>' | |
| else: | |
| field = f'<input type="text" name="{k}" id="{k}" value="{html.escape(str(cur))}">' | |
| rows.append(f'<tr><td><label for="{k}">{k}</label></td><td>{field}</td></tr>') | |
| note = ("<p class=\"note\">precision: int8 (smaller, faster) or fp32. With int8, a model that cannot run on this " | |
| "installation (for example the 8-bit search model on onnxruntime older than 1.28) falls back to fp32 by itself. " | |
| "Changing the model repository or revision makes stored vectors outdated: run " | |
| "Index my library afterwards. Models are stored in " + html.escape(model_dir()) + ".</p>") | |
| return page(f'<h2>Vibe Search settings</h2><form method="post"><table>{"".join(rows)}</table>' | |
| f'<button type="submit">Save</button></form>{note}', title="Vibe Search settings") | |
| def register(ctx): | |
| ctx.on_install(migrate) | |
| ctx.add_blueprint(bp) | |
| ctx.add_menu_item("Vibe Search", "vibe_search.home") | |
| ctx.on_song_analyzed(on_song_analyzed) | |
| ctx.add_task("index_library", index_library) | |