import json import numpy as np import faiss from sentence_transformers import SentenceTransformer, CrossEncoder import datetime import sqlite3 import gc import re from typing import List, Dict, Any, Optional, Tuple BI_ENCODER_MODEL = None CROSS_ENCODER_MODEL = None from functools import lru_cache MEMORY_DB_CACHE = {} import os BASE_DIR = "/data" if os.path.exists("/data") else "." SHARDS = [ {"name": "2013_2015", "index_path": os.path.join(BASE_DIR, "faiss_index_2013_2015.bin"), "db_path": os.path.join(BASE_DIR, "metadata_2013_2015.db"), "start": 2013, "end": 2015}, {"name": "2016_2018", "index_path": os.path.join(BASE_DIR, "faiss_index_2016_2018.bin"), "db_path": os.path.join(BASE_DIR, "metadata_2016_2018.db"), "start": 2016, "end": 2018}, # TODO: Add older years later? 2010-2012 might be cool. ] def load_models(device: str = "cpu") -> Tuple[Any, Any]: global BI_ENCODER_MODEL, CROSS_ENCODER_MODEL if BI_ENCODER_MODEL is None: BI_ENCODER_MODEL = SentenceTransformer("all-MiniLM-L6-v2", device=device) if CROSS_ENCODER_MODEL is None: CROSS_ENCODER_MODEL = CrossEncoder("cross-encoder/ms-marco-MiniLM-L-6-v2", device=device) return BI_ENCODER_MODEL, CROSS_ENCODER_MODEL def load_shard_index(shard): idx = faiss.read_index(shard["index_path"], faiss.IO_FLAG_MMAP) return idx def load_db_to_memory(db_path: str): if db_path in MEMORY_DB_CACHE: return MEMORY_DB_CACHE[db_path] if not os.path.exists(db_path): return None print(f"[Cache] Loading {os.path.basename(db_path)} into memory...", flush=True) disk_conn = sqlite3.connect(db_path) mem_conn = sqlite3.connect(':memory:') disk_conn.backup(mem_conn) disk_conn.close() MEMORY_DB_CACHE[db_path] = mem_conn print(f"[Cache] ✅ {os.path.basename(db_path)} cached in memory", flush=True) return mem_conn @lru_cache(maxsize=2048) def fetch_single_track(db_path: str, track_id: int) -> Optional[Dict]: conn = load_db_to_memory(db_path) if not conn: return None try: c = conn.cursor() c.execute("SELECT content FROM tracks WHERE id = ?", (track_id,)) row = c.fetchone() if row: return json.loads(row[0]) except: pass return None def fetch_metadata(shard: Dict, indices: List[int]) -> List[Dict]: if not indices: return [] db_path = shard["db_path"] conn = load_db_to_memory(db_path) if not conn: conn = sqlite3.connect(db_path) should_close = True else: should_close = False try: c = conn.cursor() query = f"SELECT content FROM tracks WHERE id IN ({','.join(['?']*len(indices))})" c.execute(query, indices) rows = c.fetchall() results = [] for r in rows: try: results.append(json.loads(r[0])) except: continue return results finally: if should_close: conn.close() def pick_shards(target_year: Optional[int] = None, start_year: Optional[int] = None, end_year: Optional[int] = None) -> List[Dict]: if start_year is not None and end_year is not None: picked = [s for s in SHARDS if not (s["end"] < start_year or s["start"] > end_year)] if picked: return picked if target_year is not None: picked = [s for s in SHARDS if s["start"] <= target_year <= s["end"]] if picked: return picked return SHARDS def dedup_diversity(results: List[Dict], penalty: float = 0.2) -> List[Dict]: # This function is kinda messy but it stops the spammy results. # TODO: refine the penalty logic, 0.2 is just a guess. seen = set() uniq = [] seen = set() uniq = [] # Pre-compiled regex for normalization NORM_PATTERN = re.compile(r'[^a-z0-9]') BRACKET_PATTERN = re.compile(r'[\[\(].*?[\]\)]') REMOVE_WORDS = ["official", "video", "audio", "lyrics", "feat", "ft", "prod", "hd", "hq", "remastered"] def normalize(text: str) -> str: if not text: return "" text = text.lower() for word in REMOVE_WORDS: text = text.replace(word, "") text = BRACKET_PATTERN.sub('', text) return NORM_PATTERN.sub('', text) for r in results: url = r.get("permalink_url") if url and url in seen: continue title_norm = normalize(r.get("title", "")) artist_norm = normalize(r.get("artist", "")) content_key = f"{artist_norm}_{title_norm}" if len(content_key) < 3: if url: seen.add(url) uniq.append(r) continue if content_key in seen: continue if url: seen.add(url) seen.add(content_key) uniq.append(r) return uniq def score_results(rows: List[Dict], query_text: str, target_year: Optional[int], alpha: float = 0.5, cross_encoder: Any = None) -> List[Dict]: W_CROSS = 0.6 # really important for semantic stuff W_BI = 0.1 # lowered this cause it was giving too many generic matches W_TIME = 0.1 W_POP = 0.2 # boost popular tracks a bit so we don't get total randoms query_tokens = set(query_text.lower().split()) is_long_query = len(query_tokens) > 5 pairs = [[query_text, f"{r['title']} {r['artist']} {r.get('genre','')} {r.get('tags','')}"] for r in rows] if cross_encoder and pairs: cross_scores = cross_encoder.predict(pairs) else: cross_scores = [0.0] * len(rows) scored = [] for i, r in enumerate(rows): bi_score = float(r.get("score", 0.0)) cross_score = float(cross_scores[i]) year = r.get("year") time_score = 0.0 if target_year is not None and year is not None: diff = abs(year - target_year) time_score = 1.0 / (1.0 + alpha * diff) pop = float(r.get("playback_count") or 0.0) pop_score = np.log1p(pop) pop_score = pop_score / 15.0 artist_lower = r.get("artist", "").lower() tags_lower = (r.get("tags", "") + " " + r.get("genre", "")).lower() boost = 0.0 if not is_long_query: for token in query_tokens: if len(token) > 2 and token in tags_lower: boost += 3.0 total = (W_CROSS * cross_score) + (W_BI * bi_score) + (W_TIME * time_score) + (W_POP * pop_score) + boost r["final_score"] = total r["components"] = { "cross": cross_score, "bi": bi_score, "time": time_score, "pop": pop_score, "boost": boost } scored.append(r) scored.sort(key=lambda x: x["final_score"], reverse=True) return scored def search(query_text: str, target_year: Optional[int] = None, top_k: int = 10, device: str = "cpu", batch_size: int = 1024, start_year: Optional[int] = None, end_year: Optional[int] = None, alpha: float = 0.5) -> List[Dict]: gc.collect() if target_year is None and start_year is not None and end_year is not None: target_year = int((start_year + end_year) / 2) if target_year is None: target_year = 2015 bi_model, cross_model = load_models(device) MAX_LEN = 256 if len(query_text) > MAX_LEN: query_text = query_text[:MAX_LEN] q_emb = bi_model.encode([query_text], convert_to_numpy=True, normalize_embeddings=True, device=device, batch_size=batch_size, show_progress_bar=False) # 30 candidates seems enough for re-ranking. cpu can handle it. CANDIDATE_K = 30 shards = pick_shards(target_year=target_year, start_year=start_year, end_year=end_year) candidates = [] for s in shards: idx = load_shard_index(s) scores, idxs = idx.search(q_emb.astype(np.float32), CANDIDATE_K) valid_idxs = [] idx_score_map = {} for score, i in zip(scores[0], idxs[0]): if i < 0: continue valid_idxs.append(int(i)) idx_score_map[int(i)] = float(score) if not valid_idxs: del idx continue conn = sqlite3.connect(s["db_path"]) c = conn.cursor() query = f"SELECT id, content FROM tracks WHERE id IN ({','.join(['?']*len(valid_idxs))})" c.execute(query, valid_idxs) rows = c.fetchall() conn.close() for r_id, content in rows: try: m = json.loads(content) y = m.get("year") if start_year is not None and end_year is not None: if y < start_year or y > end_year: continue m["score"] = idx_score_map.get(r_id, 0.0) candidates.append(m) except: continue del idx gc.collect() gc.collect() candidates.sort(key=lambda x: x["score"], reverse=True) candidates = dedup_diversity(candidates) candidates.sort(key=lambda x: x["score"], reverse=True) candidates = candidates[:30] ranked_results = score_results(candidates, query_text, target_year=target_year, alpha=alpha, cross_encoder=cross_model) return ranked_results[:top_k] if __name__ == "__main__": while True: try: q_input = input("\nEnter query (or 'q' to quit): ").strip() if q_input.lower() == 'q': break y_input = input("Target Year (2013-2018) [Default 2015]: ").strip() target = int(y_input) if y_input.isdigit() else 2015 print(f"\nScanning for '{q_input}' around {target}...") results = search( q_input, target_year=target, top_k=5, device="cpu", batch_size=1024 ) if not results: print("No artifacts found.") for i, r in enumerate(results, 1): print(f"\n{i}. {r['title']} | {r['artist']} ({r['year']})") print(f" Score: {r['final_score']:.3f} | Popularity: {r.get('playback_count',0)}") print(f" URL: {r.get('permalink_url')}") except KeyboardInterrupt: print("\nExiting...") break except Exception as e: print(f"Error: {e}")