Capstone / main.py
Yabuku's picture
Add database optimizations with in-memory caching and background warmup
c087a1e
Raw History Blame Contribute Delete
10.5 kB
import json
import numpy as np
import faiss
from sentence_transformers import SentenceTransformer, CrossEncoder
import datetime
import sqlite3
import gc
import re
from typing import List, Dict, Any, Optional, Tuple
BI_ENCODER_MODEL = None
CROSS_ENCODER_MODEL = None
from functools import lru_cache
MEMORY_DB_CACHE = {}
import os
BASE_DIR = "/data" if os.path.exists("/data") else "."
SHARDS = [
{"name": "2013_2015", "index_path": os.path.join(BASE_DIR, "faiss_index_2013_2015.bin"), "db_path": os.path.join(BASE_DIR, "metadata_2013_2015.db"), "start": 2013, "end": 2015},
{"name": "2016_2018", "index_path": os.path.join(BASE_DIR, "faiss_index_2016_2018.bin"), "db_path": os.path.join(BASE_DIR, "metadata_2016_2018.db"), "start": 2016, "end": 2018},
# TODO: Add older years later? 2010-2012 might be cool.
]
def load_models(device: str = "cpu") -> Tuple[Any, Any]:
global BI_ENCODER_MODEL, CROSS_ENCODER_MODEL
if BI_ENCODER_MODEL is None:
BI_ENCODER_MODEL = SentenceTransformer("all-MiniLM-L6-v2", device=device)
if CROSS_ENCODER_MODEL is None:
CROSS_ENCODER_MODEL = CrossEncoder("cross-encoder/ms-marco-MiniLM-L-6-v2", device=device)
return BI_ENCODER_MODEL, CROSS_ENCODER_MODEL
def load_shard_index(shard):
idx = faiss.read_index(shard["index_path"], faiss.IO_FLAG_MMAP)
return idx
def load_db_to_memory(db_path: str):
if db_path in MEMORY_DB_CACHE:
return MEMORY_DB_CACHE[db_path]
if not os.path.exists(db_path):
return None
print(f"[Cache] Loading {os.path.basename(db_path)} into memory...", flush=True)
disk_conn = sqlite3.connect(db_path)
mem_conn = sqlite3.connect(':memory:')
disk_conn.backup(mem_conn)
disk_conn.close()
MEMORY_DB_CACHE[db_path] = mem_conn
print(f"[Cache] ✅ {os.path.basename(db_path)} cached in memory", flush=True)
return mem_conn
@lru_cache(maxsize=2048)
def fetch_single_track(db_path: str, track_id: int) -> Optional[Dict]:
conn = load_db_to_memory(db_path)
if not conn:
return None
try:
c = conn.cursor()
c.execute("SELECT content FROM tracks WHERE id = ?", (track_id,))
row = c.fetchone()
if row:
return json.loads(row[0])
except:
pass
return None
def fetch_metadata(shard: Dict, indices: List[int]) -> List[Dict]:
if not indices: return []
db_path = shard["db_path"]
conn = load_db_to_memory(db_path)
if not conn:
conn = sqlite3.connect(db_path)
should_close = True
else:
should_close = False
try:
c = conn.cursor()
query = f"SELECT content FROM tracks WHERE id IN ({','.join(['?']*len(indices))})"
c.execute(query, indices)
rows = c.fetchall()
results = []
for r in rows:
try:
results.append(json.loads(r[0]))
except: continue
return results
finally:
if should_close:
conn.close()
def pick_shards(target_year: Optional[int] = None, start_year: Optional[int] = None, end_year: Optional[int] = None) -> List[Dict]:
if start_year is not None and end_year is not None:
picked = [s for s in SHARDS if not (s["end"] < start_year or s["start"] > end_year)]
if picked:
return picked
if target_year is not None:
picked = [s for s in SHARDS if s["start"] <= target_year <= s["end"]]
if picked:
return picked
return SHARDS
def dedup_diversity(results: List[Dict], penalty: float = 0.2) -> List[Dict]:
# This function is kinda messy but it stops the spammy results.
# TODO: refine the penalty logic, 0.2 is just a guess.
seen = set()
uniq = []
seen = set()
uniq = []
# Pre-compiled regex for normalization
NORM_PATTERN = re.compile(r'[^a-z0-9]')
BRACKET_PATTERN = re.compile(r'[\[\(].*?[\]\)]')
REMOVE_WORDS = ["official", "video", "audio", "lyrics", "feat", "ft", "prod", "hd", "hq", "remastered"]
def normalize(text: str) -> str:
if not text: return ""
text = text.lower()
for word in REMOVE_WORDS:
text = text.replace(word, "")
text = BRACKET_PATTERN.sub('', text)
return NORM_PATTERN.sub('', text)
for r in results:
url = r.get("permalink_url")
if url and url in seen:
continue
title_norm = normalize(r.get("title", ""))
artist_norm = normalize(r.get("artist", ""))
content_key = f"{artist_norm}_{title_norm}"
if len(content_key) < 3:
if url: seen.add(url)
uniq.append(r)
continue
if content_key in seen:
continue
if url: seen.add(url)
seen.add(content_key)
uniq.append(r)
return uniq
def score_results(rows: List[Dict], query_text: str, target_year: Optional[int], alpha: float = 0.5, cross_encoder: Any = None) -> List[Dict]:
W_CROSS = 0.6 # really important for semantic stuff
W_BI = 0.1 # lowered this cause it was giving too many generic matches
W_TIME = 0.1
W_POP = 0.2 # boost popular tracks a bit so we don't get total randoms
query_tokens = set(query_text.lower().split())
is_long_query = len(query_tokens) > 5
pairs = [[query_text, f"{r['title']} {r['artist']} {r.get('genre','')} {r.get('tags','')}"] for r in rows]
if cross_encoder and pairs:
cross_scores = cross_encoder.predict(pairs)
else:
cross_scores = [0.0] * len(rows)
scored = []
for i, r in enumerate(rows):
bi_score = float(r.get("score", 0.0))
cross_score = float(cross_scores[i])
year = r.get("year")
time_score = 0.0
if target_year is not None and year is not None:
diff = abs(year - target_year)
time_score = 1.0 / (1.0 + alpha * diff)
pop = float(r.get("playback_count") or 0.0)
pop_score = np.log1p(pop)
pop_score = pop_score / 15.0
artist_lower = r.get("artist", "").lower()
tags_lower = (r.get("tags", "") + " " + r.get("genre", "")).lower()
boost = 0.0
if not is_long_query:
for token in query_tokens:
if len(token) > 2 and token in tags_lower:
boost += 3.0
total = (W_CROSS * cross_score) + (W_BI * bi_score) + (W_TIME * time_score) + (W_POP * pop_score) + boost
r["final_score"] = total
r["components"] = {
"cross": cross_score,
"bi": bi_score,
"time": time_score,
"pop": pop_score,
"boost": boost
}
scored.append(r)
scored.sort(key=lambda x: x["final_score"], reverse=True)
return scored
def search(query_text: str, target_year: Optional[int] = None, top_k: int = 10, device: str = "cpu", batch_size: int = 1024, start_year: Optional[int] = None, end_year: Optional[int] = None, alpha: float = 0.5) -> List[Dict]:
gc.collect()
if target_year is None and start_year is not None and end_year is not None:
target_year = int((start_year + end_year) / 2)
if target_year is None: target_year = 2015
bi_model, cross_model = load_models(device)
MAX_LEN = 256
if len(query_text) > MAX_LEN:
query_text = query_text[:MAX_LEN]
q_emb = bi_model.encode([query_text], convert_to_numpy=True, normalize_embeddings=True, device=device, batch_size=batch_size, show_progress_bar=False)
# 30 candidates seems enough for re-ranking. cpu can handle it.
CANDIDATE_K = 30
shards = pick_shards(target_year=target_year, start_year=start_year, end_year=end_year)
candidates = []
for s in shards:
idx = load_shard_index(s)
scores, idxs = idx.search(q_emb.astype(np.float32), CANDIDATE_K)
valid_idxs = []
idx_score_map = {}
for score, i in zip(scores[0], idxs[0]):
if i < 0: continue
valid_idxs.append(int(i))
idx_score_map[int(i)] = float(score)
if not valid_idxs:
del idx
continue
conn = sqlite3.connect(s["db_path"])
c = conn.cursor()
query = f"SELECT id, content FROM tracks WHERE id IN ({','.join(['?']*len(valid_idxs))})"
c.execute(query, valid_idxs)
rows = c.fetchall()
conn.close()
for r_id, content in rows:
try:
m = json.loads(content)
y = m.get("year")
if start_year is not None and end_year is not None:
if y < start_year or y > end_year:
continue
m["score"] = idx_score_map.get(r_id, 0.0)
candidates.append(m)
except: continue
del idx
gc.collect()
gc.collect()
candidates.sort(key=lambda x: x["score"], reverse=True)
candidates = dedup_diversity(candidates)
candidates.sort(key=lambda x: x["score"], reverse=True)
candidates = candidates[:30]
ranked_results = score_results(candidates, query_text, target_year=target_year, alpha=alpha, cross_encoder=cross_model)
return ranked_results[:top_k]
if __name__ == "__main__":
while True:
try:
q_input = input("\nEnter query (or 'q' to quit): ").strip()
if q_input.lower() == 'q':
break
y_input = input("Target Year (2013-2018) [Default 2015]: ").strip()
target = int(y_input) if y_input.isdigit() else 2015
print(f"\nScanning for '{q_input}' around {target}...")
results = search(
q_input,
target_year=target,
top_k=5,
device="cpu",
batch_size=1024
)
if not results:
print("No artifacts found.")
for i, r in enumerate(results, 1):
print(f"\n{i}. {r['title']} | {r['artist']} ({r['year']})")
print(f" Score: {r['final_score']:.3f} | Popularity: {r.get('playback_count',0)}")
print(f" URL: {r.get('permalink_url')}")
except KeyboardInterrupt:
print("\nExiting...")
break
except Exception as e:
print(f"Error: {e}")