Download main.py from Yabuku/Capstone: direct link, hf CLI and curl.
- Browser
- Download file 10.5 kB
-
https://huggingface.co/spaces/Yabuku/Capstone/resolve/main/main.py
- Command line
-
hf download hf://spaces/Yabuku/Capstone/main.py
-
curl -L -o main.py https://huggingface.co/spaces/Yabuku/Capstone/resolve/main/main.py
10.5 kB
| import json | |
| import numpy as np | |
| import faiss | |
| from sentence_transformers import SentenceTransformer, CrossEncoder | |
| import datetime | |
| import sqlite3 | |
| import gc | |
| import re | |
| from typing import List, Dict, Any, Optional, Tuple | |
| BI_ENCODER_MODEL = None | |
| CROSS_ENCODER_MODEL = None | |
| from functools import lru_cache | |
| MEMORY_DB_CACHE = {} | |
| import os | |
| BASE_DIR = "/data" if os.path.exists("/data") else "." | |
| SHARDS = [ | |
| {"name": "2013_2015", "index_path": os.path.join(BASE_DIR, "faiss_index_2013_2015.bin"), "db_path": os.path.join(BASE_DIR, "metadata_2013_2015.db"), "start": 2013, "end": 2015}, | |
| {"name": "2016_2018", "index_path": os.path.join(BASE_DIR, "faiss_index_2016_2018.bin"), "db_path": os.path.join(BASE_DIR, "metadata_2016_2018.db"), "start": 2016, "end": 2018}, | |
| # TODO: Add older years later? 2010-2012 might be cool. | |
| ] | |
| def load_models(device: str = "cpu") -> Tuple[Any, Any]: | |
| global BI_ENCODER_MODEL, CROSS_ENCODER_MODEL | |
| if BI_ENCODER_MODEL is None: | |
| BI_ENCODER_MODEL = SentenceTransformer("all-MiniLM-L6-v2", device=device) | |
| if CROSS_ENCODER_MODEL is None: | |
| CROSS_ENCODER_MODEL = CrossEncoder("cross-encoder/ms-marco-MiniLM-L-6-v2", device=device) | |
| return BI_ENCODER_MODEL, CROSS_ENCODER_MODEL | |
| def load_shard_index(shard): | |
| idx = faiss.read_index(shard["index_path"], faiss.IO_FLAG_MMAP) | |
| return idx | |
| def load_db_to_memory(db_path: str): | |
| if db_path in MEMORY_DB_CACHE: | |
| return MEMORY_DB_CACHE[db_path] | |
| if not os.path.exists(db_path): | |
| return None | |
| print(f"[Cache] Loading {os.path.basename(db_path)} into memory...", flush=True) | |
| disk_conn = sqlite3.connect(db_path) | |
| mem_conn = sqlite3.connect(':memory:') | |
| disk_conn.backup(mem_conn) | |
| disk_conn.close() | |
| MEMORY_DB_CACHE[db_path] = mem_conn | |
| print(f"[Cache] ✅ {os.path.basename(db_path)} cached in memory", flush=True) | |
| return mem_conn | |
| def fetch_single_track(db_path: str, track_id: int) -> Optional[Dict]: | |
| conn = load_db_to_memory(db_path) | |
| if not conn: | |
| return None | |
| try: | |
| c = conn.cursor() | |
| c.execute("SELECT content FROM tracks WHERE id = ?", (track_id,)) | |
| row = c.fetchone() | |
| if row: | |
| return json.loads(row[0]) | |
| except: | |
| pass | |
| return None | |
| def fetch_metadata(shard: Dict, indices: List[int]) -> List[Dict]: | |
| if not indices: return [] | |
| db_path = shard["db_path"] | |
| conn = load_db_to_memory(db_path) | |
| if not conn: | |
| conn = sqlite3.connect(db_path) | |
| should_close = True | |
| else: | |
| should_close = False | |
| try: | |
| c = conn.cursor() | |
| query = f"SELECT content FROM tracks WHERE id IN ({','.join(['?']*len(indices))})" | |
| c.execute(query, indices) | |
| rows = c.fetchall() | |
| results = [] | |
| for r in rows: | |
| try: | |
| results.append(json.loads(r[0])) | |
| except: continue | |
| return results | |
| finally: | |
| if should_close: | |
| conn.close() | |
| def pick_shards(target_year: Optional[int] = None, start_year: Optional[int] = None, end_year: Optional[int] = None) -> List[Dict]: | |
| if start_year is not None and end_year is not None: | |
| picked = [s for s in SHARDS if not (s["end"] < start_year or s["start"] > end_year)] | |
| if picked: | |
| return picked | |
| if target_year is not None: | |
| picked = [s for s in SHARDS if s["start"] <= target_year <= s["end"]] | |
| if picked: | |
| return picked | |
| return SHARDS | |
| def dedup_diversity(results: List[Dict], penalty: float = 0.2) -> List[Dict]: | |
| # This function is kinda messy but it stops the spammy results. | |
| # TODO: refine the penalty logic, 0.2 is just a guess. | |
| seen = set() | |
| uniq = [] | |
| seen = set() | |
| uniq = [] | |
| # Pre-compiled regex for normalization | |
| NORM_PATTERN = re.compile(r'[^a-z0-9]') | |
| BRACKET_PATTERN = re.compile(r'[\[\(].*?[\]\)]') | |
| REMOVE_WORDS = ["official", "video", "audio", "lyrics", "feat", "ft", "prod", "hd", "hq", "remastered"] | |
| def normalize(text: str) -> str: | |
| if not text: return "" | |
| text = text.lower() | |
| for word in REMOVE_WORDS: | |
| text = text.replace(word, "") | |
| text = BRACKET_PATTERN.sub('', text) | |
| return NORM_PATTERN.sub('', text) | |
| for r in results: | |
| url = r.get("permalink_url") | |
| if url and url in seen: | |
| continue | |
| title_norm = normalize(r.get("title", "")) | |
| artist_norm = normalize(r.get("artist", "")) | |
| content_key = f"{artist_norm}_{title_norm}" | |
| if len(content_key) < 3: | |
| if url: seen.add(url) | |
| uniq.append(r) | |
| continue | |
| if content_key in seen: | |
| continue | |
| if url: seen.add(url) | |
| seen.add(content_key) | |
| uniq.append(r) | |
| return uniq | |
| def score_results(rows: List[Dict], query_text: str, target_year: Optional[int], alpha: float = 0.5, cross_encoder: Any = None) -> List[Dict]: | |
| W_CROSS = 0.6 # really important for semantic stuff | |
| W_BI = 0.1 # lowered this cause it was giving too many generic matches | |
| W_TIME = 0.1 | |
| W_POP = 0.2 # boost popular tracks a bit so we don't get total randoms | |
| query_tokens = set(query_text.lower().split()) | |
| is_long_query = len(query_tokens) > 5 | |
| pairs = [[query_text, f"{r['title']} {r['artist']} {r.get('genre','')} {r.get('tags','')}"] for r in rows] | |
| if cross_encoder and pairs: | |
| cross_scores = cross_encoder.predict(pairs) | |
| else: | |
| cross_scores = [0.0] * len(rows) | |
| scored = [] | |
| for i, r in enumerate(rows): | |
| bi_score = float(r.get("score", 0.0)) | |
| cross_score = float(cross_scores[i]) | |
| year = r.get("year") | |
| time_score = 0.0 | |
| if target_year is not None and year is not None: | |
| diff = abs(year - target_year) | |
| time_score = 1.0 / (1.0 + alpha * diff) | |
| pop = float(r.get("playback_count") or 0.0) | |
| pop_score = np.log1p(pop) | |
| pop_score = pop_score / 15.0 | |
| artist_lower = r.get("artist", "").lower() | |
| tags_lower = (r.get("tags", "") + " " + r.get("genre", "")).lower() | |
| boost = 0.0 | |
| if not is_long_query: | |
| for token in query_tokens: | |
| if len(token) > 2 and token in tags_lower: | |
| boost += 3.0 | |
| total = (W_CROSS * cross_score) + (W_BI * bi_score) + (W_TIME * time_score) + (W_POP * pop_score) + boost | |
| r["final_score"] = total | |
| r["components"] = { | |
| "cross": cross_score, | |
| "bi": bi_score, | |
| "time": time_score, | |
| "pop": pop_score, | |
| "boost": boost | |
| } | |
| scored.append(r) | |
| scored.sort(key=lambda x: x["final_score"], reverse=True) | |
| return scored | |
| def search(query_text: str, target_year: Optional[int] = None, top_k: int = 10, device: str = "cpu", batch_size: int = 1024, start_year: Optional[int] = None, end_year: Optional[int] = None, alpha: float = 0.5) -> List[Dict]: | |
| gc.collect() | |
| if target_year is None and start_year is not None and end_year is not None: | |
| target_year = int((start_year + end_year) / 2) | |
| if target_year is None: target_year = 2015 | |
| bi_model, cross_model = load_models(device) | |
| MAX_LEN = 256 | |
| if len(query_text) > MAX_LEN: | |
| query_text = query_text[:MAX_LEN] | |
| q_emb = bi_model.encode([query_text], convert_to_numpy=True, normalize_embeddings=True, device=device, batch_size=batch_size, show_progress_bar=False) | |
| # 30 candidates seems enough for re-ranking. cpu can handle it. | |
| CANDIDATE_K = 30 | |
| shards = pick_shards(target_year=target_year, start_year=start_year, end_year=end_year) | |
| candidates = [] | |
| for s in shards: | |
| idx = load_shard_index(s) | |
| scores, idxs = idx.search(q_emb.astype(np.float32), CANDIDATE_K) | |
| valid_idxs = [] | |
| idx_score_map = {} | |
| for score, i in zip(scores[0], idxs[0]): | |
| if i < 0: continue | |
| valid_idxs.append(int(i)) | |
| idx_score_map[int(i)] = float(score) | |
| if not valid_idxs: | |
| del idx | |
| continue | |
| conn = sqlite3.connect(s["db_path"]) | |
| c = conn.cursor() | |
| query = f"SELECT id, content FROM tracks WHERE id IN ({','.join(['?']*len(valid_idxs))})" | |
| c.execute(query, valid_idxs) | |
| rows = c.fetchall() | |
| conn.close() | |
| for r_id, content in rows: | |
| try: | |
| m = json.loads(content) | |
| y = m.get("year") | |
| if start_year is not None and end_year is not None: | |
| if y < start_year or y > end_year: | |
| continue | |
| m["score"] = idx_score_map.get(r_id, 0.0) | |
| candidates.append(m) | |
| except: continue | |
| del idx | |
| gc.collect() | |
| gc.collect() | |
| candidates.sort(key=lambda x: x["score"], reverse=True) | |
| candidates = dedup_diversity(candidates) | |
| candidates.sort(key=lambda x: x["score"], reverse=True) | |
| candidates = candidates[:30] | |
| ranked_results = score_results(candidates, query_text, target_year=target_year, alpha=alpha, cross_encoder=cross_model) | |
| return ranked_results[:top_k] | |
| if __name__ == "__main__": | |
| while True: | |
| try: | |
| q_input = input("\nEnter query (or 'q' to quit): ").strip() | |
| if q_input.lower() == 'q': | |
| break | |
| y_input = input("Target Year (2013-2018) [Default 2015]: ").strip() | |
| target = int(y_input) if y_input.isdigit() else 2015 | |
| print(f"\nScanning for '{q_input}' around {target}...") | |
| results = search( | |
| q_input, | |
| target_year=target, | |
| top_k=5, | |
| device="cpu", | |
| batch_size=1024 | |
| ) | |
| if not results: | |
| print("No artifacts found.") | |
| for i, r in enumerate(results, 1): | |
| print(f"\n{i}. {r['title']} | {r['artist']} ({r['year']})") | |
| print(f" Score: {r['final_score']:.3f} | Popularity: {r.get('playback_count',0)}") | |
| print(f" URL: {r.get('permalink_url')}") | |
| except KeyboardInterrupt: | |
| print("\nExiting...") | |
| break | |
| except Exception as e: | |
| print(f"Error: {e}") |