from flask import Flask, render_template, request, jsonify from main import search import concurrent.futures import requests import os from huggingface_hub import hf_hub_download import threading import time # Use a global session for connection reuse (Keep-Alive) session = requests.Session() adapter = requests.adapters.HTTPAdapter(pool_connections=20, pool_maxsize=20) session.mount('https://', adapter) session.mount('http://', adapter) app = Flask(__name__) # --- ERROR HANDLERS --- @app.errorhandler(500) def internal_error(e): return jsonify({"error": f"Internal Server Error: {str(e)}"}), 500 @app.errorhandler(404) def not_found(e): return jsonify({"error": "Resource not found (404)"}), 404 @app.errorhandler(Exception) def handle_exception(e): return jsonify({"error": f"Unexpected Error: {str(e)}"}), 500 # --- DATASET CONFIG --- DATASET_ID = "Yabuku/eraex-data" SHARD_MAPPING = { "2013_2015": ["faiss_index_2013_2015.bin", "metadata_2013_2015.db"], "2016_2018": ["faiss_index_2016_2018.bin", "metadata_2016_2018.db"] } def get_base_dir(): return "/data" if os.path.exists("/data") else "." def ensure_shard_ready(shard_name): base_dir = get_base_dir() files = SHARD_MAPPING.get(shard_name, []) token = os.environ.get("HF_TOKEN") missing = [f for f in files if not os.path.exists(os.path.join(base_dir, f))] if not missing: return True # Already ready print(f"[LazyLoader] Shard {shard_name} missing files: {missing}. Downloading now...", flush=True) for filename in missing: try: # Use hf_transfer if enabled in env hf_hub_download( repo_id=DATASET_ID, filename=filename, repo_type="dataset", local_dir=base_dir, token=token ) print(f"✅ Downloaded {filename}", flush=True) except Exception as e: print(f"❌ Failed to download {filename}: {e}", flush=True) raise e return True # --- REMOVED BACKGROUND DOWNLOADER (Lazy Load Mode) --- @app.route("/") def home(): return render_template("index.html") @app.route("/health") def health(): return jsonify({"status": "ready", "mode": "lazy_loading"}) @app.route("/search", methods=["POST"]) def search_api(): # Lazy Load Logic data = request.json query = data.get("query", "") year_start = data.get("year_start", 2014) year_end = data.get("year_end", 2016) needed_shards = [] # Logic to map years to shards (Naïve but works for current setup) if year_start <= 2015: needed_shards.append("2013_2015") if year_end >= 2016: needed_shards.append("2016_2018") try: for shard in needed_shards: ensure_shard_ready(shard) except Exception as e: return jsonify({"error": f"Failed to load dataset: {str(e)}", "retry": True}), 503 top_k = data.get("top_k", 10) alpha = data.get("alpha", 0.5) print(f"[API] Search Request: '{query}' (Years: {year_start}-{year_end})") # Grab a few extras just in case we filter some out later fetch_k = top_k + 5 try: results = search( query_text=query, target_year=None, start_year=year_start, end_year=year_end, top_k=fetch_k, device="cpu", batch_size=512, alpha=alpha ) print(f"[API] Search finished. Found {len(results)} results.") except Exception as e: print(f"[API] CRITICAL SEARCH ERROR: {e}") return jsonify({"error": str(e)}), 500 response = { "results": results[:top_k], "reserve": results[top_k:] } print(f"[API] Sending response ({len(response['results'])} items)") return jsonify(response) @app.route("/check_links", methods=["POST"]) def check_links_api(): from googlesearch import search as gsearch from difflib import SequenceMatcher data = request.json raw_input = data.get("urls", []) tracks = [] if raw_input and isinstance(raw_input[0], str): tracks = [{"url": u} for u in raw_input] elif raw_input: tracks = raw_input results_map = {} def similarity(a, b): return SequenceMatcher(None, a.lower(), b.lower()).ratio() def check_and_heal(track): # Checking one by one is slow, but it's the only way to be sure it's alive. original_url = track.get("url") if not original_url: return is_alive = False try: oembed = f"https://soundcloud.com/oembed?format=json&url={original_url}" if session.get(oembed, timeout=3.0).status_code == 200: is_alive = True except Exception as e: print(f"[Check Original Error] {original_url}: {e}") if is_alive: results_map[original_url] = {"status": "alive"} print(f"[Alive] {original_url}") return title = track.get("title", "") if not title: results_map[original_url] = {"status": "dead"} return try: query = f"site:soundcloud.com {title}" print(f"[Healing] Searching: {query} ...") found_urls = list(gsearch(query, num_results=3, lang="en")) print(f"[Healing] Candidates for '{title}': {found_urls}") healed_url = None for candidate in found_urls: try: check = f"https://soundcloud.com/oembed?format=json&url={candidate}" resp = session.get(check, timeout=3.0) if resp.status_code == 200: meta = resp.json() cand_title = meta.get("title", "") score = similarity(title, cand_title) print(f" > Cand: {candidate} | Title: {cand_title} | Score: {score:.2f}") if score > 0.6: # 0.6 is the sweet spot. 0.5 gave too many wrong songs. healed_url = candidate break else: print(f" > Rejected (low score)") except: continue if healed_url: results_map[original_url] = {"status": "healed", "new_url": healed_url} print(f"[Healed] '{title}' -> {healed_url}") return else: print(f"[Healing Failed] No valid candidates for '{title}'") print(f"[Fallback] Searching for replacement for '{title}'") fallback_results = search(query_text=title, top_k=5) valid_replacement = None for r in fallback_results: if r.get("permalink_url") == original_url: continue rep_url = r.get("permalink_url") if not rep_url: continue try: check = f"https://soundcloud.com/oembed?format=json&url={rep_url}" if session.get(check, timeout=2.0).status_code == 200: valid_replacement = r break else: print(f" > Replacement candidate dead: {rep_url}") except: continue if valid_replacement: results_map[original_url] = { "status": "replaced", "new_url": valid_replacement.get("permalink_url"), "new_track": valid_replacement } print(f"[Replaced] '{title}' -> '{valid_replacement.get('title')}'") return except Exception as e: # print(f"[Healing Error] {title}: {e}") # keep this quiet for now pass results_map[original_url] = {"status": "dead"} # Increase workers to 5 for better parallel IO performance with concurrent.futures.ThreadPoolExecutor(max_workers=5) as executor: futures = [executor.submit(check_and_heal, t) for t in tracks] concurrent.futures.wait(futures) return jsonify({"results": results_map}) if __name__ == "__main__": print("Starting Flask server on http://localhost:5000") app.run(debug=True, use_reloader=False, port=5000)