Download server.py from Yabuku/Capstone: direct link, hf CLI and curl.
- Browser
- Download file 8.7 kB
-
https://huggingface.co/spaces/Yabuku/Capstone/resolve/main/server.py
- Command line
-
hf download hf://spaces/Yabuku/Capstone/server.py
-
curl -L -o server.py https://huggingface.co/spaces/Yabuku/Capstone/resolve/main/server.py
8.7 kB
| from flask import Flask, render_template, request, jsonify | |
| from main import search | |
| import concurrent.futures | |
| import requests | |
| import os | |
| from huggingface_hub import hf_hub_download | |
| import threading | |
| import time | |
| # Use a global session for connection reuse (Keep-Alive) | |
| session = requests.Session() | |
| adapter = requests.adapters.HTTPAdapter(pool_connections=20, pool_maxsize=20) | |
| session.mount('https://', adapter) | |
| session.mount('http://', adapter) | |
| app = Flask(__name__) | |
| # --- ERROR HANDLERS --- | |
| def internal_error(e): | |
| return jsonify({"error": f"Internal Server Error: {str(e)}"}), 500 | |
| def not_found(e): | |
| return jsonify({"error": "Resource not found (404)"}), 404 | |
| def handle_exception(e): | |
| return jsonify({"error": f"Unexpected Error: {str(e)}"}), 500 | |
| # --- DATASET CONFIG --- | |
| DATASET_ID = "Yabuku/eraex-data" | |
| SHARD_MAPPING = { | |
| "2013_2015": ["faiss_index_2013_2015.bin", "metadata_2013_2015.db"], | |
| "2016_2018": ["faiss_index_2016_2018.bin", "metadata_2016_2018.db"] | |
| } | |
| def get_base_dir(): | |
| return "/data" if os.path.exists("/data") else "." | |
| def ensure_shard_ready(shard_name): | |
| base_dir = get_base_dir() | |
| files = SHARD_MAPPING.get(shard_name, []) | |
| token = os.environ.get("HF_TOKEN") | |
| missing = [f for f in files if not os.path.exists(os.path.join(base_dir, f))] | |
| if not missing: | |
| return True # Already ready | |
| print(f"[LazyLoader] Shard {shard_name} missing files: {missing}. Downloading now...", flush=True) | |
| for filename in missing: | |
| try: | |
| # Use hf_transfer if enabled in env | |
| hf_hub_download( | |
| repo_id=DATASET_ID, | |
| filename=filename, | |
| repo_type="dataset", | |
| local_dir=base_dir, | |
| token=token | |
| ) | |
| print(f"✅ Downloaded {filename}", flush=True) | |
| except Exception as e: | |
| print(f"❌ Failed to download {filename}: {e}", flush=True) | |
| raise e | |
| return True | |
| # --- REMOVED BACKGROUND DOWNLOADER (Lazy Load Mode) --- | |
| def home(): | |
| return render_template("index.html") | |
| def health(): | |
| return jsonify({"status": "ready", "mode": "lazy_loading"}) | |
| def search_api(): | |
| # Lazy Load Logic | |
| data = request.json | |
| query = data.get("query", "") | |
| year_start = data.get("year_start", 2014) | |
| year_end = data.get("year_end", 2016) | |
| needed_shards = [] | |
| # Logic to map years to shards (Naïve but works for current setup) | |
| if year_start <= 2015: needed_shards.append("2013_2015") | |
| if year_end >= 2016: needed_shards.append("2016_2018") | |
| try: | |
| for shard in needed_shards: | |
| ensure_shard_ready(shard) | |
| except Exception as e: | |
| return jsonify({"error": f"Failed to load dataset: {str(e)}", "retry": True}), 503 | |
| top_k = data.get("top_k", 10) | |
| alpha = data.get("alpha", 0.5) | |
| print(f"[API] Search Request: '{query}' (Years: {year_start}-{year_end})") | |
| # Grab a few extras just in case we filter some out later | |
| fetch_k = top_k + 5 | |
| try: | |
| results = search( | |
| query_text=query, | |
| target_year=None, | |
| start_year=year_start, | |
| end_year=year_end, | |
| top_k=fetch_k, | |
| device="cpu", | |
| batch_size=512, | |
| alpha=alpha | |
| ) | |
| print(f"[API] Search finished. Found {len(results)} results.") | |
| except Exception as e: | |
| print(f"[API] CRITICAL SEARCH ERROR: {e}") | |
| return jsonify({"error": str(e)}), 500 | |
| response = { | |
| "results": results[:top_k], | |
| "reserve": results[top_k:] | |
| } | |
| print(f"[API] Sending response ({len(response['results'])} items)") | |
| return jsonify(response) | |
| def check_links_api(): | |
| from googlesearch import search as gsearch | |
| from difflib import SequenceMatcher | |
| data = request.json | |
| raw_input = data.get("urls", []) | |
| tracks = [] | |
| if raw_input and isinstance(raw_input[0], str): | |
| tracks = [{"url": u} for u in raw_input] | |
| elif raw_input: | |
| tracks = raw_input | |
| results_map = {} | |
| def similarity(a, b): | |
| return SequenceMatcher(None, a.lower(), b.lower()).ratio() | |
| def check_and_heal(track): | |
| # Checking one by one is slow, but it's the only way to be sure it's alive. | |
| original_url = track.get("url") | |
| if not original_url: return | |
| is_alive = False | |
| try: | |
| oembed = f"https://soundcloud.com/oembed?format=json&url={original_url}" | |
| if session.get(oembed, timeout=3.0).status_code == 200: | |
| is_alive = True | |
| except Exception as e: | |
| print(f"[Check Original Error] {original_url}: {e}") | |
| if is_alive: | |
| results_map[original_url] = {"status": "alive"} | |
| print(f"[Alive] {original_url}") | |
| return | |
| title = track.get("title", "") | |
| if not title: | |
| results_map[original_url] = {"status": "dead"} | |
| return | |
| try: | |
| query = f"site:soundcloud.com {title}" | |
| print(f"[Healing] Searching: {query} ...") | |
| found_urls = list(gsearch(query, num_results=3, lang="en")) | |
| print(f"[Healing] Candidates for '{title}': {found_urls}") | |
| healed_url = None | |
| for candidate in found_urls: | |
| try: | |
| check = f"https://soundcloud.com/oembed?format=json&url={candidate}" | |
| resp = session.get(check, timeout=3.0) | |
| if resp.status_code == 200: | |
| meta = resp.json() | |
| cand_title = meta.get("title", "") | |
| score = similarity(title, cand_title) | |
| print(f" > Cand: {candidate} | Title: {cand_title} | Score: {score:.2f}") | |
| if score > 0.6: | |
| # 0.6 is the sweet spot. 0.5 gave too many wrong songs. | |
| healed_url = candidate | |
| break | |
| else: | |
| print(f" > Rejected (low score)") | |
| except: continue | |
| if healed_url: | |
| results_map[original_url] = {"status": "healed", "new_url": healed_url} | |
| print(f"[Healed] '{title}' -> {healed_url}") | |
| return | |
| else: | |
| print(f"[Healing Failed] No valid candidates for '{title}'") | |
| print(f"[Fallback] Searching for replacement for '{title}'") | |
| fallback_results = search(query_text=title, top_k=5) | |
| valid_replacement = None | |
| for r in fallback_results: | |
| if r.get("permalink_url") == original_url: continue | |
| rep_url = r.get("permalink_url") | |
| if not rep_url: continue | |
| try: | |
| check = f"https://soundcloud.com/oembed?format=json&url={rep_url}" | |
| if session.get(check, timeout=2.0).status_code == 200: | |
| valid_replacement = r | |
| break | |
| else: | |
| print(f" > Replacement candidate dead: {rep_url}") | |
| except: continue | |
| if valid_replacement: | |
| results_map[original_url] = { | |
| "status": "replaced", | |
| "new_url": valid_replacement.get("permalink_url"), | |
| "new_track": valid_replacement | |
| } | |
| print(f"[Replaced] '{title}' -> '{valid_replacement.get('title')}'") | |
| return | |
| except Exception as e: | |
| # print(f"[Healing Error] {title}: {e}") # keep this quiet for now | |
| pass | |
| results_map[original_url] = {"status": "dead"} | |
| # Increase workers to 5 for better parallel IO performance | |
| with concurrent.futures.ThreadPoolExecutor(max_workers=5) as executor: | |
| futures = [executor.submit(check_and_heal, t) for t in tracks] | |
| concurrent.futures.wait(futures) | |
| return jsonify({"results": results_map}) | |
| if __name__ == "__main__": | |
| print("Starting Flask server on http://localhost:5000") | |
| app.run(debug=True, use_reloader=False, port=5000) |