Capstone / server.py
Yabuku's picture
Clean up: Remove HuggingFace Spaces and other cloud platform files, keep only AWS deployment
d54d151
Raw History Blame Contribute Delete
8.7 kB
from flask import Flask, render_template, request, jsonify
from main import search
import concurrent.futures
import requests
import os
from huggingface_hub import hf_hub_download
import threading
import time
# Use a global session for connection reuse (Keep-Alive)
session = requests.Session()
adapter = requests.adapters.HTTPAdapter(pool_connections=20, pool_maxsize=20)
session.mount('https://', adapter)
session.mount('http://', adapter)
app = Flask(__name__)
# --- ERROR HANDLERS ---
@app.errorhandler(500)
def internal_error(e):
return jsonify({"error": f"Internal Server Error: {str(e)}"}), 500
@app.errorhandler(404)
def not_found(e):
return jsonify({"error": "Resource not found (404)"}), 404
@app.errorhandler(Exception)
def handle_exception(e):
return jsonify({"error": f"Unexpected Error: {str(e)}"}), 500
# --- DATASET CONFIG ---
DATASET_ID = "Yabuku/eraex-data"
SHARD_MAPPING = {
"2013_2015": ["faiss_index_2013_2015.bin", "metadata_2013_2015.db"],
"2016_2018": ["faiss_index_2016_2018.bin", "metadata_2016_2018.db"]
}
def get_base_dir():
return "/data" if os.path.exists("/data") else "."
def ensure_shard_ready(shard_name):
base_dir = get_base_dir()
files = SHARD_MAPPING.get(shard_name, [])
token = os.environ.get("HF_TOKEN")
missing = [f for f in files if not os.path.exists(os.path.join(base_dir, f))]
if not missing:
return True # Already ready
print(f"[LazyLoader] Shard {shard_name} missing files: {missing}. Downloading now...", flush=True)
for filename in missing:
try:
# Use hf_transfer if enabled in env
hf_hub_download(
repo_id=DATASET_ID,
filename=filename,
repo_type="dataset",
local_dir=base_dir,
token=token
)
print(f"✅ Downloaded {filename}", flush=True)
except Exception as e:
print(f"❌ Failed to download {filename}: {e}", flush=True)
raise e
return True
# --- REMOVED BACKGROUND DOWNLOADER (Lazy Load Mode) ---
@app.route("/")
def home():
return render_template("index.html")
@app.route("/health")
def health():
return jsonify({"status": "ready", "mode": "lazy_loading"})
@app.route("/search", methods=["POST"])
def search_api():
# Lazy Load Logic
data = request.json
query = data.get("query", "")
year_start = data.get("year_start", 2014)
year_end = data.get("year_end", 2016)
needed_shards = []
# Logic to map years to shards (Naïve but works for current setup)
if year_start <= 2015: needed_shards.append("2013_2015")
if year_end >= 2016: needed_shards.append("2016_2018")
try:
for shard in needed_shards:
ensure_shard_ready(shard)
except Exception as e:
return jsonify({"error": f"Failed to load dataset: {str(e)}", "retry": True}), 503
top_k = data.get("top_k", 10)
alpha = data.get("alpha", 0.5)
print(f"[API] Search Request: '{query}' (Years: {year_start}-{year_end})")
# Grab a few extras just in case we filter some out later
fetch_k = top_k + 5
try:
results = search(
query_text=query,
target_year=None,
start_year=year_start,
end_year=year_end,
top_k=fetch_k,
device="cpu",
batch_size=512,
alpha=alpha
)
print(f"[API] Search finished. Found {len(results)} results.")
except Exception as e:
print(f"[API] CRITICAL SEARCH ERROR: {e}")
return jsonify({"error": str(e)}), 500
response = {
"results": results[:top_k],
"reserve": results[top_k:]
}
print(f"[API] Sending response ({len(response['results'])} items)")
return jsonify(response)
@app.route("/check_links", methods=["POST"])
def check_links_api():
from googlesearch import search as gsearch
from difflib import SequenceMatcher
data = request.json
raw_input = data.get("urls", [])
tracks = []
if raw_input and isinstance(raw_input[0], str):
tracks = [{"url": u} for u in raw_input]
elif raw_input:
tracks = raw_input
results_map = {}
def similarity(a, b):
return SequenceMatcher(None, a.lower(), b.lower()).ratio()
def check_and_heal(track):
# Checking one by one is slow, but it's the only way to be sure it's alive.
original_url = track.get("url")
if not original_url: return
is_alive = False
try:
oembed = f"https://soundcloud.com/oembed?format=json&url={original_url}"
if session.get(oembed, timeout=3.0).status_code == 200:
is_alive = True
except Exception as e:
print(f"[Check Original Error] {original_url}: {e}")
if is_alive:
results_map[original_url] = {"status": "alive"}
print(f"[Alive] {original_url}")
return
title = track.get("title", "")
if not title:
results_map[original_url] = {"status": "dead"}
return
try:
query = f"site:soundcloud.com {title}"
print(f"[Healing] Searching: {query} ...")
found_urls = list(gsearch(query, num_results=3, lang="en"))
print(f"[Healing] Candidates for '{title}': {found_urls}")
healed_url = None
for candidate in found_urls:
try:
check = f"https://soundcloud.com/oembed?format=json&url={candidate}"
resp = session.get(check, timeout=3.0)
if resp.status_code == 200:
meta = resp.json()
cand_title = meta.get("title", "")
score = similarity(title, cand_title)
print(f" > Cand: {candidate} | Title: {cand_title} | Score: {score:.2f}")
if score > 0.6:
# 0.6 is the sweet spot. 0.5 gave too many wrong songs.
healed_url = candidate
break
else:
print(f" > Rejected (low score)")
except: continue
if healed_url:
results_map[original_url] = {"status": "healed", "new_url": healed_url}
print(f"[Healed] '{title}' -> {healed_url}")
return
else:
print(f"[Healing Failed] No valid candidates for '{title}'")
print(f"[Fallback] Searching for replacement for '{title}'")
fallback_results = search(query_text=title, top_k=5)
valid_replacement = None
for r in fallback_results:
if r.get("permalink_url") == original_url: continue
rep_url = r.get("permalink_url")
if not rep_url: continue
try:
check = f"https://soundcloud.com/oembed?format=json&url={rep_url}"
if session.get(check, timeout=2.0).status_code == 200:
valid_replacement = r
break
else:
print(f" > Replacement candidate dead: {rep_url}")
except: continue
if valid_replacement:
results_map[original_url] = {
"status": "replaced",
"new_url": valid_replacement.get("permalink_url"),
"new_track": valid_replacement
}
print(f"[Replaced] '{title}' -> '{valid_replacement.get('title')}'")
return
except Exception as e:
# print(f"[Healing Error] {title}: {e}") # keep this quiet for now
pass
results_map[original_url] = {"status": "dead"}
# Increase workers to 5 for better parallel IO performance
with concurrent.futures.ThreadPoolExecutor(max_workers=5) as executor:
futures = [executor.submit(check_and_heal, t) for t in tracks]
concurrent.futures.wait(futures)
return jsonify({"results": results_map})
if __name__ == "__main__":
print("Starting Flask server on http://localhost:5000")
app.run(debug=True, use_reloader=False, port=5000)