Swap yt-dlp for SaverAPI

README.md

@@ -20,7 +20,7 @@ per-window fake probabilities alongside a final prediction.
 
 ## How it works
 
-1. Video is downloaded via `yt-dlp` (CPU, no GPU quota used)
+1. Video is downloaded via the **SaverAPI** HTTP API (CPU, no GPU quota used). A pool of up to 10 API keys is rotated automatically with sticky failover.
 2. Frames are decoded and normalized with `decord` (CPU)
 3. V-JEPA 2.1 encodes clips into patch token sequences (GPU)
 4. AttentiveClassifier cross-attends over all tokens and outputs a fake probability (GPU)
@@ -35,14 +35,27 @@ Set these in **Settings → Repository secrets**:
 | `HF_TOKEN` | Your HuggingFace access token |
 | `ENCODER_REPO` | HF repo ID where `vjepa2_1_vitg_384.pt` is stored |
 | `PROBE_REPO` | HF repo ID where `attentive_probe_optimized.pt` is stored |
+| `SAVER_API_KEY_1` … `SAVER_API_KEY_10` | SaverAPI keys (one secret per key; unset ones are ignored) |
 
 `ENCODER_REPO` and `PROBE_REPO` can point to the same private repo.
 
+### Optional SaverAPI tuning secrets
+
+| Secret | Default | Purpose |
+|---|---|---|
+| `SAVER_API_KEY_COOLDOWN_SEC` | `3600` | How long a failing key is skipped before being re-tested |
+| `SAVER_API_LOOKUP_TIMEOUT_SEC` | `15` | Timeout for SaverAPI's JSON lookup |
+| `SAVER_API_DOWNLOAD_TIMEOUT_SEC` | `60` | Timeout for the streamed byte pull |
+| `SAVER_API_MAX_RETRIES_PER_REQUEST` | `3` | How many keys to try before failing a single user request |
+| `SAVER_API_MAX_FILESIZE_MB` | `100` | Hard cap on downloaded video size |
+| `SAVER_API_STARTUP_PROBE_URL` | _(unset)_ | A known-good video URL; if set, all keys are probed at Space boot |
+| `SKIP_STARTUP_PROBE` | _(unset)_ | Set to `1` to skip the boot probe entirely |
+
 ## Files in this Space
 
 | File | Purpose |
 |---|---|
-| `app.py` | Main Gradio app + ZeroGPU integration |
+| `detector.py` | Main Gradio app + ZeroGPU integration + SaverAPI client + key rotator |
 | `inference_core.py` | Model architecture + video windowing (unchanged from training) |
-| `requirements.txt` | Python deps (`decord`, `yt-dlp`) |
+| `requirements.txt` | Python deps (`decord`, `requests`) |
 | `packages.txt` | System dep (`ffmpeg`) |

detector.py

@@ -3,7 +3,6 @@ V-JEPA 2.1 ViT-Giant + AttentiveClassifier probe
 """
 import glob
 import os
-import subprocess
 import tempfile
 import warnings
 import time
@@ -22,6 +21,7 @@ import spaces
 import torch
 from huggingface_hub import hf_hub_download, HfApi
 from inference_core import AttentiveClassifier, get_video_windows
+from saverapi_client import saverapi_fetch
 
 # ─────────────────────────────────────────────
 # CONFIG & SECRETS
@@ -36,6 +36,14 @@ DATASET_REPO_ID = os.environ.get("DATASET_REPO_ID", "")
 TELEGRAM_BOT_TOKEN = os.environ.get("TELEGRAM_BOT_TOKEN", "")
 TELEGRAM_CHAT_ID = os.environ.get("TELEGRAM_CHAT_ID", "")
 
+# ─────────────────────────────────────────────
+# SAVERAPI ROTATOR CONFIG (the client itself lives in saverapi_client.py)
+# ─────────────────────────────────────────────
+SAVER_API_KEY_COOLDOWN_SEC = int(os.environ.get("SAVER_API_KEY_COOLDOWN_SEC", "3600"))
+SAVER_API_MAX_RETRIES_PER_REQUEST = int(os.environ.get("SAVER_API_MAX_RETRIES_PER_REQUEST", "3"))
+SAVER_API_MAX_FILESIZE_MB = int(os.environ.get("SAVER_API_MAX_FILESIZE_MB", "100"))
+SAVER_API_STARTUP_PROBE_URL = os.environ.get("SAVER_API_STARTUP_PROBE_URL", "").strip()
+
 # ─────────────────────────────────────────────
 # MODEL LOADING (CPU)
 # ─────────────────────────────────────────────
@@ -94,33 +102,148 @@ def extract_platform(url: str) -> str:
     else:
         return "Direct Link / Other"
 
-def download_video(url: str) -> tuple[bool, str]:
-    tmp = tempfile.NamedTemporaryFile(suffix=".mp4", delete=False)
-    tmp.close()
-    cmd = [
-        "yt-dlp",
-        "-f", "best[height<=720]/best",
-        "--max-filesize", "100M",
-        "--match-filter", "duration <= 120", 
-        "--no-playlist",
-        "--no-warnings",
-        "--force-overwrites",
-        "-o", tmp.name,
-        url,
-    ]
+
+# ─────────────────────────────────────────────
+# SAVERAPI KEY ROTATOR
+# ─────────────────────────────────────────────
+class KeyRotator:
+    """Sticky-failover pool of SaverAPI keys.
+
+    Reads keys from SAVER_API_KEY_1..SAVER_API_KEY_10. On any error, the
+    failing key is marked dead for `cooldown_sec` seconds; the cursor
+    advances past it so the next call uses a different live key. When the
+    cooldown expires, the key re-enters the pool automatically. State is
+    in-memory only; resets on Space restart.
+    """
+    KEY_ENV_NAMES = [f"SAVER_API_KEY_{i}" for i in range(1, 11)]
+
+    def __init__(self, cooldown_sec: int):
+        self.keys = [os.environ.get(name, "").strip() for name in self.KEY_ENV_NAMES]
+        self.keys = [k for k in self.keys if k]  # drop empties
+        self.cooldown_sec = cooldown_sec
+        self._dead_until: dict = {}            # key -> unix-ts when it recovers
+        self._lock = threading.Lock()
+        self._cursor = 0                        # sticky index: prefers this slot
+
+    def get_key(self) -> str | None:
+        with self._lock:
+            if not self.keys:
+                return None
+            n = len(self.keys)
+            for offset in range(n):
+                idx = (self._cursor + offset) % n
+                key = self.keys[idx]
+                if self._dead_until.get(key, 0) <= time.time():
+                    return key
+            return None  # all keys currently in cooldown
+
+    def mark_dead(self, key: str) -> None:
+        with self._lock:
+            self._dead_until[key] = time.time() + self.cooldown_sec
+            try:
+                self._cursor = (self.keys.index(key) + 1) % len(self.keys)
+            except (ValueError, ZeroDivisionError):
+                pass
+
+    def status(self) -> dict:
+        with self._lock:
+            now = time.time()
+            return {
+                "configured": len(self.keys),
+                "live": sum(1 for k in self.keys if self._dead_until.get(k, 0) <= now),
+                "dead": [self.mask(k) for k, ts in self._dead_until.items() if ts > now],
+                "cursor_index": self._cursor,
+            }
+
+    @staticmethod
+    def mask(key: str) -> str:
+        if len(key) <= 10:
+            return "***"
+        return f"{key[:6]}***{key[-4:]}"
+
+
+key_rotator = KeyRotator(cooldown_sec=SAVER_API_KEY_COOLDOWN_SEC)
+print(f"[SaverAPI] Rotator initialised with {len(key_rotator.keys)} key(s).")
+
+
+def _saverapi_fetch_one(url: str, key: str) -> tuple[bool, str]:
+    """Wrapper around the client that normalises its return to (ok, file_or_err).
+
+    On success: (True, file_path)
+    On failure: (False, "key-level:<reason>") or (False, "<human message>")
+    The caller uses the "key-level:" prefix to decide whether to mark the key dead.
+    """
     try:
-        result = subprocess.run(cmd, capture_output=True, text=True, timeout=90)
-        candidates = sorted(glob.glob(tmp.name + "*") + [tmp.name])
-        actual_file = next((f for f in candidates if os.path.exists(f) and os.path.getsize(f) > 0), None)
-        if actual_file is None:
-            if "does not pass filter" in result.stdout or "does not pass filter" in result.stderr:
-                return False, "Video length exceeds the maximum 2-minute limit."
-            return False, f"yt-dlp wrote no output. stderr: {result.stderr[:400]}"
-        return True, actual_file
-    except subprocess.TimeoutExpired:
-        return False, "Download timed out (90s limit)."
-    except FileNotFoundError:
-        return False, "yt-dlp not installed."
+        return saverapi_fetch(url, key, max_filesize_mb=SAVER_API_MAX_FILESIZE_MB)
+    except Exception as e:
+        return False, f"key-level:exception: {type(e).__name__}: {e}"
+
+
+def download_video(url: str) -> tuple[bool, str]:
+    """Tries up to N keys in sequence. Marks a key dead only on key-level errors;
+    URL-level errors (bad URL, unsupported platform, etc.) are returned to the
+    user without consuming a key from the pool.
+    """
+    if not url.lower().startswith(("http://", "https://")):
+        return False, "Unsupported URL scheme. Use http:// or https://"
+
+    last_err = ""
+    for attempt in range(SAVER_API_MAX_RETRIES_PER_REQUEST):
+        key = key_rotator.get_key()
+        if key is None:
+            return False, "SaverAPI key pool exhausted; try again later."
+
+        ok, payload = _saverapi_fetch_one(url, key)
+        if ok:
+            return True, payload
+
+        last_err = payload
+        if last_err.startswith("key-level:"):
+            key_rotator.mark_dead(key)
+            print(f"[SaverAPI] key {KeyRotator.mask(key)} failed (attempt {attempt + 1}/"
+                  f"{SAVER_API_MAX_RETRIES_PER_REQUEST}): {last_err[:200]}")
+            continue
+        # URL-level error — do NOT burn a key, return to the user.
+        return False, last_err
+
+    return False, f"SaverAPI: all {SAVER_API_MAX_RETRIES_PER_REQUEST} retries failed. Last: {last_err[:200]}"
+
+
+def _startup_probe():
+    """Best-effort probe of each configured key at Space boot.
+
+    Disabled by default (set SAVER_API_STARTUP_PROBE_URL to a known-good
+    video URL to enable, or set SKIP_STARTUP_PROBE=1 to silence the
+    'no probe URL' log). Marks dead keys so the first real user request
+    doesn't hit a dead key.
+    """
+    if os.environ.get("SKIP_STARTUP_PROBE", "").lower() in ("1", "true", "yes"):
+        print("[SaverAPI] Startup probe skipped (SKIP_STARTUP_PROBE=1).")
+        return
+    if not SAVER_API_STARTUP_PROBE_URL:
+        print("[SaverAPI] No SAVER_API_STARTUP_PROBE_URL set; skipping startup probe.")
+        return
+    if not key_rotator.keys:
+        print("[SaverAPI] No keys configured; skipping startup probe.")
+        return
+
+    print(f"[SaverAPI] Probing {len(key_rotator.keys)} key(s) with {SAVER_API_STARTUP_PROBE_URL} ...")
+    for key in key_rotator.keys:
+        ok, payload = _saverapi_fetch_one(SAVER_API_STARTUP_PROBE_URL, key)
+        if ok:
+            try:
+                os.remove(payload)
+            except OSError:
+                pass
+            print(f"[SaverAPI]   key {KeyRotator.mask(key)}: OK")
+            continue
+        # Only mark dead on key-level failures. URL-level errors mean the probe
+        # URL itself is bad, not the key.
+        if payload.startswith("key-level:"):
+            key_rotator.mark_dead(key)
+            print(f"[SaverAPI]   key {KeyRotator.mask(key)}: DEAD ({payload[:100]})")
+        else:
+            print(f"[SaverAPI]   key {KeyRotator.mask(key)}: probe-URL error ({payload[:100]})")
 
 # ─────────────────────────────────────────────
 # GPU INFERENCE
@@ -391,4 +514,9 @@ with gr.Blocks(title="Deepfake Detector") as demo:
         outputs=[label_out, conf_out, breakdown_out, session_state],
     )
 
+# ─────────────────────────────────────────────
+# STARTUP PROBE (best-effort key health check)
+# ─────────────────────────────────────────────
+_startup_probe()
+
 demo.launch()

requirements.txt

@@ -7,8 +7,7 @@ einops
 # Video decoding
 decord
 
-# Social media video download
-yt-dlp
+# Social media video download (SaverAPI HTTP, no yt-dlp)
 
 
 gradio

saverapi_client.py

@@ -0,0 +1,241 @@
+"""
+SaverAPI video fetcher.
+
+Dispatches to the right SaverAPI endpoint based on URL:
+  - YouTube / youtu.be / youtube Shorts  ->  /api/youtube-api
+  - Everything else (Instagram, TikTok, X, ...)  ->  /api/all-in-one-downloader-api
+
+Stdlib only (urllib). Runs anywhere Python 3.7+ is installed, no pip needed.
+
+Response shapes handled:
+  all-in-one / IG+TT  : {"error": false, "download_url": "https://...", ...}
+  all-in-one / Vimeo  : {"medias": [{"url": "https://...", "quality": "..."}, ...], ...}
+  all-in-one / others : {"url": "https://...", "source": "...", ...}  (less common)
+  all-in-one URL-err  : {"error": true,  "message": "..."}                    (URL-level, do NOT mark key dead)
+  all-in-one / key    : HTTP 402 + {"message": "Insufficient credits"}        (KEY-level, mark dead)
+  youtube success     : {"status": "tunnel", "url": "https://...", ...}
+  youtube error       : (unconfirmed — passed through as best-effort)
+  Cloudflare block    : HTTP 403, body contains "error code: 1010"             (key/network, mark dead)
+"""
+import json
+import os
+import sys
+import tempfile
+import urllib.error
+import urllib.parse
+import urllib.request
+
+# ── Endpoints ──────────────────────────────────────────────────────────────
+ALL_IN_ONE_URL = "https://saverapi.net/api/all-in-one-downloader-api"
+YOUTUBE_URL    = "https://saverapi.net/api/youtube-api"
+
+# ── Cloudflare 1010 bypass: send a real-browser User-Agent ────────────────
+BROWSER_HEADERS = {
+    "User-Agent": (
+        "Mozilla/5.0 (Windows NT 10.0; Win64; x64) "
+        "AppleWebKit/537.36 (KHTML, like Gecko) "
+        "Chrome/120.0.0.0 Safari/537.36"
+    ),
+    "Accept":          "application/json, text/plain, */*",
+    "Accept-Language": "en-US,en;q=0.9",
+    "Origin":          "https://saverapi.net",
+    "Referer":         "https://saverapi.net/",
+}
+
+# ── Tuning ─────────────────────────────────────────────────────────────────
+LOOKUP_TIMEOUT_SEC   = 15
+DOWNLOAD_TIMEOUT_SEC = 60
+DEFAULT_MAX_FILESIZE_MB = 100
+YOUTUBE_DEFAULT_QUALITY  = "720"   # SaverAPI's param is misspelled "farmat"
+
+
+# ── Platform detection ────────────────────────────────────────────────────
+def _is_youtube(url: str) -> bool:
+    u = url.lower()
+    return ("youtube.com" in u) or ("youtu.be" in u) or ("youtube-nocookie.com" in u)
+
+
+# ── Step 1: ask SaverAPI for a direct video URL ──────────────────────────
+def _saverapi_lookup(url: str, api_key: str) -> tuple[bool, str, str]:
+    """Returns (ok, direct_video_url, error_message)."""
+    if _is_youtube(url):
+        endpoint = YOUTUBE_URL
+        params   = {"url": url, "farmat": YOUTUBE_DEFAULT_QUALITY}
+    else:
+        endpoint = ALL_IN_ONE_URL
+        params   = {"url": url}
+
+    full_url = f"{endpoint}?{urllib.parse.urlencode(params)}"
+    req = urllib.request.Request(
+        full_url,
+        headers={**BROWSER_HEADERS, "x-api-key": api_key, "Content-Type": "application/json"},
+        method="GET",
+    )
+
+    try:
+        with urllib.request.urlopen(req, timeout=LOOKUP_TIMEOUT_SEC) as r:
+            status = r.status
+            body   = r.read().decode("utf-8", errors="replace")
+    except urllib.error.HTTPError as e:
+        body = e.read().decode("utf-8", errors="replace")
+        if e.code == 403 and "error code" in body:
+            return False, "", "key-level:cloudflare"
+        if e.code == 402 and "insufficient" in body.lower():
+            return False, "", "key-level:insufficient-credits"
+        return False, "", f"key-level:http-{e.code}: {body[:200]}"
+    except Exception as e:
+        return False, "", f"key-level:network: {type(e).__name__}: {e}"
+
+    if status != 200:
+        return False, "", f"key-level:http-{status}: {body[:200]}"
+
+    try:
+        result = json.loads(body)
+    except json.JSONDecodeError:
+        return False, "", f"Non-JSON response: {body[:200]}"
+
+    if not isinstance(result, dict):
+        return False, "", f"Unexpected response (not an object): {str(result)[:200]}"
+
+    # YouTube success: {"status": "tunnel", "url": "https://..."}
+    if result.get("status") == "tunnel" and result.get("url"):
+        return True, result["url"], ""
+
+    # Vimeo-style success: {"medias": [{"url": "...", "quality": "..."}, ...], ...}
+    medias = result.get("medias")
+    if isinstance(medias, list) and medias:
+        # SaverAPI lists medias best-first; pick the first with a URL.
+        for m in medias:
+            if isinstance(m, dict) and m.get("url"):
+                return True, m["url"], ""
+        return False, "", "medias[] present but no url in any item"
+
+    # all-in-one success: {"error": false, "download_url": "https://..."}
+    if result.get("error") is False:
+        direct = result.get("download_url")
+        if direct:
+            return True, direct, ""
+        return False, "", "all-in-one: error=false but no download_url"
+
+    # all-in-one URL-level error: {"error": true, "message": "..."}
+    if result.get("error") is True:
+        return False, "", f"SaverAPI: {result.get('message', 'unspecified error')}"
+
+    # YouTube non-tunnel status (unconfirmed error shape — pass through)
+    if "status" in result:
+        return False, "", f"YouTube API status={result.get('status')!r}"
+
+    # Generic fallback: some platforms return a top-level "url" + "source"
+    if result.get("url") and result.get("source"):
+        return True, result["url"], ""
+
+    return False, "", f"Unexpected shape: {str(result)[:200]}"
+
+
+# ── Step 2: stream the direct URL to a tempfile with a size cap ──────────
+def _stream_to_file(direct_url: str, max_bytes: int) -> tuple[bool, str]:
+    # Strip saverapi.net-specific headers (Referer/Origin) — those are for the
+    # SaverAPI lookup, not for the underlying CDN (twimg.com, fbcdn.net, etc.).
+    # Keep User-Agent + Accept so the CDN sees a real browser.
+    headers = {
+        "User-Agent": BROWSER_HEADERS["User-Agent"],
+        "Accept":     "*/*",
+    }
+    req = urllib.request.Request(direct_url, headers=headers, method="GET")
+
+    tmp = tempfile.NamedTemporaryFile(suffix=".mp4", delete=False)
+    tmp.close()
+    try:
+        with urllib.request.urlopen(req, timeout=DOWNLOAD_TIMEOUT_SEC) as r:
+            # Peek first 1 KB to catch HTML error pages
+            first_chunk = r.read(1024)
+            if first_chunk.lstrip().lower().startswith((b"<!doctype", b"<html")):
+                return False, "download_url returned HTML, not a video"
+
+            bytes_written = 0
+            with open(tmp.name, "wb") as f:
+                if first_chunk:
+                    f.write(first_chunk)
+                    bytes_written = len(first_chunk)
+                if bytes_written > max_bytes:
+                    return False, f"Exceeds {max_bytes // (1024 * 1024)} MB cap"
+                while True:
+                    chunk = r.read(64 * 1024)
+                    if not chunk:
+                        break
+                    bytes_written += len(chunk)
+                    if bytes_written > max_bytes:
+                        return False, f"Exceeds {max_bytes // (1024 * 1024)} MB cap"
+                    f.write(chunk)
+        if bytes_written == 0:
+            return False, "Empty response"
+        return True, tmp.name
+    except urllib.error.HTTPError as e:
+        try:
+            os.remove(tmp.name)
+        except OSError:
+            pass
+        return False, f"Download HTTP {e.code}"
+    except Exception as e:
+        try:
+            os.remove(tmp.name)
+        except OSError:
+            pass
+        return False, f"Download error: {type(e).__name__}: {e}"
+
+
+# ── Public entry point ────────────────────────────────────────────────────
+def saverapi_fetch(
+    url: str,
+    api_key: str,
+    max_filesize_mb: int = DEFAULT_MAX_FILESIZE_MB,
+) -> tuple[bool, str]:
+    """
+    Fetch a video from a supported platform via SaverAPI.
+
+    Args:
+        url:             Public video URL (Instagram, TikTok, YouTube, ...)
+        api_key:         SaverAPI key
+        max_filesize_mb: Hard cap on downloaded video size (default 100)
+
+    Returns:
+        (True,  file_path)   on success — caller must `os.remove(path)` when done
+        (False, err_message) on any failure
+    """
+    if not url.lower().startswith(("http://", "https://")):
+        return False, "Unsupported URL scheme (use http:// or https://)"
+
+    ok, direct, err = _saverapi_lookup(url, api_key)
+    if not ok:
+        return False, err
+
+    return _stream_to_file(direct, max_filesize_mb * 1024 * 1024)
+
+
+# ── Quick self-test ───────────────────────────────────────────────────────
+if __name__ == "__main__":
+    import time
+    api_key = os.environ.get("SAVER_API_KEY", "").strip()
+    if not api_key:
+        print("Set SAVER_API_KEY env var first.", file=sys.stderr)
+        sys.exit(1)
+
+    tests = [
+        "https://www.instagram.com/reel/DZTCpI2SQsq",
+        "https://www.tiktok.com/@scout2015/video/6718335390845095173",
+        "https://www.facebook.com/reel/763412386794044/",
+        "https://x.com/traderpodcaste/status/2063797483629629691",
+        "https://vimeo.com/76979871",
+        "https://www.youtube.com/shorts/LXb3EKWsInQ",
+    ]
+    for u in tests:
+        print(f"\n→ {u}")
+        t0 = time.time()
+        ok, result = saverapi_fetch(u, api_key)
+        dt = time.time() - t0
+        if ok:
+            size = os.path.getsize(result) // 1024
+            print(f"  ✅  {size} KB in {dt:.1f}s  →  {result}")
+            os.remove(result)
+        else:
+            print(f"  ❌  {result}")