# ───────────────────────────────────────────── # NINA — pasted-link ingestion (yt-dlp) # ───────────────────────────────────────────── # Downloads a pasted video URL from a KNOWN video site so the normal pipeline can # analyze it. Best-effort: cloud/Space IPs are frequently blocked (esp. YouTube's # "confirm you're not a bot"), so the website always offers file upload as the # fallback. We pull a small H.264 stream (NINA only needs audio + sampled frames), # cap the duration and the size, give the download a hard deadline, and clean up. # # What this is NOT: a generic downloader. The generic extractor is disabled and the # host is resolved and checked before yt-dlp sees it, so a pasted link cannot make # this Space fetch an internal address, and the operator's cookies only ever travel # to the sites they were exported for. import base64 import ipaddress import os import shutil import socket import tempfile import threading from concurrent.futures import ThreadPoolExecutor, TimeoutError as FutTimeout from urllib.parse import urlsplit from auth import verify_usage_token from cache import register_tmp from config import FETCH_DEADLINE_SEC, FETCH_MAX_BYTES, MAX_VIDEO_SECONDS MAX_FETCH_SECONDS = MAX_VIDEO_SECONDS # H.264 + AAC first: anything else makes Gradio re-encode the whole file after the # handler returns, on this Space's CPU, usually past the website's deadline. YT_DLP_FORMAT = ("bv*[height<=480][vcodec^=avc1]+ba[acodec^=mp4a]/b[height<=480][ext=mp4]/" "bv*[height<=480]+ba/b[height<=480]/best") # The sites the website offers. yt-dlp's generic extractor (any URL that serves a # media file) stays OFF. ALLOWED_EXTRACTORS = ["youtube", "youtube:tab", "vimeo", "facebook", "tiktok", "twitter", "instagram", "generic:unsupported"] ALLOWED_HOSTS = ("youtube.com", "youtu.be", "vimeo.com", "facebook.com", "fb.watch", "tiktok.com", "x.com", "twitter.com", "instagram.com") # Cookies (if configured) are only kept for these domains. COOKIE_DOMAINS = tuple(d.strip().lower() for d in os.getenv("YT_DLP_COOKIE_DOMAINS", "youtube.com,google.com,facebook.com").split(",") if d.strip()) # A real browser UA helps a little against bot checks; cookies are the actual fix. _UA = ("Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 " "(KHTML, like Gecko) Chrome/124.0.0.0 Safari/537.36") _cookie_path = None _cookie_lock = threading.Lock() _pool = ThreadPoolExecutor(max_workers=2, thread_name_prefix="nina-fetch") def _host_allowed(host: str) -> bool: host = (host or "").lower().rstrip(".") if host.startswith("www.") or host.startswith("m."): host = host.split(".", 1)[1] return any(host == h or host.endswith("." + h) for h in ALLOWED_HOSTS) def check_url(url: str) -> str | None: """Return None when the URL may be fetched, else a reason for the user. Checks: scheme, a known video host (no IP literals, no internal names), and that every address the host resolves to is a public one.""" try: u = urlsplit((url or "").strip()) except Exception: return "Please paste a valid video link (starting with http:// or https://)." if u.scheme not in ("http", "https") or not u.hostname: return "Please paste a valid video link (starting with http:// or https://)." if u.username or u.password or u.port not in (None, 80, 443): return "That link isn't supported. Please upload the file instead." host = u.hostname try: ipaddress.ip_address(host) return "That link isn't supported. Please upload the file instead." # IP literal except ValueError: pass if not _host_allowed(host): return "Links from that site aren't supported. Please upload the file instead." try: addrs = {ai[4][0] for ai in socket.getaddrinfo(host, None)} except Exception: return "That site can't be reached right now. Please upload the file instead." for a in addrs: ip = ipaddress.ip_address(a) if (ip.is_private or ip.is_loopback or ip.is_link_local or ip.is_reserved or ip.is_multicast or ip.is_unspecified): return "That link isn't supported. Please upload the file instead." return None def _cookiefile(): """YouTube/Facebook block datacenter IPs ('confirm you're not a bot' / age gate). Set a Space secret with an exported Netscape cookies.txt to authenticate yt-dlp: • YT_DLP_COOKIES_B64 — base64 of the cookies.txt (preferred; survives newlines), or • YT_DLP_COOKIES — the raw cookies.txt contents. Only lines for COOKIE_DOMAINS are kept. Returns a path, or None.""" global _cookie_path with _cookie_lock: if _cookie_path and os.path.exists(_cookie_path): return _cookie_path raw = None b64 = os.getenv("YT_DLP_COOKIES_B64") if b64: try: raw = base64.b64decode(b64).decode("utf-8", "replace") except Exception: raw = None if raw is None: raw = os.getenv("YT_DLP_COOKIES") if not raw or not raw.strip(): return None kept = [] for line in raw.splitlines(): if not line.strip() or line.startswith("#"): kept.append(line) continue dom = line.split("\t", 1)[0].lstrip(".").lower() if any(dom == d or dom.endswith("." + d) for d in COOKIE_DOMAINS): kept.append(line) try: fd, p = tempfile.mkstemp(prefix="nina_yt_cookies_", suffix=".txt") with os.fdopen(fd, "w", encoding="utf-8") as f: f.write("\n".join(kept) + "\n") os.chmod(p, 0o600) _cookie_path = p return p except Exception: return None def _opts(outdir: str) -> dict: import yt_dlp opts = { "format": YT_DLP_FORMAT, "outtmpl": os.path.join(outdir, "%(id)s.%(ext)s"), "noplaylist": True, "quiet": True, "no_warnings": True, "merge_output_format": "mp4", "socket_timeout": 30, "retries": 2, "concurrent_fragment_downloads": 4, "http_headers": {"User-Agent": _UA}, "allowed_extractors": ALLOWED_EXTRACTORS, "max_filesize": FETCH_MAX_BYTES, # No live streams, and nothing whose length is unknown: the duration cap must # be enforceable before the download starts. "match_filter": yt_dlp.utils.match_filter_func(f"!is_live & duration <= {MAX_FETCH_SECONDS}"), } cookies = _cookiefile() if cookies: opts["cookiefile"] = cookies proxy = os.getenv("YT_DLP_PROXY") if proxy: opts["proxy"] = proxy return opts, bool(cookies), bool(proxy) def _download(url: str, outdir: str): """Runs on the fetch pool. Returns (path, title) or raises.""" import yt_dlp opts, has_cookies, has_proxy = _opts(outdir) print(f"[NINA] fetch: cookies={'yes' if has_cookies else 'no'}, proxy={'yes' if has_proxy else 'no'}, " f"host={urlsplit(url).hostname}", flush=True) with yt_dlp.YoutubeDL(opts) as ydl: info = ydl.extract_info(url, download=False) if info and "entries" in info: # a playlist/channel → first item entries = [e for e in (info.get("entries") or []) if e] if not entries: raise ValueError("no playable video") info = entries[0] url = info.get("webpage_url") or info.get("url") or url dur = (info or {}).get("duration") if not dur: raise ValueError("unknown duration") if dur > MAX_FETCH_SECONDS: raise ValueError(f"too long: {int(dur // 60)} min") info = ydl.extract_info(url, download=True) if info and "entries" in info: entries = [e for e in (info.get("entries") or []) if e] if not entries: raise ValueError("no playable video") info = entries[0] path = ydl.prepare_filename(info) if not os.path.exists(path): # merged/remuxed → different ext base = os.path.splitext(path)[0] for ext in (".mp4", ".mkv", ".webm", ".m4a", ".mov"): if os.path.exists(base + ext): path = base + ext break if not os.path.exists(path): raise ValueError("no file produced") title = (info.get("title") or "Video from link").strip()[:200] print(f"[NINA] Fetched a link ({os.path.getsize(path) // (1024 * 1024)} MB).", flush=True) return path, title def fetch_video(url: str, token: str = ""): """Download `url` via yt-dlp. Returns (filepath, title) on success, or (None, error_message) for the website to show.""" if not verify_usage_token(token, "fetch"): return None, "Not authorized." url = (url or "").strip() why = check_url(url) if why: return None, why try: import yt_dlp # noqa: F401 except Exception: return None, "Link download isn't available right now — please upload the file instead." outdir = tempfile.mkdtemp(prefix="nina_url_") register_tmp(outdir) # swept by cache.evict_stale after the reuse window fut = _pool.submit(_download, url, outdir) try: return fut.result(timeout=FETCH_DEADLINE_SEC) except FutTimeout: # The thread keeps running; whatever it produces is deleted when it finishes. fut.add_done_callback(lambda f: shutil.rmtree(outdir, ignore_errors=True)) return None, "That download took too long. Please download the video yourself and upload the file." except Exception as e: shutil.rmtree(outdir, ignore_errors=True) msg = str(e) low = msg.lower() if low.startswith("too long"): return None, f"That video is {msg.split(':', 1)[1].strip()} long — the limit is {MAX_FETCH_SECONDS // 60} minutes." if "unknown duration" in low or "live" in low: return None, "Live streams and videos without a known length can't be fetched. Please upload the file instead." if "sign in" in low or "bot" in low or "confirm you" in low: return None, "This site is blocking downloads from our servers. Please download the video and upload the file instead." if "private" in low or "login" in low or "members-only" in low: return None, "That video looks private or login-only — please upload the file instead." if "too large" in low or "file is larger" in low: return None, "That video is too large to fetch. Please upload a smaller file instead." print(f"[NINA] fetch failed: {type(e).__name__}", flush=True) return None, "Couldn't fetch that link — it may be unavailable or unsupported. Please upload the file instead."