NINA / fetch.py
Malina
Claude Fable 5.1
v0.90: scoped single-use tokens, hardened link fetch, 45-minute cap, whole-video digest, one classification call, no OCR
1d697a7
Raw History Blame
11.2 kB
# ─────────────────────────────────────────────
# NINA β€” pasted-link ingestion (yt-dlp)
# ─────────────────────────────────────────────
# Downloads a pasted video URL from a KNOWN video site so the normal pipeline can
# analyze it. Best-effort: cloud/Space IPs are frequently blocked (esp. YouTube's
# "confirm you're not a bot"), so the website always offers file upload as the
# fallback. We pull a small H.264 stream (NINA only needs audio + sampled frames),
# cap the duration and the size, give the download a hard deadline, and clean up.
#
# What this is NOT: a generic downloader. The generic extractor is disabled and the
# host is resolved and checked before yt-dlp sees it, so a pasted link cannot make
# this Space fetch an internal address, and the operator's cookies only ever travel
# to the sites they were exported for.
import base64
import ipaddress
import os
import shutil
import socket
import tempfile
import threading
from concurrent.futures import ThreadPoolExecutor, TimeoutError as FutTimeout
from urllib.parse import urlsplit
from auth import verify_usage_token
from cache import register_tmp
from config import FETCH_DEADLINE_SEC, FETCH_MAX_BYTES, MAX_VIDEO_SECONDS
MAX_FETCH_SECONDS = MAX_VIDEO_SECONDS
# H.264 + AAC first: anything else makes Gradio re-encode the whole file after the
# handler returns, on this Space's CPU, usually past the website's deadline.
YT_DLP_FORMAT = ("bv*[height<=480][vcodec^=avc1]+ba[acodec^=mp4a]/b[height<=480][ext=mp4]/"
"bv*[height<=480]+ba/b[height<=480]/best")
# The sites the website offers. yt-dlp's generic extractor (any URL that serves a
# media file) stays OFF.
ALLOWED_EXTRACTORS = ["youtube", "youtube:tab", "vimeo", "facebook", "tiktok", "twitter",
"instagram", "generic:unsupported"]
ALLOWED_HOSTS = ("youtube.com", "youtu.be", "vimeo.com", "facebook.com", "fb.watch",
"tiktok.com", "x.com", "twitter.com", "instagram.com")
# Cookies (if configured) are only kept for these domains.
COOKIE_DOMAINS = tuple(d.strip().lower() for d in
os.getenv("YT_DLP_COOKIE_DOMAINS", "youtube.com,google.com,facebook.com").split(",")
if d.strip())
# A real browser UA helps a little against bot checks; cookies are the actual fix.
_UA = ("Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 "
"(KHTML, like Gecko) Chrome/124.0.0.0 Safari/537.36")
_cookie_path = None
_cookie_lock = threading.Lock()
_pool = ThreadPoolExecutor(max_workers=2, thread_name_prefix="nina-fetch")
def _host_allowed(host: str) -> bool:
host = (host or "").lower().rstrip(".")
if host.startswith("www.") or host.startswith("m."):
host = host.split(".", 1)[1]
return any(host == h or host.endswith("." + h) for h in ALLOWED_HOSTS)
def check_url(url: str) -> str | None:
"""Return None when the URL may be fetched, else a reason for the user.
Checks: scheme, a known video host (no IP literals, no internal names), and that
every address the host resolves to is a public one."""
try:
u = urlsplit((url or "").strip())
except Exception:
return "Please paste a valid video link (starting with http:// or https://)."
if u.scheme not in ("http", "https") or not u.hostname:
return "Please paste a valid video link (starting with http:// or https://)."
if u.username or u.password or u.port not in (None, 80, 443):
return "That link isn't supported. Please upload the file instead."
host = u.hostname
try:
ipaddress.ip_address(host)
return "That link isn't supported. Please upload the file instead." # IP literal
except ValueError:
pass
if not _host_allowed(host):
return "Links from that site aren't supported. Please upload the file instead."
try:
addrs = {ai[4][0] for ai in socket.getaddrinfo(host, None)}
except Exception:
return "That site can't be reached right now. Please upload the file instead."
for a in addrs:
ip = ipaddress.ip_address(a)
if (ip.is_private or ip.is_loopback or ip.is_link_local or ip.is_reserved
or ip.is_multicast or ip.is_unspecified):
return "That link isn't supported. Please upload the file instead."
return None
def _cookiefile():
"""YouTube/Facebook block datacenter IPs ('confirm you're not a bot' / age gate).
Set a Space secret with an exported Netscape cookies.txt to authenticate yt-dlp:
β€’ YT_DLP_COOKIES_B64 β€” base64 of the cookies.txt (preferred; survives newlines), or
β€’ YT_DLP_COOKIES β€” the raw cookies.txt contents.
Only lines for COOKIE_DOMAINS are kept. Returns a path, or None."""
global _cookie_path
with _cookie_lock:
if _cookie_path and os.path.exists(_cookie_path):
return _cookie_path
raw = None
b64 = os.getenv("YT_DLP_COOKIES_B64")
if b64:
try:
raw = base64.b64decode(b64).decode("utf-8", "replace")
except Exception:
raw = None
if raw is None:
raw = os.getenv("YT_DLP_COOKIES")
if not raw or not raw.strip():
return None
kept = []
for line in raw.splitlines():
if not line.strip() or line.startswith("#"):
kept.append(line)
continue
dom = line.split("\t", 1)[0].lstrip(".").lower()
if any(dom == d or dom.endswith("." + d) for d in COOKIE_DOMAINS):
kept.append(line)
try:
fd, p = tempfile.mkstemp(prefix="nina_yt_cookies_", suffix=".txt")
with os.fdopen(fd, "w", encoding="utf-8") as f:
f.write("\n".join(kept) + "\n")
os.chmod(p, 0o600)
_cookie_path = p
return p
except Exception:
return None
def _opts(outdir: str) -> dict:
import yt_dlp
opts = {
"format": YT_DLP_FORMAT,
"outtmpl": os.path.join(outdir, "%(id)s.%(ext)s"),
"noplaylist": True,
"quiet": True,
"no_warnings": True,
"merge_output_format": "mp4",
"socket_timeout": 30,
"retries": 2,
"concurrent_fragment_downloads": 4,
"http_headers": {"User-Agent": _UA},
"allowed_extractors": ALLOWED_EXTRACTORS,
"max_filesize": FETCH_MAX_BYTES,
# No live streams, and nothing whose length is unknown: the duration cap must
# be enforceable before the download starts.
"match_filter": yt_dlp.utils.match_filter_func(f"!is_live & duration <= {MAX_FETCH_SECONDS}"),
}
cookies = _cookiefile()
if cookies:
opts["cookiefile"] = cookies
proxy = os.getenv("YT_DLP_PROXY")
if proxy:
opts["proxy"] = proxy
return opts, bool(cookies), bool(proxy)
def _download(url: str, outdir: str):
"""Runs on the fetch pool. Returns (path, title) or raises."""
import yt_dlp
opts, has_cookies, has_proxy = _opts(outdir)
print(f"[NINA] fetch: cookies={'yes' if has_cookies else 'no'}, proxy={'yes' if has_proxy else 'no'}, "
f"host={urlsplit(url).hostname}", flush=True)
with yt_dlp.YoutubeDL(opts) as ydl:
info = ydl.extract_info(url, download=False)
if info and "entries" in info: # a playlist/channel β†’ first item
entries = [e for e in (info.get("entries") or []) if e]
if not entries:
raise ValueError("no playable video")
info = entries[0]
url = info.get("webpage_url") or info.get("url") or url
dur = (info or {}).get("duration")
if not dur:
raise ValueError("unknown duration")
if dur > MAX_FETCH_SECONDS:
raise ValueError(f"too long: {int(dur // 60)} min")
info = ydl.extract_info(url, download=True)
if info and "entries" in info:
entries = [e for e in (info.get("entries") or []) if e]
if not entries:
raise ValueError("no playable video")
info = entries[0]
path = ydl.prepare_filename(info)
if not os.path.exists(path): # merged/remuxed β†’ different ext
base = os.path.splitext(path)[0]
for ext in (".mp4", ".mkv", ".webm", ".m4a", ".mov"):
if os.path.exists(base + ext):
path = base + ext
break
if not os.path.exists(path):
raise ValueError("no file produced")
title = (info.get("title") or "Video from link").strip()[:200]
print(f"[NINA] Fetched a link ({os.path.getsize(path) // (1024 * 1024)} MB).", flush=True)
return path, title
def fetch_video(url: str, token: str = ""):
"""Download `url` via yt-dlp. Returns (filepath, title) on success, or
(None, error_message) for the website to show."""
if not verify_usage_token(token, "fetch"):
return None, "Not authorized."
url = (url or "").strip()
why = check_url(url)
if why:
return None, why
try:
import yt_dlp # noqa: F401
except Exception:
return None, "Link download isn't available right now β€” please upload the file instead."
outdir = tempfile.mkdtemp(prefix="nina_url_")
register_tmp(outdir) # swept by cache.evict_stale after the reuse window
fut = _pool.submit(_download, url, outdir)
try:
return fut.result(timeout=FETCH_DEADLINE_SEC)
except FutTimeout:
# The thread keeps running; whatever it produces is deleted when it finishes.
fut.add_done_callback(lambda f: shutil.rmtree(outdir, ignore_errors=True))
return None, "That download took too long. Please download the video yourself and upload the file."
except Exception as e:
shutil.rmtree(outdir, ignore_errors=True)
msg = str(e)
low = msg.lower()
if low.startswith("too long"):
return None, f"That video is {msg.split(':', 1)[1].strip()} long β€” the limit is {MAX_FETCH_SECONDS // 60} minutes."
if "unknown duration" in low or "live" in low:
return None, "Live streams and videos without a known length can't be fetched. Please upload the file instead."
if "sign in" in low or "bot" in low or "confirm you" in low:
return None, "This site is blocking downloads from our servers. Please download the video and upload the file instead."
if "private" in low or "login" in low or "members-only" in low:
return None, "That video looks private or login-only β€” please upload the file instead."
if "too large" in low or "file is larger" in low:
return None, "That video is too large to fetch. Please upload a smaller file instead."
print(f"[NINA] fetch failed: {type(e).__name__}", flush=True)
return None, "Couldn't fetch that link β€” it may be unavailable or unsupported. Please upload the file instead."