Spaces:
Paused
Paused
Malina
Claude Fable 5.1
v0.90: scoped single-use tokens, hardened link fetch, 45-minute cap, whole-video digest, one classification call, no OCR
1d697a7 Download fetch.py from NextNeural/NINA: direct link, hf CLI and curl.
- Browser
- Download file 11.2 kB
-
https://huggingface.co/spaces/NextNeural/NINA/resolve/main/fetch.py
- Command line
-
hf download hf://spaces/NextNeural/NINA/fetch.py
-
curl -L -o fetch.py https://huggingface.co/spaces/NextNeural/NINA/resolve/main/fetch.py
11.2 kB
| # βββββββββββββββββββββββββββββββββββββββββββββ | |
| # NINA β pasted-link ingestion (yt-dlp) | |
| # βββββββββββββββββββββββββββββββββββββββββββββ | |
| # Downloads a pasted video URL from a KNOWN video site so the normal pipeline can | |
| # analyze it. Best-effort: cloud/Space IPs are frequently blocked (esp. YouTube's | |
| # "confirm you're not a bot"), so the website always offers file upload as the | |
| # fallback. We pull a small H.264 stream (NINA only needs audio + sampled frames), | |
| # cap the duration and the size, give the download a hard deadline, and clean up. | |
| # | |
| # What this is NOT: a generic downloader. The generic extractor is disabled and the | |
| # host is resolved and checked before yt-dlp sees it, so a pasted link cannot make | |
| # this Space fetch an internal address, and the operator's cookies only ever travel | |
| # to the sites they were exported for. | |
| import base64 | |
| import ipaddress | |
| import os | |
| import shutil | |
| import socket | |
| import tempfile | |
| import threading | |
| from concurrent.futures import ThreadPoolExecutor, TimeoutError as FutTimeout | |
| from urllib.parse import urlsplit | |
| from auth import verify_usage_token | |
| from cache import register_tmp | |
| from config import FETCH_DEADLINE_SEC, FETCH_MAX_BYTES, MAX_VIDEO_SECONDS | |
| MAX_FETCH_SECONDS = MAX_VIDEO_SECONDS | |
| # H.264 + AAC first: anything else makes Gradio re-encode the whole file after the | |
| # handler returns, on this Space's CPU, usually past the website's deadline. | |
| YT_DLP_FORMAT = ("bv*[height<=480][vcodec^=avc1]+ba[acodec^=mp4a]/b[height<=480][ext=mp4]/" | |
| "bv*[height<=480]+ba/b[height<=480]/best") | |
| # The sites the website offers. yt-dlp's generic extractor (any URL that serves a | |
| # media file) stays OFF. | |
| ALLOWED_EXTRACTORS = ["youtube", "youtube:tab", "vimeo", "facebook", "tiktok", "twitter", | |
| "instagram", "generic:unsupported"] | |
| ALLOWED_HOSTS = ("youtube.com", "youtu.be", "vimeo.com", "facebook.com", "fb.watch", | |
| "tiktok.com", "x.com", "twitter.com", "instagram.com") | |
| # Cookies (if configured) are only kept for these domains. | |
| COOKIE_DOMAINS = tuple(d.strip().lower() for d in | |
| os.getenv("YT_DLP_COOKIE_DOMAINS", "youtube.com,google.com,facebook.com").split(",") | |
| if d.strip()) | |
| # A real browser UA helps a little against bot checks; cookies are the actual fix. | |
| _UA = ("Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 " | |
| "(KHTML, like Gecko) Chrome/124.0.0.0 Safari/537.36") | |
| _cookie_path = None | |
| _cookie_lock = threading.Lock() | |
| _pool = ThreadPoolExecutor(max_workers=2, thread_name_prefix="nina-fetch") | |
| def _host_allowed(host: str) -> bool: | |
| host = (host or "").lower().rstrip(".") | |
| if host.startswith("www.") or host.startswith("m."): | |
| host = host.split(".", 1)[1] | |
| return any(host == h or host.endswith("." + h) for h in ALLOWED_HOSTS) | |
| def check_url(url: str) -> str | None: | |
| """Return None when the URL may be fetched, else a reason for the user. | |
| Checks: scheme, a known video host (no IP literals, no internal names), and that | |
| every address the host resolves to is a public one.""" | |
| try: | |
| u = urlsplit((url or "").strip()) | |
| except Exception: | |
| return "Please paste a valid video link (starting with http:// or https://)." | |
| if u.scheme not in ("http", "https") or not u.hostname: | |
| return "Please paste a valid video link (starting with http:// or https://)." | |
| if u.username or u.password or u.port not in (None, 80, 443): | |
| return "That link isn't supported. Please upload the file instead." | |
| host = u.hostname | |
| try: | |
| ipaddress.ip_address(host) | |
| return "That link isn't supported. Please upload the file instead." # IP literal | |
| except ValueError: | |
| pass | |
| if not _host_allowed(host): | |
| return "Links from that site aren't supported. Please upload the file instead." | |
| try: | |
| addrs = {ai[4][0] for ai in socket.getaddrinfo(host, None)} | |
| except Exception: | |
| return "That site can't be reached right now. Please upload the file instead." | |
| for a in addrs: | |
| ip = ipaddress.ip_address(a) | |
| if (ip.is_private or ip.is_loopback or ip.is_link_local or ip.is_reserved | |
| or ip.is_multicast or ip.is_unspecified): | |
| return "That link isn't supported. Please upload the file instead." | |
| return None | |
| def _cookiefile(): | |
| """YouTube/Facebook block datacenter IPs ('confirm you're not a bot' / age gate). | |
| Set a Space secret with an exported Netscape cookies.txt to authenticate yt-dlp: | |
| β’ YT_DLP_COOKIES_B64 β base64 of the cookies.txt (preferred; survives newlines), or | |
| β’ YT_DLP_COOKIES β the raw cookies.txt contents. | |
| Only lines for COOKIE_DOMAINS are kept. Returns a path, or None.""" | |
| global _cookie_path | |
| with _cookie_lock: | |
| if _cookie_path and os.path.exists(_cookie_path): | |
| return _cookie_path | |
| raw = None | |
| b64 = os.getenv("YT_DLP_COOKIES_B64") | |
| if b64: | |
| try: | |
| raw = base64.b64decode(b64).decode("utf-8", "replace") | |
| except Exception: | |
| raw = None | |
| if raw is None: | |
| raw = os.getenv("YT_DLP_COOKIES") | |
| if not raw or not raw.strip(): | |
| return None | |
| kept = [] | |
| for line in raw.splitlines(): | |
| if not line.strip() or line.startswith("#"): | |
| kept.append(line) | |
| continue | |
| dom = line.split("\t", 1)[0].lstrip(".").lower() | |
| if any(dom == d or dom.endswith("." + d) for d in COOKIE_DOMAINS): | |
| kept.append(line) | |
| try: | |
| fd, p = tempfile.mkstemp(prefix="nina_yt_cookies_", suffix=".txt") | |
| with os.fdopen(fd, "w", encoding="utf-8") as f: | |
| f.write("\n".join(kept) + "\n") | |
| os.chmod(p, 0o600) | |
| _cookie_path = p | |
| return p | |
| except Exception: | |
| return None | |
| def _opts(outdir: str) -> dict: | |
| import yt_dlp | |
| opts = { | |
| "format": YT_DLP_FORMAT, | |
| "outtmpl": os.path.join(outdir, "%(id)s.%(ext)s"), | |
| "noplaylist": True, | |
| "quiet": True, | |
| "no_warnings": True, | |
| "merge_output_format": "mp4", | |
| "socket_timeout": 30, | |
| "retries": 2, | |
| "concurrent_fragment_downloads": 4, | |
| "http_headers": {"User-Agent": _UA}, | |
| "allowed_extractors": ALLOWED_EXTRACTORS, | |
| "max_filesize": FETCH_MAX_BYTES, | |
| # No live streams, and nothing whose length is unknown: the duration cap must | |
| # be enforceable before the download starts. | |
| "match_filter": yt_dlp.utils.match_filter_func(f"!is_live & duration <= {MAX_FETCH_SECONDS}"), | |
| } | |
| cookies = _cookiefile() | |
| if cookies: | |
| opts["cookiefile"] = cookies | |
| proxy = os.getenv("YT_DLP_PROXY") | |
| if proxy: | |
| opts["proxy"] = proxy | |
| return opts, bool(cookies), bool(proxy) | |
| def _download(url: str, outdir: str): | |
| """Runs on the fetch pool. Returns (path, title) or raises.""" | |
| import yt_dlp | |
| opts, has_cookies, has_proxy = _opts(outdir) | |
| print(f"[NINA] fetch: cookies={'yes' if has_cookies else 'no'}, proxy={'yes' if has_proxy else 'no'}, " | |
| f"host={urlsplit(url).hostname}", flush=True) | |
| with yt_dlp.YoutubeDL(opts) as ydl: | |
| info = ydl.extract_info(url, download=False) | |
| if info and "entries" in info: # a playlist/channel β first item | |
| entries = [e for e in (info.get("entries") or []) if e] | |
| if not entries: | |
| raise ValueError("no playable video") | |
| info = entries[0] | |
| url = info.get("webpage_url") or info.get("url") or url | |
| dur = (info or {}).get("duration") | |
| if not dur: | |
| raise ValueError("unknown duration") | |
| if dur > MAX_FETCH_SECONDS: | |
| raise ValueError(f"too long: {int(dur // 60)} min") | |
| info = ydl.extract_info(url, download=True) | |
| if info and "entries" in info: | |
| entries = [e for e in (info.get("entries") or []) if e] | |
| if not entries: | |
| raise ValueError("no playable video") | |
| info = entries[0] | |
| path = ydl.prepare_filename(info) | |
| if not os.path.exists(path): # merged/remuxed β different ext | |
| base = os.path.splitext(path)[0] | |
| for ext in (".mp4", ".mkv", ".webm", ".m4a", ".mov"): | |
| if os.path.exists(base + ext): | |
| path = base + ext | |
| break | |
| if not os.path.exists(path): | |
| raise ValueError("no file produced") | |
| title = (info.get("title") or "Video from link").strip()[:200] | |
| print(f"[NINA] Fetched a link ({os.path.getsize(path) // (1024 * 1024)} MB).", flush=True) | |
| return path, title | |
| def fetch_video(url: str, token: str = ""): | |
| """Download `url` via yt-dlp. Returns (filepath, title) on success, or | |
| (None, error_message) for the website to show.""" | |
| if not verify_usage_token(token, "fetch"): | |
| return None, "Not authorized." | |
| url = (url or "").strip() | |
| why = check_url(url) | |
| if why: | |
| return None, why | |
| try: | |
| import yt_dlp # noqa: F401 | |
| except Exception: | |
| return None, "Link download isn't available right now β please upload the file instead." | |
| outdir = tempfile.mkdtemp(prefix="nina_url_") | |
| register_tmp(outdir) # swept by cache.evict_stale after the reuse window | |
| fut = _pool.submit(_download, url, outdir) | |
| try: | |
| return fut.result(timeout=FETCH_DEADLINE_SEC) | |
| except FutTimeout: | |
| # The thread keeps running; whatever it produces is deleted when it finishes. | |
| fut.add_done_callback(lambda f: shutil.rmtree(outdir, ignore_errors=True)) | |
| return None, "That download took too long. Please download the video yourself and upload the file." | |
| except Exception as e: | |
| shutil.rmtree(outdir, ignore_errors=True) | |
| msg = str(e) | |
| low = msg.lower() | |
| if low.startswith("too long"): | |
| return None, f"That video is {msg.split(':', 1)[1].strip()} long β the limit is {MAX_FETCH_SECONDS // 60} minutes." | |
| if "unknown duration" in low or "live" in low: | |
| return None, "Live streams and videos without a known length can't be fetched. Please upload the file instead." | |
| if "sign in" in low or "bot" in low or "confirm you" in low: | |
| return None, "This site is blocking downloads from our servers. Please download the video and upload the file instead." | |
| if "private" in low or "login" in low or "members-only" in low: | |
| return None, "That video looks private or login-only β please upload the file instead." | |
| if "too large" in low or "file is larger" in low: | |
| return None, "That video is too large to fetch. Please upload a smaller file instead." | |
| print(f"[NINA] fetch failed: {type(e).__name__}", flush=True) | |
| return None, "Couldn't fetch that link β it may be unavailable or unsupported. Please upload the file instead." | |