File size: 11,208 Bytes
51e7ae9
 
 
1d697a7
 
 
 
 
 
 
 
 
 
51e7ae9
d0ff022
1d697a7
51e7ae9
1d697a7
 
51e7ae9
1d697a7
 
 
51e7ae9
 
1d697a7
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
d0ff022
 
 
 
 
1d697a7
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
d0ff022
 
 
 
 
 
 
1d697a7
d0ff022
1d697a7
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
d0ff022
1d697a7
 
 
 
 
 
d0ff022
1d697a7
51e7ae9
 
1d697a7
 
51e7ae9
 
 
 
 
 
 
 
 
 
d0ff022
1d697a7
 
 
 
 
51e7ae9
d0ff022
 
 
8a9ebda
 
 
1d697a7
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
51e7ae9
1d697a7
 
 
 
51e7ae9
1d697a7
 
 
51e7ae9
1d697a7
 
 
 
 
 
 
 
 
51e7ae9
1d697a7
51e7ae9
 
1d697a7
 
 
 
51e7ae9
 
 
 
1d697a7
 
 
51e7ae9
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
# ─────────────────────────────────────────────
#  NINA β€” pasted-link ingestion (yt-dlp)
# ─────────────────────────────────────────────
#  Downloads a pasted video URL from a KNOWN video site so the normal pipeline can
#  analyze it. Best-effort: cloud/Space IPs are frequently blocked (esp. YouTube's
#  "confirm you're not a bot"), so the website always offers file upload as the
#  fallback. We pull a small H.264 stream (NINA only needs audio + sampled frames),
#  cap the duration and the size, give the download a hard deadline, and clean up.
#
#  What this is NOT: a generic downloader. The generic extractor is disabled and the
#  host is resolved and checked before yt-dlp sees it, so a pasted link cannot make
#  this Space fetch an internal address, and the operator's cookies only ever travel
#  to the sites they were exported for.

import base64
import ipaddress
import os
import shutil
import socket
import tempfile
import threading
from concurrent.futures import ThreadPoolExecutor, TimeoutError as FutTimeout
from urllib.parse import urlsplit

from auth import verify_usage_token
from cache import register_tmp
from config import FETCH_DEADLINE_SEC, FETCH_MAX_BYTES, MAX_VIDEO_SECONDS

MAX_FETCH_SECONDS = MAX_VIDEO_SECONDS
# H.264 + AAC first: anything else makes Gradio re-encode the whole file after the
# handler returns, on this Space's CPU, usually past the website's deadline.
YT_DLP_FORMAT = ("bv*[height<=480][vcodec^=avc1]+ba[acodec^=mp4a]/b[height<=480][ext=mp4]/"
                 "bv*[height<=480]+ba/b[height<=480]/best")
# The sites the website offers. yt-dlp's generic extractor (any URL that serves a
# media file) stays OFF.
ALLOWED_EXTRACTORS = ["youtube", "youtube:tab", "vimeo", "facebook", "tiktok", "twitter",
                      "instagram", "generic:unsupported"]
ALLOWED_HOSTS = ("youtube.com", "youtu.be", "vimeo.com", "facebook.com", "fb.watch",
                 "tiktok.com", "x.com", "twitter.com", "instagram.com")
# Cookies (if configured) are only kept for these domains.
COOKIE_DOMAINS = tuple(d.strip().lower() for d in
                       os.getenv("YT_DLP_COOKIE_DOMAINS", "youtube.com,google.com,facebook.com").split(",")
                       if d.strip())
# A real browser UA helps a little against bot checks; cookies are the actual fix.
_UA = ("Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 "
       "(KHTML, like Gecko) Chrome/124.0.0.0 Safari/537.36")

_cookie_path = None
_cookie_lock = threading.Lock()
_pool = ThreadPoolExecutor(max_workers=2, thread_name_prefix="nina-fetch")


def _host_allowed(host: str) -> bool:
    host = (host or "").lower().rstrip(".")
    if host.startswith("www.") or host.startswith("m."):
        host = host.split(".", 1)[1]
    return any(host == h or host.endswith("." + h) for h in ALLOWED_HOSTS)


def check_url(url: str) -> str | None:
    """Return None when the URL may be fetched, else a reason for the user.

    Checks: scheme, a known video host (no IP literals, no internal names), and that
    every address the host resolves to is a public one."""
    try:
        u = urlsplit((url or "").strip())
    except Exception:
        return "Please paste a valid video link (starting with http:// or https://)."
    if u.scheme not in ("http", "https") or not u.hostname:
        return "Please paste a valid video link (starting with http:// or https://)."
    if u.username or u.password or u.port not in (None, 80, 443):
        return "That link isn't supported. Please upload the file instead."
    host = u.hostname
    try:
        ipaddress.ip_address(host)
        return "That link isn't supported. Please upload the file instead."   # IP literal
    except ValueError:
        pass
    if not _host_allowed(host):
        return "Links from that site aren't supported. Please upload the file instead."
    try:
        addrs = {ai[4][0] for ai in socket.getaddrinfo(host, None)}
    except Exception:
        return "That site can't be reached right now. Please upload the file instead."
    for a in addrs:
        ip = ipaddress.ip_address(a)
        if (ip.is_private or ip.is_loopback or ip.is_link_local or ip.is_reserved
                or ip.is_multicast or ip.is_unspecified):
            return "That link isn't supported. Please upload the file instead."
    return None


def _cookiefile():
    """YouTube/Facebook block datacenter IPs ('confirm you're not a bot' / age gate).
    Set a Space secret with an exported Netscape cookies.txt to authenticate yt-dlp:
      β€’ YT_DLP_COOKIES_B64 β€” base64 of the cookies.txt (preferred; survives newlines), or
      β€’ YT_DLP_COOKIES     β€” the raw cookies.txt contents.
    Only lines for COOKIE_DOMAINS are kept. Returns a path, or None."""
    global _cookie_path
    with _cookie_lock:
        if _cookie_path and os.path.exists(_cookie_path):
            return _cookie_path
        raw = None
        b64 = os.getenv("YT_DLP_COOKIES_B64")
        if b64:
            try:
                raw = base64.b64decode(b64).decode("utf-8", "replace")
            except Exception:
                raw = None
        if raw is None:
            raw = os.getenv("YT_DLP_COOKIES")
        if not raw or not raw.strip():
            return None
        kept = []
        for line in raw.splitlines():
            if not line.strip() or line.startswith("#"):
                kept.append(line)
                continue
            dom = line.split("\t", 1)[0].lstrip(".").lower()
            if any(dom == d or dom.endswith("." + d) for d in COOKIE_DOMAINS):
                kept.append(line)
        try:
            fd, p = tempfile.mkstemp(prefix="nina_yt_cookies_", suffix=".txt")
            with os.fdopen(fd, "w", encoding="utf-8") as f:
                f.write("\n".join(kept) + "\n")
            os.chmod(p, 0o600)
            _cookie_path = p
            return p
        except Exception:
            return None


def _opts(outdir: str) -> dict:
    import yt_dlp
    opts = {
        "format": YT_DLP_FORMAT,
        "outtmpl": os.path.join(outdir, "%(id)s.%(ext)s"),
        "noplaylist": True,
        "quiet": True,
        "no_warnings": True,
        "merge_output_format": "mp4",
        "socket_timeout": 30,
        "retries": 2,
        "concurrent_fragment_downloads": 4,
        "http_headers": {"User-Agent": _UA},
        "allowed_extractors": ALLOWED_EXTRACTORS,
        "max_filesize": FETCH_MAX_BYTES,
        # No live streams, and nothing whose length is unknown: the duration cap must
        # be enforceable before the download starts.
        "match_filter": yt_dlp.utils.match_filter_func(f"!is_live & duration <= {MAX_FETCH_SECONDS}"),
    }
    cookies = _cookiefile()
    if cookies:
        opts["cookiefile"] = cookies
    proxy = os.getenv("YT_DLP_PROXY")
    if proxy:
        opts["proxy"] = proxy
    return opts, bool(cookies), bool(proxy)


def _download(url: str, outdir: str):
    """Runs on the fetch pool. Returns (path, title) or raises."""
    import yt_dlp
    opts, has_cookies, has_proxy = _opts(outdir)
    print(f"[NINA] fetch: cookies={'yes' if has_cookies else 'no'}, proxy={'yes' if has_proxy else 'no'}, "
          f"host={urlsplit(url).hostname}", flush=True)
    with yt_dlp.YoutubeDL(opts) as ydl:
        info = ydl.extract_info(url, download=False)
        if info and "entries" in info:                 # a playlist/channel β†’ first item
            entries = [e for e in (info.get("entries") or []) if e]
            if not entries:
                raise ValueError("no playable video")
            info = entries[0]
            url = info.get("webpage_url") or info.get("url") or url
        dur = (info or {}).get("duration")
        if not dur:
            raise ValueError("unknown duration")
        if dur > MAX_FETCH_SECONDS:
            raise ValueError(f"too long: {int(dur // 60)} min")
        info = ydl.extract_info(url, download=True)
        if info and "entries" in info:
            entries = [e for e in (info.get("entries") or []) if e]
            if not entries:
                raise ValueError("no playable video")
            info = entries[0]
        path = ydl.prepare_filename(info)
        if not os.path.exists(path):                   # merged/remuxed β†’ different ext
            base = os.path.splitext(path)[0]
            for ext in (".mp4", ".mkv", ".webm", ".m4a", ".mov"):
                if os.path.exists(base + ext):
                    path = base + ext
                    break
        if not os.path.exists(path):
            raise ValueError("no file produced")
        title = (info.get("title") or "Video from link").strip()[:200]
        print(f"[NINA] Fetched a link ({os.path.getsize(path) // (1024 * 1024)} MB).", flush=True)
        return path, title


def fetch_video(url: str, token: str = ""):
    """Download `url` via yt-dlp. Returns (filepath, title) on success, or
    (None, error_message) for the website to show."""
    if not verify_usage_token(token, "fetch"):
        return None, "Not authorized."

    url = (url or "").strip()
    why = check_url(url)
    if why:
        return None, why
    try:
        import yt_dlp  # noqa: F401
    except Exception:
        return None, "Link download isn't available right now β€” please upload the file instead."

    outdir = tempfile.mkdtemp(prefix="nina_url_")
    register_tmp(outdir)          # swept by cache.evict_stale after the reuse window
    fut = _pool.submit(_download, url, outdir)
    try:
        return fut.result(timeout=FETCH_DEADLINE_SEC)
    except FutTimeout:
        # The thread keeps running; whatever it produces is deleted when it finishes.
        fut.add_done_callback(lambda f: shutil.rmtree(outdir, ignore_errors=True))
        return None, "That download took too long. Please download the video yourself and upload the file."
    except Exception as e:
        shutil.rmtree(outdir, ignore_errors=True)
        msg = str(e)
        low = msg.lower()
        if low.startswith("too long"):
            return None, f"That video is {msg.split(':', 1)[1].strip()} long β€” the limit is {MAX_FETCH_SECONDS // 60} minutes."
        if "unknown duration" in low or "live" in low:
            return None, "Live streams and videos without a known length can't be fetched. Please upload the file instead."
        if "sign in" in low or "bot" in low or "confirm you" in low:
            return None, "This site is blocking downloads from our servers. Please download the video and upload the file instead."
        if "private" in low or "login" in low or "members-only" in low:
            return None, "That video looks private or login-only β€” please upload the file instead."
        if "too large" in low or "file is larger" in low:
            return None, "That video is too large to fetch. Please upload a smaller file instead."
        print(f"[NINA] fetch failed: {type(e).__name__}", flush=True)
        return None, "Couldn't fetch that link β€” it may be unavailable or unsupported. Please upload the file instead."