Spaces:
Running
Running
Tighten SSRF checks and remote download cleanup
Browse files- app/services/downloader.py +43 -19
app/services/downloader.py
CHANGED
|
@@ -29,14 +29,16 @@ class Downloader:
|
|
| 29 |
current_url = url
|
| 30 |
timeout = httpx.Timeout(self.settings.download_timeout_seconds, connect=30.0)
|
| 31 |
headers = {"User-Agent": "MediaAPI/1.0"}
|
| 32 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
| 33 |
for _ in range(6):
|
| 34 |
await self.validate_url(current_url)
|
| 35 |
destination: Path | None = None
|
| 36 |
try:
|
| 37 |
-
async with client.stream(
|
| 38 |
-
"GET", current_url, follow_redirects=False
|
| 39 |
-
) as response:
|
| 40 |
if response.status_code in {301, 302, 303, 307, 308}:
|
| 41 |
location = response.headers.get("location")
|
| 42 |
if not location:
|
|
@@ -45,21 +47,18 @@ class Downloader:
|
|
| 45 |
continue
|
| 46 |
response.raise_for_status()
|
| 47 |
declared = self._content_length(response.headers.get("content-length"))
|
| 48 |
-
if declared and declared > self.settings.max_upload_size:
|
| 49 |
raise InputError("Remote media exceeds MAX_UPLOAD_SIZE")
|
| 50 |
name = self.validator.safe_filename(
|
| 51 |
filename or self._response_filename(response, current_url),
|
| 52 |
"download.bin",
|
| 53 |
)
|
| 54 |
-
|
| 55 |
-
destination = destination_dir / unique_name
|
| 56 |
size = 0
|
| 57 |
async with aiofiles.open(destination, "wb") as output:
|
| 58 |
async for chunk in response.aiter_bytes(1024 * 1024):
|
| 59 |
size += len(chunk)
|
| 60 |
if size > self.settings.max_upload_size:
|
| 61 |
-
await output.close()
|
| 62 |
-
destination.unlink(missing_ok=True)
|
| 63 |
raise InputError("Remote media exceeds MAX_UPLOAD_SIZE")
|
| 64 |
await output.write(chunk)
|
| 65 |
mime_type = response.headers.get("content-type", "").split(";", 1)[0]
|
|
@@ -88,21 +87,43 @@ class Downloader:
|
|
| 88 |
raise DownloadError("Too many redirects while downloading media")
|
| 89 |
|
| 90 |
async def validate_url(self, value: str) -> None:
|
| 91 |
-
|
| 92 |
-
|
| 93 |
-
|
| 94 |
-
|
| 95 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 96 |
if self.settings.allow_private_urls:
|
| 97 |
return
|
|
|
|
|
|
|
| 98 |
try:
|
|
|
|
|
|
|
|
|
|
| 99 |
records = await asyncio.to_thread(
|
| 100 |
-
socket.getaddrinfo,
|
|
|
|
|
|
|
|
|
|
| 101 |
)
|
| 102 |
-
except socket.gaierror as exc:
|
| 103 |
raise DownloadError("The remote hostname could not be resolved") from exc
|
|
|
|
|
|
|
| 104 |
for record in records:
|
| 105 |
-
|
|
|
|
|
|
|
|
|
|
| 106 |
if not address.is_global:
|
| 107 |
raise InputError("Private, loopback, and link-local URLs are not allowed")
|
| 108 |
|
|
@@ -117,7 +138,10 @@ class Downloader:
|
|
| 117 |
|
| 118 |
@staticmethod
|
| 119 |
def _content_length(value: str | None) -> int | None:
|
|
|
|
|
|
|
| 120 |
try:
|
| 121 |
-
|
| 122 |
-
except ValueError:
|
| 123 |
return None
|
|
|
|
|
|
| 29 |
current_url = url
|
| 30 |
timeout = httpx.Timeout(self.settings.download_timeout_seconds, connect=30.0)
|
| 31 |
headers = {"User-Agent": "MediaAPI/1.0"}
|
| 32 |
+
# Do not silently route a security-sensitive fetch through ambient
|
| 33 |
+
# HTTP(S)_PROXY settings that can change the destination semantics.
|
| 34 |
+
async with httpx.AsyncClient(
|
| 35 |
+
timeout=timeout, headers=headers, follow_redirects=False, trust_env=False
|
| 36 |
+
) as client:
|
| 37 |
for _ in range(6):
|
| 38 |
await self.validate_url(current_url)
|
| 39 |
destination: Path | None = None
|
| 40 |
try:
|
| 41 |
+
async with client.stream("GET", current_url, follow_redirects=False) as response:
|
|
|
|
|
|
|
| 42 |
if response.status_code in {301, 302, 303, 307, 308}:
|
| 43 |
location = response.headers.get("location")
|
| 44 |
if not location:
|
|
|
|
| 47 |
continue
|
| 48 |
response.raise_for_status()
|
| 49 |
declared = self._content_length(response.headers.get("content-length"))
|
| 50 |
+
if declared is not None and declared > self.settings.max_upload_size:
|
| 51 |
raise InputError("Remote media exceeds MAX_UPLOAD_SIZE")
|
| 52 |
name = self.validator.safe_filename(
|
| 53 |
filename or self._response_filename(response, current_url),
|
| 54 |
"download.bin",
|
| 55 |
)
|
| 56 |
+
destination = destination_dir / f"{uuid4().hex}_{name}"
|
|
|
|
| 57 |
size = 0
|
| 58 |
async with aiofiles.open(destination, "wb") as output:
|
| 59 |
async for chunk in response.aiter_bytes(1024 * 1024):
|
| 60 |
size += len(chunk)
|
| 61 |
if size > self.settings.max_upload_size:
|
|
|
|
|
|
|
| 62 |
raise InputError("Remote media exceeds MAX_UPLOAD_SIZE")
|
| 63 |
await output.write(chunk)
|
| 64 |
mime_type = response.headers.get("content-type", "").split(";", 1)[0]
|
|
|
|
| 87 |
raise DownloadError("Too many redirects while downloading media")
|
| 88 |
|
| 89 |
async def validate_url(self, value: str) -> None:
|
| 90 |
+
try:
|
| 91 |
+
parsed = urlparse(value)
|
| 92 |
+
if (
|
| 93 |
+
parsed.scheme.lower() not in {"http", "https"}
|
| 94 |
+
or not parsed.hostname
|
| 95 |
+
or parsed.username is not None
|
| 96 |
+
or parsed.password is not None
|
| 97 |
+
or any(char.isspace() for char in parsed.hostname)
|
| 98 |
+
):
|
| 99 |
+
raise InputError("Only credential-free HTTP and HTTPS media URLs are supported")
|
| 100 |
+
port = parsed.port
|
| 101 |
+
except ValueError as exc:
|
| 102 |
+
raise InputError("Media URL contains an invalid host or port") from exc
|
| 103 |
+
|
| 104 |
if self.settings.allow_private_urls:
|
| 105 |
return
|
| 106 |
+
|
| 107 |
+
target_port = port or (443 if parsed.scheme.lower() == "https" else 80)
|
| 108 |
try:
|
| 109 |
+
# Reject the entire hostname if any resolved address is non-global.
|
| 110 |
+
# This blocks mixed public/private DNS answers rather than trusting
|
| 111 |
+
# the first address returned by the resolver.
|
| 112 |
records = await asyncio.to_thread(
|
| 113 |
+
socket.getaddrinfo,
|
| 114 |
+
parsed.hostname,
|
| 115 |
+
target_port,
|
| 116 |
+
type=socket.SOCK_STREAM,
|
| 117 |
)
|
| 118 |
+
except (socket.gaierror, OSError) as exc:
|
| 119 |
raise DownloadError("The remote hostname could not be resolved") from exc
|
| 120 |
+
if not records:
|
| 121 |
+
raise DownloadError("The remote hostname has no usable address")
|
| 122 |
for record in records:
|
| 123 |
+
try:
|
| 124 |
+
address = ipaddress.ip_address(record[4][0].split("%", 1)[0])
|
| 125 |
+
except (ValueError, IndexError, TypeError) as exc:
|
| 126 |
+
raise DownloadError("The remote hostname resolved to an invalid address") from exc
|
| 127 |
if not address.is_global:
|
| 128 |
raise InputError("Private, loopback, and link-local URLs are not allowed")
|
| 129 |
|
|
|
|
| 138 |
|
| 139 |
@staticmethod
|
| 140 |
def _content_length(value: str | None) -> int | None:
|
| 141 |
+
if value is None:
|
| 142 |
+
return None
|
| 143 |
try:
|
| 144 |
+
parsed = int(value)
|
| 145 |
+
except (TypeError, ValueError):
|
| 146 |
return None
|
| 147 |
+
return parsed if parsed >= 0 else None
|