Spaces:
Running
Running
Fail closed when validated DNS address list is empty
Browse files- app/services/downloader.py +2 -94
app/services/downloader.py
CHANGED
|
@@ -29,7 +29,7 @@ class _PinnedNetworkBackend:
|
|
| 29 |
|
| 30 |
async def connect_tcp(self, host: str, port: int, timeout: float | None = None, local_address: str | None = None, socket_options=None):
|
| 31 |
pinned = self.downloader._pinned.get((host.lower().rstrip("."), port))
|
| 32 |
-
targets = pinned
|
| 33 |
last_error: Exception | None = None
|
| 34 |
for target in targets:
|
| 35 |
try:
|
|
@@ -91,96 +91,4 @@ class Downloader:
|
|
| 91 |
current_url = urljoin(current_url, location)
|
| 92 |
continue
|
| 93 |
response.raise_for_status()
|
| 94 |
-
|
| 95 |
-
if declared is not None and declared > self.settings.max_upload_size:
|
| 96 |
-
raise InputError("Remote media exceeds MAX_UPLOAD_SIZE")
|
| 97 |
-
name = self.validator.safe_filename(filename or self._response_filename(response, current_url), "download.bin")
|
| 98 |
-
destination = destination_dir / f"{uuid4().hex}_{name}"
|
| 99 |
-
size = 0
|
| 100 |
-
async with aiofiles.open(destination, "wb") as output:
|
| 101 |
-
async for chunk in response.aiter_bytes(1024 * 1024):
|
| 102 |
-
size += len(chunk)
|
| 103 |
-
if size > self.settings.max_upload_size:
|
| 104 |
-
raise InputError("Remote media exceeds MAX_UPLOAD_SIZE")
|
| 105 |
-
await output.write(chunk)
|
| 106 |
-
mime_type = response.headers.get("content-type", "").split(";", 1)[0]
|
| 107 |
-
mime_type = mime_type or self.validator.infer_mime(destination)
|
| 108 |
-
self.validator.validate_declared(name, mime_type, size)
|
| 109 |
-
return destination, mime_type
|
| 110 |
-
except httpx.HTTPStatusError as exc:
|
| 111 |
-
if destination is not None:
|
| 112 |
-
destination.unlink(missing_ok=True)
|
| 113 |
-
raise DownloadError("Remote server rejected the download", details={"status_code": exc.response.status_code}) from exc
|
| 114 |
-
except httpx.HTTPError as exc:
|
| 115 |
-
if destination is not None:
|
| 116 |
-
destination.unlink(missing_ok=True)
|
| 117 |
-
raise DownloadError("Unable to download remote media") from exc
|
| 118 |
-
except InputError:
|
| 119 |
-
if destination is not None:
|
| 120 |
-
destination.unlink(missing_ok=True)
|
| 121 |
-
raise
|
| 122 |
-
except OSError as exc:
|
| 123 |
-
if destination is not None:
|
| 124 |
-
destination.unlink(missing_ok=True)
|
| 125 |
-
raise DownloadError("Unable to store downloaded media") from exc
|
| 126 |
-
raise DownloadError("Too many redirects while downloading media")
|
| 127 |
-
|
| 128 |
-
async def validate_url(self, value: str) -> None:
|
| 129 |
-
try:
|
| 130 |
-
parsed = urlparse(value)
|
| 131 |
-
if (
|
| 132 |
-
parsed.scheme.lower() not in {"http", "https"} or not parsed.hostname
|
| 133 |
-
or parsed.username is not None or parsed.password is not None
|
| 134 |
-
or any(char.isspace() for char in parsed.hostname)
|
| 135 |
-
):
|
| 136 |
-
raise InputError("Only credential-free HTTP and HTTPS media URLs are supported")
|
| 137 |
-
port = parsed.port
|
| 138 |
-
except ValueError as exc:
|
| 139 |
-
raise InputError("Media URL contains an invalid host or port") from exc
|
| 140 |
-
|
| 141 |
-
hostname = parsed.hostname.encode("idna").decode("ascii").lower().rstrip(".")
|
| 142 |
-
target_port = port or (443 if parsed.scheme.lower() == "https" else 80)
|
| 143 |
-
if self.settings.allow_private_urls:
|
| 144 |
-
# Explicit opt-in permits private hosts; still pin the current DNS
|
| 145 |
-
# result to prevent a second, inconsistent lookup by the HTTP client.
|
| 146 |
-
pass
|
| 147 |
-
try:
|
| 148 |
-
records = await asyncio.to_thread(socket.getaddrinfo, hostname, target_port, type=socket.SOCK_STREAM)
|
| 149 |
-
except (socket.gaierror, OSError) as exc:
|
| 150 |
-
raise DownloadError("The remote hostname could not be resolved") from exc
|
| 151 |
-
if not records:
|
| 152 |
-
raise DownloadError("The remote hostname has no usable address")
|
| 153 |
-
addresses: list[str] = []
|
| 154 |
-
for record in records:
|
| 155 |
-
try:
|
| 156 |
-
address = ipaddress.ip_address(record[4][0].split("%", 1)[0])
|
| 157 |
-
except (ValueError, IndexError, TypeError) as exc:
|
| 158 |
-
raise DownloadError("The remote hostname resolved to an invalid address") from exc
|
| 159 |
-
if not self.settings.allow_private_urls and not address.is_global:
|
| 160 |
-
raise InputError("Private, loopback, and link-local URLs are not allowed")
|
| 161 |
-
if str(address) not in addresses:
|
| 162 |
-
addresses.append(str(address))
|
| 163 |
-
self._pinned[(hostname, target_port)] = tuple(addresses)
|
| 164 |
-
|
| 165 |
-
@staticmethod
|
| 166 |
-
def _response_filename(response: httpx.Response, url: str) -> str:
|
| 167 |
-
disposition = response.headers.get("content-disposition", "")
|
| 168 |
-
# Parse common filename and RFC 5987 filename* Content-Disposition values.
|
| 169 |
-
extended = re.search(r"""filename\*\s*=\s*UTF-8''([^;]+)""", disposition, re.I)
|
| 170 |
-
ordinary = re.search(r"""filename\s*=\s*(?:"([^"]*)"|([^;]+))""", disposition, re.I)
|
| 171 |
-
if extended:
|
| 172 |
-
return unquote(extended.group(1).strip().strip('"'))
|
| 173 |
-
if ordinary:
|
| 174 |
-
return unquote((ordinary.group(1) or ordinary.group(2) or "").strip())
|
| 175 |
-
path_name = Path(unquote(urlparse(url).path)).name
|
| 176 |
-
return path_name or "download.bin"
|
| 177 |
-
|
| 178 |
-
@staticmethod
|
| 179 |
-
def _content_length(value: str | None) -> int | None:
|
| 180 |
-
if value is None:
|
| 181 |
-
return None
|
| 182 |
-
try:
|
| 183 |
-
parsed = int(value)
|
| 184 |
-
except (TypeError, ValueError):
|
| 185 |
-
return None
|
| 186 |
-
return parsed if parsed >= 0 else None
|
|
|
|
| 29 |
|
| 30 |
async def connect_tcp(self, host: str, port: int, timeout: float | None = None, local_address: str | None = None, socket_options=None):
|
| 31 |
pinned = self.downloader._pinned.get((host.lower().rstrip("."), port))
|
| 32 |
+
targets = pinned if pinned is not None else (host,)
|
| 33 |
last_error: Exception | None = None
|
| 34 |
for target in targets:
|
| 35 |
try:
|
|
|
|
| 91 |
current_url = urljoin(current_url, location)
|
| 92 |
continue
|
| 93 |
response.raise_for_status()
|
| 94 |
+
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|