basyx commited on
Commit
5eedda5
·
verified ·
1 Parent(s): 4b33deb

Fail closed when validated DNS address list is empty

Browse files
Files changed (1) hide show
  1. app/services/downloader.py +2 -94
app/services/downloader.py CHANGED
@@ -29,7 +29,7 @@ class _PinnedNetworkBackend:
29
 
30
  async def connect_tcp(self, host: str, port: int, timeout: float | None = None, local_address: str | None = None, socket_options=None):
31
  pinned = self.downloader._pinned.get((host.lower().rstrip("."), port))
32
- targets = pinned or (host,)
33
  last_error: Exception | None = None
34
  for target in targets:
35
  try:
@@ -91,96 +91,4 @@ class Downloader:
91
  current_url = urljoin(current_url, location)
92
  continue
93
  response.raise_for_status()
94
- declared = self._content_length(response.headers.get("content-length"))
95
- if declared is not None and declared > self.settings.max_upload_size:
96
- raise InputError("Remote media exceeds MAX_UPLOAD_SIZE")
97
- name = self.validator.safe_filename(filename or self._response_filename(response, current_url), "download.bin")
98
- destination = destination_dir / f"{uuid4().hex}_{name}"
99
- size = 0
100
- async with aiofiles.open(destination, "wb") as output:
101
- async for chunk in response.aiter_bytes(1024 * 1024):
102
- size += len(chunk)
103
- if size > self.settings.max_upload_size:
104
- raise InputError("Remote media exceeds MAX_UPLOAD_SIZE")
105
- await output.write(chunk)
106
- mime_type = response.headers.get("content-type", "").split(";", 1)[0]
107
- mime_type = mime_type or self.validator.infer_mime(destination)
108
- self.validator.validate_declared(name, mime_type, size)
109
- return destination, mime_type
110
- except httpx.HTTPStatusError as exc:
111
- if destination is not None:
112
- destination.unlink(missing_ok=True)
113
- raise DownloadError("Remote server rejected the download", details={"status_code": exc.response.status_code}) from exc
114
- except httpx.HTTPError as exc:
115
- if destination is not None:
116
- destination.unlink(missing_ok=True)
117
- raise DownloadError("Unable to download remote media") from exc
118
- except InputError:
119
- if destination is not None:
120
- destination.unlink(missing_ok=True)
121
- raise
122
- except OSError as exc:
123
- if destination is not None:
124
- destination.unlink(missing_ok=True)
125
- raise DownloadError("Unable to store downloaded media") from exc
126
- raise DownloadError("Too many redirects while downloading media")
127
-
128
- async def validate_url(self, value: str) -> None:
129
- try:
130
- parsed = urlparse(value)
131
- if (
132
- parsed.scheme.lower() not in {"http", "https"} or not parsed.hostname
133
- or parsed.username is not None or parsed.password is not None
134
- or any(char.isspace() for char in parsed.hostname)
135
- ):
136
- raise InputError("Only credential-free HTTP and HTTPS media URLs are supported")
137
- port = parsed.port
138
- except ValueError as exc:
139
- raise InputError("Media URL contains an invalid host or port") from exc
140
-
141
- hostname = parsed.hostname.encode("idna").decode("ascii").lower().rstrip(".")
142
- target_port = port or (443 if parsed.scheme.lower() == "https" else 80)
143
- if self.settings.allow_private_urls:
144
- # Explicit opt-in permits private hosts; still pin the current DNS
145
- # result to prevent a second, inconsistent lookup by the HTTP client.
146
- pass
147
- try:
148
- records = await asyncio.to_thread(socket.getaddrinfo, hostname, target_port, type=socket.SOCK_STREAM)
149
- except (socket.gaierror, OSError) as exc:
150
- raise DownloadError("The remote hostname could not be resolved") from exc
151
- if not records:
152
- raise DownloadError("The remote hostname has no usable address")
153
- addresses: list[str] = []
154
- for record in records:
155
- try:
156
- address = ipaddress.ip_address(record[4][0].split("%", 1)[0])
157
- except (ValueError, IndexError, TypeError) as exc:
158
- raise DownloadError("The remote hostname resolved to an invalid address") from exc
159
- if not self.settings.allow_private_urls and not address.is_global:
160
- raise InputError("Private, loopback, and link-local URLs are not allowed")
161
- if str(address) not in addresses:
162
- addresses.append(str(address))
163
- self._pinned[(hostname, target_port)] = tuple(addresses)
164
-
165
- @staticmethod
166
- def _response_filename(response: httpx.Response, url: str) -> str:
167
- disposition = response.headers.get("content-disposition", "")
168
- # Parse common filename and RFC 5987 filename* Content-Disposition values.
169
- extended = re.search(r"""filename\*\s*=\s*UTF-8''([^;]+)""", disposition, re.I)
170
- ordinary = re.search(r"""filename\s*=\s*(?:"([^"]*)"|([^;]+))""", disposition, re.I)
171
- if extended:
172
- return unquote(extended.group(1).strip().strip('"'))
173
- if ordinary:
174
- return unquote((ordinary.group(1) or ordinary.group(2) or "").strip())
175
- path_name = Path(unquote(urlparse(url).path)).name
176
- return path_name or "download.bin"
177
-
178
- @staticmethod
179
- def _content_length(value: str | None) -> int | None:
180
- if value is None:
181
- return None
182
- try:
183
- parsed = int(value)
184
- except (TypeError, ValueError):
185
- return None
186
- return parsed if parsed >= 0 else None
 
29
 
30
  async def connect_tcp(self, host: str, port: int, timeout: float | None = None, local_address: str | None = None, socket_options=None):
31
  pinned = self.downloader._pinned.get((host.lower().rstrip("."), port))
32
+ targets = pinned if pinned is not None else (host,)
33
  last_error: Exception | None = None
34
  for target in targets:
35
  try:
 
91
  current_url = urljoin(current_url, location)
92
  continue
93
  response.raise_for_status()
94
+