basyx commited on
Commit
d3b4950
·
verified ·
1 Parent(s): fece13c

Tighten SSRF checks and remote download cleanup

Browse files
Files changed (1) hide show
  1. app/services/downloader.py +43 -19
app/services/downloader.py CHANGED
@@ -29,14 +29,16 @@ class Downloader:
29
  current_url = url
30
  timeout = httpx.Timeout(self.settings.download_timeout_seconds, connect=30.0)
31
  headers = {"User-Agent": "MediaAPI/1.0"}
32
- async with httpx.AsyncClient(timeout=timeout, headers=headers) as client:
 
 
 
 
33
  for _ in range(6):
34
  await self.validate_url(current_url)
35
  destination: Path | None = None
36
  try:
37
- async with client.stream(
38
- "GET", current_url, follow_redirects=False
39
- ) as response:
40
  if response.status_code in {301, 302, 303, 307, 308}:
41
  location = response.headers.get("location")
42
  if not location:
@@ -45,21 +47,18 @@ class Downloader:
45
  continue
46
  response.raise_for_status()
47
  declared = self._content_length(response.headers.get("content-length"))
48
- if declared and declared > self.settings.max_upload_size:
49
  raise InputError("Remote media exceeds MAX_UPLOAD_SIZE")
50
  name = self.validator.safe_filename(
51
  filename or self._response_filename(response, current_url),
52
  "download.bin",
53
  )
54
- unique_name = f"{uuid4().hex}_{name}"
55
- destination = destination_dir / unique_name
56
  size = 0
57
  async with aiofiles.open(destination, "wb") as output:
58
  async for chunk in response.aiter_bytes(1024 * 1024):
59
  size += len(chunk)
60
  if size > self.settings.max_upload_size:
61
- await output.close()
62
- destination.unlink(missing_ok=True)
63
  raise InputError("Remote media exceeds MAX_UPLOAD_SIZE")
64
  await output.write(chunk)
65
  mime_type = response.headers.get("content-type", "").split(";", 1)[0]
@@ -88,21 +87,43 @@ class Downloader:
88
  raise DownloadError("Too many redirects while downloading media")
89
 
90
  async def validate_url(self, value: str) -> None:
91
- parsed = urlparse(value)
92
- if parsed.scheme not in {"http", "https"} or not parsed.hostname:
93
- raise InputError("Only HTTP and HTTPS media URLs are supported")
94
- if parsed.username or parsed.password:
95
- raise InputError("URLs containing credentials are not allowed")
 
 
 
 
 
 
 
 
 
96
  if self.settings.allow_private_urls:
97
  return
 
 
98
  try:
 
 
 
99
  records = await asyncio.to_thread(
100
- socket.getaddrinfo, parsed.hostname, parsed.port or 443, type=socket.SOCK_STREAM
 
 
 
101
  )
102
- except socket.gaierror as exc:
103
  raise DownloadError("The remote hostname could not be resolved") from exc
 
 
104
  for record in records:
105
- address = ipaddress.ip_address(record[4][0])
 
 
 
106
  if not address.is_global:
107
  raise InputError("Private, loopback, and link-local URLs are not allowed")
108
 
@@ -117,7 +138,10 @@ class Downloader:
117
 
118
  @staticmethod
119
  def _content_length(value: str | None) -> int | None:
 
 
120
  try:
121
- return int(value) if value else None
122
- except ValueError:
123
  return None
 
 
29
  current_url = url
30
  timeout = httpx.Timeout(self.settings.download_timeout_seconds, connect=30.0)
31
  headers = {"User-Agent": "MediaAPI/1.0"}
32
+ # Do not silently route a security-sensitive fetch through ambient
33
+ # HTTP(S)_PROXY settings that can change the destination semantics.
34
+ async with httpx.AsyncClient(
35
+ timeout=timeout, headers=headers, follow_redirects=False, trust_env=False
36
+ ) as client:
37
  for _ in range(6):
38
  await self.validate_url(current_url)
39
  destination: Path | None = None
40
  try:
41
+ async with client.stream("GET", current_url, follow_redirects=False) as response:
 
 
42
  if response.status_code in {301, 302, 303, 307, 308}:
43
  location = response.headers.get("location")
44
  if not location:
 
47
  continue
48
  response.raise_for_status()
49
  declared = self._content_length(response.headers.get("content-length"))
50
+ if declared is not None and declared > self.settings.max_upload_size:
51
  raise InputError("Remote media exceeds MAX_UPLOAD_SIZE")
52
  name = self.validator.safe_filename(
53
  filename or self._response_filename(response, current_url),
54
  "download.bin",
55
  )
56
+ destination = destination_dir / f"{uuid4().hex}_{name}"
 
57
  size = 0
58
  async with aiofiles.open(destination, "wb") as output:
59
  async for chunk in response.aiter_bytes(1024 * 1024):
60
  size += len(chunk)
61
  if size > self.settings.max_upload_size:
 
 
62
  raise InputError("Remote media exceeds MAX_UPLOAD_SIZE")
63
  await output.write(chunk)
64
  mime_type = response.headers.get("content-type", "").split(";", 1)[0]
 
87
  raise DownloadError("Too many redirects while downloading media")
88
 
89
  async def validate_url(self, value: str) -> None:
90
+ try:
91
+ parsed = urlparse(value)
92
+ if (
93
+ parsed.scheme.lower() not in {"http", "https"}
94
+ or not parsed.hostname
95
+ or parsed.username is not None
96
+ or parsed.password is not None
97
+ or any(char.isspace() for char in parsed.hostname)
98
+ ):
99
+ raise InputError("Only credential-free HTTP and HTTPS media URLs are supported")
100
+ port = parsed.port
101
+ except ValueError as exc:
102
+ raise InputError("Media URL contains an invalid host or port") from exc
103
+
104
  if self.settings.allow_private_urls:
105
  return
106
+
107
+ target_port = port or (443 if parsed.scheme.lower() == "https" else 80)
108
  try:
109
+ # Reject the entire hostname if any resolved address is non-global.
110
+ # This blocks mixed public/private DNS answers rather than trusting
111
+ # the first address returned by the resolver.
112
  records = await asyncio.to_thread(
113
+ socket.getaddrinfo,
114
+ parsed.hostname,
115
+ target_port,
116
+ type=socket.SOCK_STREAM,
117
  )
118
+ except (socket.gaierror, OSError) as exc:
119
  raise DownloadError("The remote hostname could not be resolved") from exc
120
+ if not records:
121
+ raise DownloadError("The remote hostname has no usable address")
122
  for record in records:
123
+ try:
124
+ address = ipaddress.ip_address(record[4][0].split("%", 1)[0])
125
+ except (ValueError, IndexError, TypeError) as exc:
126
+ raise DownloadError("The remote hostname resolved to an invalid address") from exc
127
  if not address.is_global:
128
  raise InputError("Private, loopback, and link-local URLs are not allowed")
129
 
 
138
 
139
  @staticmethod
140
  def _content_length(value: str | None) -> int | None:
141
+ if value is None:
142
+ return None
143
  try:
144
+ parsed = int(value)
145
+ except (TypeError, ValueError):
146
  return None
147
+ return parsed if parsed >= 0 else None