Peng Ding commited on
Commit
573cdfc
·
1 Parent(s): 0e7856a

security: fix SSRF, add URL validation, fail2ban, and body size limit

Browse files

- Add url_validator.py: scheme whitelist (http/https only), DNS
resolution with private IP blocking (loopback, RFC1918, link-local,
metadata endpoint), prevents file://, data:, javascript:// URLs
- Add fail2ban-style IP rate limiting to auth: 3 failures in 5min
window → 10min ban, with X-Forwarded-For support
- Reduce max_body_size from 10MB to 64KB
- Promote filter blocked-request log from DEBUG to INFO

src/veilrender/app.py CHANGED
@@ -37,7 +37,9 @@ async def _pipe(source: asyncio.StreamReader, dest: asyncio.StreamReader) -> Non
37
 
38
  def create_app() -> App:
39
  """Create and configure the VeilRender application."""
40
- app = App(max_body_size=10 * 1024 * 1024) # 10 MB
 
 
41
 
42
  # Register routes
43
  health.register(app)
 
37
 
38
  def create_app() -> App:
39
  """Create and configure the VeilRender application."""
40
+ app = App(
41
+ max_body_size=64 * 1024
42
+ ) # 64 KB — render/screenshot payloads are small JSON
43
 
44
  # Register routes
45
  health.register(app)
src/veilrender/auth.py CHANGED
@@ -1,10 +1,73 @@
1
- """Token-based authentication."""
 
 
 
 
2
 
3
  from __future__ import annotations
4
 
 
 
 
 
5
  from veilrender._vendor.httpserver import HTTPException, Request
6
  from veilrender.config import settings
7
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
8
 
9
  def verify_token(request: Request) -> None:
10
  """Verify the API token from header or query param.
@@ -13,7 +76,8 @@ def verify_token(request: Request) -> None:
13
  back to ``?token=<token>`` query parameter.
14
 
15
  Raises:
16
- HTTPException: 401 if token is invalid, 403 if token is missing.
 
17
 
18
  If ``VEILRENDER_API_TOKEN`` is not configured, auth is disabled.
19
  """
@@ -21,19 +85,30 @@ def verify_token(request: Request) -> None:
21
  if expected is None:
22
  return
23
 
 
 
 
 
 
 
24
  # Check Authorization header
25
  auth_header = request.headers.get("authorization", "")
26
  if auth_header.startswith("Bearer "):
27
  token = auth_header[7:].strip()
28
  if token == expected:
 
29
  return
 
30
  raise HTTPException(401, "Invalid token")
31
 
32
  # Check query param
33
  token_params = request.query_params.get("token", [])
34
  if token_params:
35
  if token_params[0] == expected:
 
36
  return
 
37
  raise HTTPException(401, "Invalid token")
38
 
 
39
  raise HTTPException(403, "Authentication required")
 
1
+ """Token-based authentication with IP-based rate limiting.
2
+
3
+ Failed auth attempts are tracked per IP. After 3 failures within a
4
+ rolling window, the IP is banned for a configurable duration.
5
+ """
6
 
7
  from __future__ import annotations
8
 
9
+ import logging
10
+ import time
11
+ from collections import defaultdict
12
+
13
  from veilrender._vendor.httpserver import HTTPException, Request
14
  from veilrender.config import settings
15
 
16
+ logger = logging.getLogger(__name__)
17
+
18
+ # Fail2ban settings
19
+ _MAX_FAILURES = 3
20
+ _WINDOW_SECONDS = 300 # 5 minute rolling window
21
+ _BAN_SECONDS = 600 # 10 minute ban
22
+
23
+ # State: per-IP failure timestamps and ban expiry
24
+ _failures: dict[str, list[float]] = defaultdict(list)
25
+ _banned: dict[str, float] = {}
26
+
27
+
28
+ def _get_client_ip(request: Request) -> str:
29
+ """Extract client IP, respecting X-Forwarded-For behind reverse proxy."""
30
+ forwarded = request.headers.get("x-forwarded-for", "")
31
+ if forwarded:
32
+ # Take the first (leftmost) IP — the original client
33
+ return forwarded.split(",")[0].strip()
34
+ return request.client_addr[0] if hasattr(request, "client_addr") else "unknown"
35
+
36
+
37
+ def _is_banned(ip: str) -> bool:
38
+ """Check if an IP is currently banned."""
39
+ ban_until = _banned.get(ip)
40
+ if ban_until is None:
41
+ return False
42
+ if time.monotonic() >= ban_until:
43
+ del _banned[ip]
44
+ return False
45
+ return True
46
+
47
+
48
+ def _record_failure(ip: str) -> None:
49
+ """Record an auth failure and ban if threshold exceeded."""
50
+ now = time.monotonic()
51
+
52
+ # Clean old entries outside the window
53
+ _failures[ip] = [t for t in _failures[ip] if now - t < _WINDOW_SECONDS]
54
+ _failures[ip].append(now)
55
+
56
+ if len(_failures[ip]) >= _MAX_FAILURES:
57
+ _banned[ip] = now + _BAN_SECONDS
58
+ _failures.pop(ip, None)
59
+ logger.warning(
60
+ "IP %s banned for %ds after %d auth failures",
61
+ ip,
62
+ _BAN_SECONDS,
63
+ _MAX_FAILURES,
64
+ )
65
+
66
+
67
+ def _clear_failures(ip: str) -> None:
68
+ """Clear failure history on successful auth."""
69
+ _failures.pop(ip, None)
70
+
71
 
72
  def verify_token(request: Request) -> None:
73
  """Verify the API token from header or query param.
 
76
  back to ``?token=<token>`` query parameter.
77
 
78
  Raises:
79
+ HTTPException: 403 if IP is banned, 401 if token is invalid,
80
+ 403 if token is missing.
81
 
82
  If ``VEILRENDER_API_TOKEN`` is not configured, auth is disabled.
83
  """
 
85
  if expected is None:
86
  return
87
 
88
+ ip = _get_client_ip(request)
89
+
90
+ # Check ban first
91
+ if _is_banned(ip):
92
+ raise HTTPException(403, "Temporarily banned due to repeated auth failures")
93
+
94
  # Check Authorization header
95
  auth_header = request.headers.get("authorization", "")
96
  if auth_header.startswith("Bearer "):
97
  token = auth_header[7:].strip()
98
  if token == expected:
99
+ _clear_failures(ip)
100
  return
101
+ _record_failure(ip)
102
  raise HTTPException(401, "Invalid token")
103
 
104
  # Check query param
105
  token_params = request.query_params.get("token", [])
106
  if token_params:
107
  if token_params[0] == expected:
108
+ _clear_failures(ip)
109
  return
110
+ _record_failure(ip)
111
  raise HTTPException(401, "Invalid token")
112
 
113
+ _record_failure(ip)
114
  raise HTTPException(403, "Authentication required")
src/veilrender/filters.py CHANGED
@@ -93,7 +93,7 @@ def make_route_handler(blocklist: frozenset[str]):
93
  async def _handle_route(route: Route) -> None:
94
  url = route.request.url
95
  if should_block(url, blocklist):
96
- logger.debug("Blocked: %s", url[:200])
97
  await route.abort("blockedbyclient")
98
  else:
99
  await route.continue_()
 
93
  async def _handle_route(route: Route) -> None:
94
  url = route.request.url
95
  if should_block(url, blocklist):
96
+ logger.info("Blocked: %s", url[:200])
97
  await route.abort("blockedbyclient")
98
  else:
99
  await route.continue_()
src/veilrender/routes/render.py CHANGED
@@ -21,6 +21,7 @@ from veilrender.models import (
21
  RenderResponse,
22
  )
23
  from veilrender.storage import storage_manager
 
24
 
25
  logger = logging.getLogger(__name__)
26
 
@@ -72,6 +73,13 @@ def register(app: App) -> None:
72
  return JSONResponse({"error": "Missing 'url' field"}, status_code=400)
73
 
74
  req = RenderRequest.from_dict(data)
 
 
 
 
 
 
 
75
  timeout = req.timeout or settings.timeout
76
  stats.render.requests += 1
77
  t0 = time.monotonic()
 
21
  RenderResponse,
22
  )
23
  from veilrender.storage import storage_manager
24
+ from veilrender.url_validator import URLValidationError, validate_url
25
 
26
  logger = logging.getLogger(__name__)
27
 
 
73
  return JSONResponse({"error": "Missing 'url' field"}, status_code=400)
74
 
75
  req = RenderRequest.from_dict(data)
76
+
77
+ # Validate URL before any processing
78
+ try:
79
+ validate_url(req.url)
80
+ except URLValidationError as exc:
81
+ return JSONResponse({"error": f"URL rejected: {exc!s}"}, status_code=400)
82
+
83
  timeout = req.timeout or settings.timeout
84
  stats.render.requests += 1
85
  t0 = time.monotonic()
src/veilrender/routes/screenshot.py CHANGED
@@ -11,6 +11,7 @@ from veilrender.auth import verify_token
11
  from veilrender.browser import browser_manager
12
  from veilrender.config import settings
13
  from veilrender.models import ScreenshotRequest
 
14
 
15
  logger = logging.getLogger(__name__)
16
 
@@ -39,6 +40,17 @@ def register(app: App) -> None:
39
  )
40
 
41
  req = ScreenshotRequest.from_dict(data)
 
 
 
 
 
 
 
 
 
 
 
42
  timeout = req.timeout or settings.timeout
43
  stats.screenshot.requests += 1
44
  t0 = time.monotonic()
 
11
  from veilrender.browser import browser_manager
12
  from veilrender.config import settings
13
  from veilrender.models import ScreenshotRequest
14
+ from veilrender.url_validator import URLValidationError, validate_url
15
 
16
  logger = logging.getLogger(__name__)
17
 
 
40
  )
41
 
42
  req = ScreenshotRequest.from_dict(data)
43
+
44
+ # Validate URL before any processing
45
+ try:
46
+ validate_url(req.url)
47
+ except URLValidationError as exc:
48
+ return Response(
49
+ body=f'{{"error": "URL rejected: {exc!s}"}}'.encode(),
50
+ status_code=400,
51
+ content_type="application/json",
52
+ )
53
+
54
  timeout = req.timeout or settings.timeout
55
  stats.screenshot.requests += 1
56
  t0 = time.monotonic()
src/veilrender/url_validator.py ADDED
@@ -0,0 +1,123 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ """URL validation to prevent SSRF, local file reads, and protocol abuse.
2
+
3
+ Blocks:
4
+ - Non-HTTP(S) schemes: file://, data:, javascript:, ftp://, etc.
5
+ - Private/reserved IP ranges: 127.0.0.0/8, 10.0.0.0/8, 172.16.0.0/12,
6
+ 192.168.0.0/16, 169.254.0.0/16, ::1, fd00::/8, etc.
7
+ - DNS resolution is checked before navigation to prevent DNS rebinding.
8
+ """
9
+
10
+ from __future__ import annotations
11
+
12
+ import ipaddress
13
+ import logging
14
+ import socket
15
+ from urllib.parse import urlparse
16
+
17
+ logger = logging.getLogger(__name__)
18
+
19
+ _ALLOWED_SCHEMES = {"http", "https"}
20
+
21
+
22
+ class URLValidationError(Exception):
23
+ """Raised when a URL fails validation."""
24
+
25
+
26
+ def validate_url(url: str) -> str:
27
+ """Validate and sanitize a URL before passing it to Playwright.
28
+
29
+ Args:
30
+ url: The user-supplied URL to validate.
31
+
32
+ Returns:
33
+ The validated URL (unchanged if valid).
34
+
35
+ Raises:
36
+ URLValidationError: If the URL is unsafe.
37
+ """
38
+ if not url or not url.strip():
39
+ raise URLValidationError("Empty URL")
40
+
41
+ parsed = urlparse(url)
42
+
43
+ # Scheme check
44
+ scheme = parsed.scheme.lower()
45
+ if scheme not in _ALLOWED_SCHEMES:
46
+ raise URLValidationError(f"Scheme '{scheme}' not allowed (only http/https)")
47
+
48
+ # Hostname check
49
+ hostname = parsed.hostname
50
+ if not hostname:
51
+ raise URLValidationError("Missing hostname")
52
+
53
+ # Resolve DNS and check against private IP ranges
54
+ _check_resolved_ips(hostname)
55
+
56
+ return url
57
+
58
+
59
+ def _check_resolved_ips(hostname: str) -> None:
60
+ """Resolve hostname and reject private/reserved IP addresses.
61
+
62
+ Resolves DNS before Playwright navigates to prevent DNS rebinding
63
+ attacks where a hostname initially resolves to a public IP but
64
+ later resolves to a private one.
65
+
66
+ Args:
67
+ hostname: The hostname to check.
68
+
69
+ Raises:
70
+ URLValidationError: If the hostname resolves to a private IP.
71
+ """
72
+ # Check if hostname is a raw IP literal
73
+ try:
74
+ addr = ipaddress.ip_address(hostname)
75
+ if _is_blocked_ip(addr):
76
+ raise URLValidationError(f"IP address {hostname} is in a blocked range")
77
+ return
78
+ except ValueError:
79
+ pass # Not an IP literal, proceed to DNS resolution
80
+
81
+ # Resolve hostname
82
+ try:
83
+ results = socket.getaddrinfo(
84
+ hostname, None, socket.AF_UNSPEC, socket.SOCK_STREAM
85
+ )
86
+ except socket.gaierror as exc:
87
+ raise URLValidationError(
88
+ f"DNS resolution failed for '{hostname}': {exc}"
89
+ ) from exc
90
+
91
+ if not results:
92
+ raise URLValidationError(f"No DNS results for '{hostname}'")
93
+
94
+ for family, _type, _proto, _canonname, sockaddr in results:
95
+ ip_str = sockaddr[0]
96
+ try:
97
+ addr = ipaddress.ip_address(ip_str)
98
+ if _is_blocked_ip(addr):
99
+ raise URLValidationError(
100
+ f"'{hostname}' resolves to blocked IP {ip_str}"
101
+ )
102
+ except ValueError:
103
+ continue
104
+
105
+
106
+ def _is_blocked_ip(addr: ipaddress.IPv4Address | ipaddress.IPv6Address) -> bool:
107
+ """Check if an IP address is in a blocked range.
108
+
109
+ Blocks:
110
+ - Loopback (127.0.0.0/8, ::1)
111
+ - Private networks (10/8, 172.16/12, 192.168/16)
112
+ - Link-local (169.254/16, fe80::/10)
113
+ - Unique local (fd00::/8)
114
+ - Multicast, reserved, unspecified
115
+ """
116
+ return (
117
+ addr.is_loopback
118
+ or addr.is_private
119
+ or addr.is_reserved
120
+ or addr.is_multicast
121
+ or addr.is_link_local
122
+ or addr.is_unspecified
123
+ )