Spaces:
Paused
Paused
Peng Ding commited on
Commit ·
573cdfc
1
Parent(s): 0e7856a
security: fix SSRF, add URL validation, fail2ban, and body size limit
Browse files- Add url_validator.py: scheme whitelist (http/https only), DNS
resolution with private IP blocking (loopback, RFC1918, link-local,
metadata endpoint), prevents file://, data:, javascript:// URLs
- Add fail2ban-style IP rate limiting to auth: 3 failures in 5min
window → 10min ban, with X-Forwarded-For support
- Reduce max_body_size from 10MB to 64KB
- Promote filter blocked-request log from DEBUG to INFO
- src/veilrender/app.py +3 -1
- src/veilrender/auth.py +77 -2
- src/veilrender/filters.py +1 -1
- src/veilrender/routes/render.py +8 -0
- src/veilrender/routes/screenshot.py +12 -0
- src/veilrender/url_validator.py +123 -0
src/veilrender/app.py
CHANGED
|
@@ -37,7 +37,9 @@ async def _pipe(source: asyncio.StreamReader, dest: asyncio.StreamReader) -> Non
|
|
| 37 |
|
| 38 |
def create_app() -> App:
|
| 39 |
"""Create and configure the VeilRender application."""
|
| 40 |
-
app = App(
|
|
|
|
|
|
|
| 41 |
|
| 42 |
# Register routes
|
| 43 |
health.register(app)
|
|
|
|
| 37 |
|
| 38 |
def create_app() -> App:
|
| 39 |
"""Create and configure the VeilRender application."""
|
| 40 |
+
app = App(
|
| 41 |
+
max_body_size=64 * 1024
|
| 42 |
+
) # 64 KB — render/screenshot payloads are small JSON
|
| 43 |
|
| 44 |
# Register routes
|
| 45 |
health.register(app)
|
src/veilrender/auth.py
CHANGED
|
@@ -1,10 +1,73 @@
|
|
| 1 |
-
"""Token-based authentication.
|
|
|
|
|
|
|
|
|
|
|
|
|
| 2 |
|
| 3 |
from __future__ import annotations
|
| 4 |
|
|
|
|
|
|
|
|
|
|
|
|
|
| 5 |
from veilrender._vendor.httpserver import HTTPException, Request
|
| 6 |
from veilrender.config import settings
|
| 7 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 8 |
|
| 9 |
def verify_token(request: Request) -> None:
|
| 10 |
"""Verify the API token from header or query param.
|
|
@@ -13,7 +76,8 @@ def verify_token(request: Request) -> None:
|
|
| 13 |
back to ``?token=<token>`` query parameter.
|
| 14 |
|
| 15 |
Raises:
|
| 16 |
-
HTTPException:
|
|
|
|
| 17 |
|
| 18 |
If ``VEILRENDER_API_TOKEN`` is not configured, auth is disabled.
|
| 19 |
"""
|
|
@@ -21,19 +85,30 @@ def verify_token(request: Request) -> None:
|
|
| 21 |
if expected is None:
|
| 22 |
return
|
| 23 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 24 |
# Check Authorization header
|
| 25 |
auth_header = request.headers.get("authorization", "")
|
| 26 |
if auth_header.startswith("Bearer "):
|
| 27 |
token = auth_header[7:].strip()
|
| 28 |
if token == expected:
|
|
|
|
| 29 |
return
|
|
|
|
| 30 |
raise HTTPException(401, "Invalid token")
|
| 31 |
|
| 32 |
# Check query param
|
| 33 |
token_params = request.query_params.get("token", [])
|
| 34 |
if token_params:
|
| 35 |
if token_params[0] == expected:
|
|
|
|
| 36 |
return
|
|
|
|
| 37 |
raise HTTPException(401, "Invalid token")
|
| 38 |
|
|
|
|
| 39 |
raise HTTPException(403, "Authentication required")
|
|
|
|
| 1 |
+
"""Token-based authentication with IP-based rate limiting.
|
| 2 |
+
|
| 3 |
+
Failed auth attempts are tracked per IP. After 3 failures within a
|
| 4 |
+
rolling window, the IP is banned for a configurable duration.
|
| 5 |
+
"""
|
| 6 |
|
| 7 |
from __future__ import annotations
|
| 8 |
|
| 9 |
+
import logging
|
| 10 |
+
import time
|
| 11 |
+
from collections import defaultdict
|
| 12 |
+
|
| 13 |
from veilrender._vendor.httpserver import HTTPException, Request
|
| 14 |
from veilrender.config import settings
|
| 15 |
|
| 16 |
+
logger = logging.getLogger(__name__)
|
| 17 |
+
|
| 18 |
+
# Fail2ban settings
|
| 19 |
+
_MAX_FAILURES = 3
|
| 20 |
+
_WINDOW_SECONDS = 300 # 5 minute rolling window
|
| 21 |
+
_BAN_SECONDS = 600 # 10 minute ban
|
| 22 |
+
|
| 23 |
+
# State: per-IP failure timestamps and ban expiry
|
| 24 |
+
_failures: dict[str, list[float]] = defaultdict(list)
|
| 25 |
+
_banned: dict[str, float] = {}
|
| 26 |
+
|
| 27 |
+
|
| 28 |
+
def _get_client_ip(request: Request) -> str:
|
| 29 |
+
"""Extract client IP, respecting X-Forwarded-For behind reverse proxy."""
|
| 30 |
+
forwarded = request.headers.get("x-forwarded-for", "")
|
| 31 |
+
if forwarded:
|
| 32 |
+
# Take the first (leftmost) IP — the original client
|
| 33 |
+
return forwarded.split(",")[0].strip()
|
| 34 |
+
return request.client_addr[0] if hasattr(request, "client_addr") else "unknown"
|
| 35 |
+
|
| 36 |
+
|
| 37 |
+
def _is_banned(ip: str) -> bool:
|
| 38 |
+
"""Check if an IP is currently banned."""
|
| 39 |
+
ban_until = _banned.get(ip)
|
| 40 |
+
if ban_until is None:
|
| 41 |
+
return False
|
| 42 |
+
if time.monotonic() >= ban_until:
|
| 43 |
+
del _banned[ip]
|
| 44 |
+
return False
|
| 45 |
+
return True
|
| 46 |
+
|
| 47 |
+
|
| 48 |
+
def _record_failure(ip: str) -> None:
|
| 49 |
+
"""Record an auth failure and ban if threshold exceeded."""
|
| 50 |
+
now = time.monotonic()
|
| 51 |
+
|
| 52 |
+
# Clean old entries outside the window
|
| 53 |
+
_failures[ip] = [t for t in _failures[ip] if now - t < _WINDOW_SECONDS]
|
| 54 |
+
_failures[ip].append(now)
|
| 55 |
+
|
| 56 |
+
if len(_failures[ip]) >= _MAX_FAILURES:
|
| 57 |
+
_banned[ip] = now + _BAN_SECONDS
|
| 58 |
+
_failures.pop(ip, None)
|
| 59 |
+
logger.warning(
|
| 60 |
+
"IP %s banned for %ds after %d auth failures",
|
| 61 |
+
ip,
|
| 62 |
+
_BAN_SECONDS,
|
| 63 |
+
_MAX_FAILURES,
|
| 64 |
+
)
|
| 65 |
+
|
| 66 |
+
|
| 67 |
+
def _clear_failures(ip: str) -> None:
|
| 68 |
+
"""Clear failure history on successful auth."""
|
| 69 |
+
_failures.pop(ip, None)
|
| 70 |
+
|
| 71 |
|
| 72 |
def verify_token(request: Request) -> None:
|
| 73 |
"""Verify the API token from header or query param.
|
|
|
|
| 76 |
back to ``?token=<token>`` query parameter.
|
| 77 |
|
| 78 |
Raises:
|
| 79 |
+
HTTPException: 403 if IP is banned, 401 if token is invalid,
|
| 80 |
+
403 if token is missing.
|
| 81 |
|
| 82 |
If ``VEILRENDER_API_TOKEN`` is not configured, auth is disabled.
|
| 83 |
"""
|
|
|
|
| 85 |
if expected is None:
|
| 86 |
return
|
| 87 |
|
| 88 |
+
ip = _get_client_ip(request)
|
| 89 |
+
|
| 90 |
+
# Check ban first
|
| 91 |
+
if _is_banned(ip):
|
| 92 |
+
raise HTTPException(403, "Temporarily banned due to repeated auth failures")
|
| 93 |
+
|
| 94 |
# Check Authorization header
|
| 95 |
auth_header = request.headers.get("authorization", "")
|
| 96 |
if auth_header.startswith("Bearer "):
|
| 97 |
token = auth_header[7:].strip()
|
| 98 |
if token == expected:
|
| 99 |
+
_clear_failures(ip)
|
| 100 |
return
|
| 101 |
+
_record_failure(ip)
|
| 102 |
raise HTTPException(401, "Invalid token")
|
| 103 |
|
| 104 |
# Check query param
|
| 105 |
token_params = request.query_params.get("token", [])
|
| 106 |
if token_params:
|
| 107 |
if token_params[0] == expected:
|
| 108 |
+
_clear_failures(ip)
|
| 109 |
return
|
| 110 |
+
_record_failure(ip)
|
| 111 |
raise HTTPException(401, "Invalid token")
|
| 112 |
|
| 113 |
+
_record_failure(ip)
|
| 114 |
raise HTTPException(403, "Authentication required")
|
src/veilrender/filters.py
CHANGED
|
@@ -93,7 +93,7 @@ def make_route_handler(blocklist: frozenset[str]):
|
|
| 93 |
async def _handle_route(route: Route) -> None:
|
| 94 |
url = route.request.url
|
| 95 |
if should_block(url, blocklist):
|
| 96 |
-
logger.
|
| 97 |
await route.abort("blockedbyclient")
|
| 98 |
else:
|
| 99 |
await route.continue_()
|
|
|
|
| 93 |
async def _handle_route(route: Route) -> None:
|
| 94 |
url = route.request.url
|
| 95 |
if should_block(url, blocklist):
|
| 96 |
+
logger.info("Blocked: %s", url[:200])
|
| 97 |
await route.abort("blockedbyclient")
|
| 98 |
else:
|
| 99 |
await route.continue_()
|
src/veilrender/routes/render.py
CHANGED
|
@@ -21,6 +21,7 @@ from veilrender.models import (
|
|
| 21 |
RenderResponse,
|
| 22 |
)
|
| 23 |
from veilrender.storage import storage_manager
|
|
|
|
| 24 |
|
| 25 |
logger = logging.getLogger(__name__)
|
| 26 |
|
|
@@ -72,6 +73,13 @@ def register(app: App) -> None:
|
|
| 72 |
return JSONResponse({"error": "Missing 'url' field"}, status_code=400)
|
| 73 |
|
| 74 |
req = RenderRequest.from_dict(data)
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 75 |
timeout = req.timeout or settings.timeout
|
| 76 |
stats.render.requests += 1
|
| 77 |
t0 = time.monotonic()
|
|
|
|
| 21 |
RenderResponse,
|
| 22 |
)
|
| 23 |
from veilrender.storage import storage_manager
|
| 24 |
+
from veilrender.url_validator import URLValidationError, validate_url
|
| 25 |
|
| 26 |
logger = logging.getLogger(__name__)
|
| 27 |
|
|
|
|
| 73 |
return JSONResponse({"error": "Missing 'url' field"}, status_code=400)
|
| 74 |
|
| 75 |
req = RenderRequest.from_dict(data)
|
| 76 |
+
|
| 77 |
+
# Validate URL before any processing
|
| 78 |
+
try:
|
| 79 |
+
validate_url(req.url)
|
| 80 |
+
except URLValidationError as exc:
|
| 81 |
+
return JSONResponse({"error": f"URL rejected: {exc!s}"}, status_code=400)
|
| 82 |
+
|
| 83 |
timeout = req.timeout or settings.timeout
|
| 84 |
stats.render.requests += 1
|
| 85 |
t0 = time.monotonic()
|
src/veilrender/routes/screenshot.py
CHANGED
|
@@ -11,6 +11,7 @@ from veilrender.auth import verify_token
|
|
| 11 |
from veilrender.browser import browser_manager
|
| 12 |
from veilrender.config import settings
|
| 13 |
from veilrender.models import ScreenshotRequest
|
|
|
|
| 14 |
|
| 15 |
logger = logging.getLogger(__name__)
|
| 16 |
|
|
@@ -39,6 +40,17 @@ def register(app: App) -> None:
|
|
| 39 |
)
|
| 40 |
|
| 41 |
req = ScreenshotRequest.from_dict(data)
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 42 |
timeout = req.timeout or settings.timeout
|
| 43 |
stats.screenshot.requests += 1
|
| 44 |
t0 = time.monotonic()
|
|
|
|
| 11 |
from veilrender.browser import browser_manager
|
| 12 |
from veilrender.config import settings
|
| 13 |
from veilrender.models import ScreenshotRequest
|
| 14 |
+
from veilrender.url_validator import URLValidationError, validate_url
|
| 15 |
|
| 16 |
logger = logging.getLogger(__name__)
|
| 17 |
|
|
|
|
| 40 |
)
|
| 41 |
|
| 42 |
req = ScreenshotRequest.from_dict(data)
|
| 43 |
+
|
| 44 |
+
# Validate URL before any processing
|
| 45 |
+
try:
|
| 46 |
+
validate_url(req.url)
|
| 47 |
+
except URLValidationError as exc:
|
| 48 |
+
return Response(
|
| 49 |
+
body=f'{{"error": "URL rejected: {exc!s}"}}'.encode(),
|
| 50 |
+
status_code=400,
|
| 51 |
+
content_type="application/json",
|
| 52 |
+
)
|
| 53 |
+
|
| 54 |
timeout = req.timeout or settings.timeout
|
| 55 |
stats.screenshot.requests += 1
|
| 56 |
t0 = time.monotonic()
|
src/veilrender/url_validator.py
ADDED
|
@@ -0,0 +1,123 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""URL validation to prevent SSRF, local file reads, and protocol abuse.
|
| 2 |
+
|
| 3 |
+
Blocks:
|
| 4 |
+
- Non-HTTP(S) schemes: file://, data:, javascript:, ftp://, etc.
|
| 5 |
+
- Private/reserved IP ranges: 127.0.0.0/8, 10.0.0.0/8, 172.16.0.0/12,
|
| 6 |
+
192.168.0.0/16, 169.254.0.0/16, ::1, fd00::/8, etc.
|
| 7 |
+
- DNS resolution is checked before navigation to prevent DNS rebinding.
|
| 8 |
+
"""
|
| 9 |
+
|
| 10 |
+
from __future__ import annotations
|
| 11 |
+
|
| 12 |
+
import ipaddress
|
| 13 |
+
import logging
|
| 14 |
+
import socket
|
| 15 |
+
from urllib.parse import urlparse
|
| 16 |
+
|
| 17 |
+
logger = logging.getLogger(__name__)
|
| 18 |
+
|
| 19 |
+
_ALLOWED_SCHEMES = {"http", "https"}
|
| 20 |
+
|
| 21 |
+
|
| 22 |
+
class URLValidationError(Exception):
|
| 23 |
+
"""Raised when a URL fails validation."""
|
| 24 |
+
|
| 25 |
+
|
| 26 |
+
def validate_url(url: str) -> str:
|
| 27 |
+
"""Validate and sanitize a URL before passing it to Playwright.
|
| 28 |
+
|
| 29 |
+
Args:
|
| 30 |
+
url: The user-supplied URL to validate.
|
| 31 |
+
|
| 32 |
+
Returns:
|
| 33 |
+
The validated URL (unchanged if valid).
|
| 34 |
+
|
| 35 |
+
Raises:
|
| 36 |
+
URLValidationError: If the URL is unsafe.
|
| 37 |
+
"""
|
| 38 |
+
if not url or not url.strip():
|
| 39 |
+
raise URLValidationError("Empty URL")
|
| 40 |
+
|
| 41 |
+
parsed = urlparse(url)
|
| 42 |
+
|
| 43 |
+
# Scheme check
|
| 44 |
+
scheme = parsed.scheme.lower()
|
| 45 |
+
if scheme not in _ALLOWED_SCHEMES:
|
| 46 |
+
raise URLValidationError(f"Scheme '{scheme}' not allowed (only http/https)")
|
| 47 |
+
|
| 48 |
+
# Hostname check
|
| 49 |
+
hostname = parsed.hostname
|
| 50 |
+
if not hostname:
|
| 51 |
+
raise URLValidationError("Missing hostname")
|
| 52 |
+
|
| 53 |
+
# Resolve DNS and check against private IP ranges
|
| 54 |
+
_check_resolved_ips(hostname)
|
| 55 |
+
|
| 56 |
+
return url
|
| 57 |
+
|
| 58 |
+
|
| 59 |
+
def _check_resolved_ips(hostname: str) -> None:
|
| 60 |
+
"""Resolve hostname and reject private/reserved IP addresses.
|
| 61 |
+
|
| 62 |
+
Resolves DNS before Playwright navigates to prevent DNS rebinding
|
| 63 |
+
attacks where a hostname initially resolves to a public IP but
|
| 64 |
+
later resolves to a private one.
|
| 65 |
+
|
| 66 |
+
Args:
|
| 67 |
+
hostname: The hostname to check.
|
| 68 |
+
|
| 69 |
+
Raises:
|
| 70 |
+
URLValidationError: If the hostname resolves to a private IP.
|
| 71 |
+
"""
|
| 72 |
+
# Check if hostname is a raw IP literal
|
| 73 |
+
try:
|
| 74 |
+
addr = ipaddress.ip_address(hostname)
|
| 75 |
+
if _is_blocked_ip(addr):
|
| 76 |
+
raise URLValidationError(f"IP address {hostname} is in a blocked range")
|
| 77 |
+
return
|
| 78 |
+
except ValueError:
|
| 79 |
+
pass # Not an IP literal, proceed to DNS resolution
|
| 80 |
+
|
| 81 |
+
# Resolve hostname
|
| 82 |
+
try:
|
| 83 |
+
results = socket.getaddrinfo(
|
| 84 |
+
hostname, None, socket.AF_UNSPEC, socket.SOCK_STREAM
|
| 85 |
+
)
|
| 86 |
+
except socket.gaierror as exc:
|
| 87 |
+
raise URLValidationError(
|
| 88 |
+
f"DNS resolution failed for '{hostname}': {exc}"
|
| 89 |
+
) from exc
|
| 90 |
+
|
| 91 |
+
if not results:
|
| 92 |
+
raise URLValidationError(f"No DNS results for '{hostname}'")
|
| 93 |
+
|
| 94 |
+
for family, _type, _proto, _canonname, sockaddr in results:
|
| 95 |
+
ip_str = sockaddr[0]
|
| 96 |
+
try:
|
| 97 |
+
addr = ipaddress.ip_address(ip_str)
|
| 98 |
+
if _is_blocked_ip(addr):
|
| 99 |
+
raise URLValidationError(
|
| 100 |
+
f"'{hostname}' resolves to blocked IP {ip_str}"
|
| 101 |
+
)
|
| 102 |
+
except ValueError:
|
| 103 |
+
continue
|
| 104 |
+
|
| 105 |
+
|
| 106 |
+
def _is_blocked_ip(addr: ipaddress.IPv4Address | ipaddress.IPv6Address) -> bool:
|
| 107 |
+
"""Check if an IP address is in a blocked range.
|
| 108 |
+
|
| 109 |
+
Blocks:
|
| 110 |
+
- Loopback (127.0.0.0/8, ::1)
|
| 111 |
+
- Private networks (10/8, 172.16/12, 192.168/16)
|
| 112 |
+
- Link-local (169.254/16, fe80::/10)
|
| 113 |
+
- Unique local (fd00::/8)
|
| 114 |
+
- Multicast, reserved, unspecified
|
| 115 |
+
"""
|
| 116 |
+
return (
|
| 117 |
+
addr.is_loopback
|
| 118 |
+
or addr.is_private
|
| 119 |
+
or addr.is_reserved
|
| 120 |
+
or addr.is_multicast
|
| 121 |
+
or addr.is_link_local
|
| 122 |
+
or addr.is_unspecified
|
| 123 |
+
)
|