Spaces:
Paused
Paused
AI Bot commited on
Commit ·
4e6b2ca
1
Parent(s): c21f8a5
fix: add DEFAULT_KEY fallback and resilient standard requests fetcher to prevent timeout hangs
Browse files- api.py +5 -2
- app.py +3 -2
- src/crawler.py +28 -31
api.py
CHANGED
|
@@ -20,8 +20,9 @@ from src.monitor import SystemMonitor
|
|
| 20 |
logging.basicConfig(level=logging.INFO, format='%(asctime)s - %(name)s - %(levelname)s - %(message)s')
|
| 21 |
logger = logging.getLogger("API")
|
| 22 |
|
| 23 |
-
# Securely load the key from Hugging Face environment variables
|
| 24 |
-
|
|
|
|
| 25 |
|
| 26 |
def verify_key(
|
| 27 |
x_autoalt_key: Optional[str] = Header(None, alias="X-AutoAlt-Key"),
|
|
@@ -230,6 +231,8 @@ def health_check():
|
|
| 230 |
def get_system_status(domain: Optional[str] = None):
|
| 231 |
# If a domain is provided, we check its connectivity specifically
|
| 232 |
stats = SystemMonitor.get_system_stats(target_url=domain)
|
|
|
|
|
|
|
| 233 |
return stats
|
| 234 |
|
| 235 |
@app.post("/api/scanstart")
|
|
|
|
| 20 |
logging.basicConfig(level=logging.INFO, format='%(asctime)s - %(name)s - %(levelname)s - %(message)s')
|
| 21 |
logger = logging.getLogger("API")
|
| 22 |
|
| 23 |
+
# Securely load the key from Hugging Face environment variables, with a default fallback key
|
| 24 |
+
DEFAULT_KEY = "autoalt-secret-key-2026"
|
| 25 |
+
SECRET_KEY = os.getenv("AUTOALT_SECRET_KEY") or DEFAULT_KEY
|
| 26 |
|
| 27 |
def verify_key(
|
| 28 |
x_autoalt_key: Optional[str] = Header(None, alias="X-AutoAlt-Key"),
|
|
|
|
| 231 |
def get_system_status(domain: Optional[str] = None):
|
| 232 |
# If a domain is provided, we check its connectivity specifically
|
| 233 |
stats = SystemMonitor.get_system_stats(target_url=domain)
|
| 234 |
+
stats["auth_configured"] = bool(os.getenv("AUTOALT_SECRET_KEY"))
|
| 235 |
+
stats["active_key_source"] = "Custom Hugging Face Secret" if os.getenv("AUTOALT_SECRET_KEY") else f"Default Secret ({DEFAULT_KEY})"
|
| 236 |
return stats
|
| 237 |
|
| 238 |
@app.post("/api/scanstart")
|
app.py
CHANGED
|
@@ -1,8 +1,9 @@
|
|
| 1 |
import os
|
| 2 |
from fastapi import FastAPI, Header, HTTPException
|
| 3 |
|
| 4 |
-
# Securely load the key from Hugging Face environment variables
|
| 5 |
-
|
|
|
|
| 6 |
|
| 7 |
try:
|
| 8 |
from api import app, verify_key, run_scraper
|
|
|
|
| 1 |
import os
|
| 2 |
from fastapi import FastAPI, Header, HTTPException
|
| 3 |
|
| 4 |
+
# Securely load the key from Hugging Face environment variables, with a default fallback key
|
| 5 |
+
DEFAULT_KEY = "autoalt-secret-key-2026"
|
| 6 |
+
SECRET_KEY = os.getenv("AUTOALT_SECRET_KEY") or DEFAULT_KEY
|
| 7 |
|
| 8 |
try:
|
| 9 |
from api import app, verify_key, run_scraper
|
src/crawler.py
CHANGED
|
@@ -66,7 +66,7 @@ class Crawler:
|
|
| 66 |
# Tier 2: Cloudscraper (Intermediate, handles JS challenges)
|
| 67 |
logger.info(f"Tier 1 failed. Trying Cloudscraper info for {url}...")
|
| 68 |
try:
|
| 69 |
-
resp = self.scraper.get(url, timeout=
|
| 70 |
if 200 <= resp.status_code < 300:
|
| 71 |
if not self._is_blocked(resp.text):
|
| 72 |
return resp.text
|
|
@@ -115,39 +115,36 @@ class Crawler:
|
|
| 115 |
|
| 116 |
def _fetch_http(self, url):
|
| 117 |
"""
|
| 118 |
-
Fast HTTP fetch with aggressive timeouts and
|
|
|
|
|
|
|
| 119 |
"""
|
| 120 |
-
|
| 121 |
-
|
| 122 |
-
|
| 123 |
-
|
| 124 |
-
|
| 125 |
-
|
| 126 |
-
|
| 127 |
-
|
| 128 |
-
|
| 129 |
-
|
| 130 |
-
# Check for 200 OK (or close to it)
|
| 131 |
-
if 200 <= response.status_code < 300:
|
| 132 |
-
return response.text
|
| 133 |
-
|
| 134 |
-
if response.status_code == 403:
|
| 135 |
-
# Fail fast on 403 to trigger fallback immediately if needed
|
| 136 |
-
return response.text
|
| 137 |
-
|
| 138 |
-
if response.status_code in [429, 500, 502, 503]:
|
| 139 |
-
if attempt < max_retries - 1:
|
| 140 |
-
# Minimal sleep for speed
|
| 141 |
-
time.sleep(0.5)
|
| 142 |
-
continue
|
| 143 |
|
| 144 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 145 |
|
| 146 |
-
except Exception as e:
|
| 147 |
-
# If curl_cffi fails, we skip the slow standard requests fallback
|
| 148 |
-
# and return None to let the main loop decide (or trigger Playwright if configured)
|
| 149 |
-
logger.warning(f"Fast fetch failed for {url}: {e}")
|
| 150 |
-
|
| 151 |
return None
|
| 152 |
|
| 153 |
def _fetch_playwright(self, url):
|
|
|
|
| 66 |
# Tier 2: Cloudscraper (Intermediate, handles JS challenges)
|
| 67 |
logger.info(f"Tier 1 failed. Trying Cloudscraper info for {url}...")
|
| 68 |
try:
|
| 69 |
+
resp = self.scraper.get(url, timeout=15)
|
| 70 |
if 200 <= resp.status_code < 300:
|
| 71 |
if not self._is_blocked(resp.text):
|
| 72 |
return resp.text
|
|
|
|
| 115 |
|
| 116 |
def _fetch_http(self, url):
|
| 117 |
"""
|
| 118 |
+
Fast HTTP fetch with aggressive timeouts and resilient fallback.
|
| 119 |
+
1. Try curl_cffi with 12s timeout
|
| 120 |
+
2. If curl_cffi times out or fails (e.g. TLS/H2 fingerprint issues), fallback to standard requests
|
| 121 |
"""
|
| 122 |
+
try:
|
| 123 |
+
self.session.headers["User-Agent"] = self.ua.random
|
| 124 |
+
response = self.session.get(url, timeout=12, allow_redirects=True)
|
| 125 |
+
if 200 <= response.status_code < 300:
|
| 126 |
+
return response.text
|
| 127 |
+
if response.status_code == 403:
|
| 128 |
+
return response.text
|
| 129 |
+
except Exception as e:
|
| 130 |
+
logger.warning(f"curl_cffi fetch failed for {url}: {e}. Trying standard requests fallback...")
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 131 |
|
| 132 |
+
# Fallback to standard requests (handles servers where curl_cffi TLS/HTTP2 hangs)
|
| 133 |
+
try:
|
| 134 |
+
import requests as std_requests
|
| 135 |
+
headers = {
|
| 136 |
+
"User-Agent": self.ua.random,
|
| 137 |
+
"Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8",
|
| 138 |
+
"Accept-Language": "en-US,en;q=0.9",
|
| 139 |
+
}
|
| 140 |
+
resp = std_requests.get(url, headers=headers, timeout=12, allow_redirects=True)
|
| 141 |
+
if 200 <= resp.status_code < 300:
|
| 142 |
+
return resp.text
|
| 143 |
+
if resp.status_code == 403:
|
| 144 |
+
return resp.text
|
| 145 |
+
except Exception as e2:
|
| 146 |
+
logger.warning(f"Standard requests fallback failed for {url}: {e2}")
|
| 147 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 148 |
return None
|
| 149 |
|
| 150 |
def _fetch_playwright(self, url):
|