AI Bot commited on
Commit
4e6b2ca
·
1 Parent(s): c21f8a5

fix: add DEFAULT_KEY fallback and resilient standard requests fetcher to prevent timeout hangs

Browse files
Files changed (3) hide show
  1. api.py +5 -2
  2. app.py +3 -2
  3. src/crawler.py +28 -31
api.py CHANGED
@@ -20,8 +20,9 @@ from src.monitor import SystemMonitor
20
  logging.basicConfig(level=logging.INFO, format='%(asctime)s - %(name)s - %(levelname)s - %(message)s')
21
  logger = logging.getLogger("API")
22
 
23
- # Securely load the key from Hugging Face environment variables
24
- SECRET_KEY = os.getenv("AUTOALT_SECRET_KEY")
 
25
 
26
  def verify_key(
27
  x_autoalt_key: Optional[str] = Header(None, alias="X-AutoAlt-Key"),
@@ -230,6 +231,8 @@ def health_check():
230
  def get_system_status(domain: Optional[str] = None):
231
  # If a domain is provided, we check its connectivity specifically
232
  stats = SystemMonitor.get_system_stats(target_url=domain)
 
 
233
  return stats
234
 
235
  @app.post("/api/scanstart")
 
20
  logging.basicConfig(level=logging.INFO, format='%(asctime)s - %(name)s - %(levelname)s - %(message)s')
21
  logger = logging.getLogger("API")
22
 
23
+ # Securely load the key from Hugging Face environment variables, with a default fallback key
24
+ DEFAULT_KEY = "autoalt-secret-key-2026"
25
+ SECRET_KEY = os.getenv("AUTOALT_SECRET_KEY") or DEFAULT_KEY
26
 
27
  def verify_key(
28
  x_autoalt_key: Optional[str] = Header(None, alias="X-AutoAlt-Key"),
 
231
  def get_system_status(domain: Optional[str] = None):
232
  # If a domain is provided, we check its connectivity specifically
233
  stats = SystemMonitor.get_system_stats(target_url=domain)
234
+ stats["auth_configured"] = bool(os.getenv("AUTOALT_SECRET_KEY"))
235
+ stats["active_key_source"] = "Custom Hugging Face Secret" if os.getenv("AUTOALT_SECRET_KEY") else f"Default Secret ({DEFAULT_KEY})"
236
  return stats
237
 
238
  @app.post("/api/scanstart")
app.py CHANGED
@@ -1,8 +1,9 @@
1
  import os
2
  from fastapi import FastAPI, Header, HTTPException
3
 
4
- # Securely load the key from Hugging Face environment variables
5
- SECRET_KEY = os.getenv("AUTOALT_SECRET_KEY")
 
6
 
7
  try:
8
  from api import app, verify_key, run_scraper
 
1
  import os
2
  from fastapi import FastAPI, Header, HTTPException
3
 
4
+ # Securely load the key from Hugging Face environment variables, with a default fallback key
5
+ DEFAULT_KEY = "autoalt-secret-key-2026"
6
+ SECRET_KEY = os.getenv("AUTOALT_SECRET_KEY") or DEFAULT_KEY
7
 
8
  try:
9
  from api import app, verify_key, run_scraper
src/crawler.py CHANGED
@@ -66,7 +66,7 @@ class Crawler:
66
  # Tier 2: Cloudscraper (Intermediate, handles JS challenges)
67
  logger.info(f"Tier 1 failed. Trying Cloudscraper info for {url}...")
68
  try:
69
- resp = self.scraper.get(url, timeout=30)
70
  if 200 <= resp.status_code < 300:
71
  if not self._is_blocked(resp.text):
72
  return resp.text
@@ -115,39 +115,36 @@ class Crawler:
115
 
116
  def _fetch_http(self, url):
117
  """
118
- Fast HTTP fetch with aggressive timeouts and minimal retries.
 
 
119
  """
120
- max_retries = 2
121
- for attempt in range(max_retries):
122
- try:
123
- # Rotate User Agent
124
- self.session.headers["User-Agent"] = self.ua.random
125
-
126
- # Increased timeout to 30 seconds to handle slow sites without falling back to Playwright
127
- # verify=False is inherent in the session from init, but good to be explicit if needed (curl_cffi uses session setting)
128
- response = self.session.get(url, timeout=30, allow_redirects=True)
129
-
130
- # Check for 200 OK (or close to it)
131
- if 200 <= response.status_code < 300:
132
- return response.text
133
-
134
- if response.status_code == 403:
135
- # Fail fast on 403 to trigger fallback immediately if needed
136
- return response.text
137
-
138
- if response.status_code in [429, 500, 502, 503]:
139
- if attempt < max_retries - 1:
140
- # Minimal sleep for speed
141
- time.sleep(0.5)
142
- continue
143
 
144
- return None
 
 
 
 
 
 
 
 
 
 
 
 
 
 
145
 
146
- except Exception as e:
147
- # If curl_cffi fails, we skip the slow standard requests fallback
148
- # and return None to let the main loop decide (or trigger Playwright if configured)
149
- logger.warning(f"Fast fetch failed for {url}: {e}")
150
-
151
  return None
152
 
153
  def _fetch_playwright(self, url):
 
66
  # Tier 2: Cloudscraper (Intermediate, handles JS challenges)
67
  logger.info(f"Tier 1 failed. Trying Cloudscraper info for {url}...")
68
  try:
69
+ resp = self.scraper.get(url, timeout=15)
70
  if 200 <= resp.status_code < 300:
71
  if not self._is_blocked(resp.text):
72
  return resp.text
 
115
 
116
  def _fetch_http(self, url):
117
  """
118
+ Fast HTTP fetch with aggressive timeouts and resilient fallback.
119
+ 1. Try curl_cffi with 12s timeout
120
+ 2. If curl_cffi times out or fails (e.g. TLS/H2 fingerprint issues), fallback to standard requests
121
  """
122
+ try:
123
+ self.session.headers["User-Agent"] = self.ua.random
124
+ response = self.session.get(url, timeout=12, allow_redirects=True)
125
+ if 200 <= response.status_code < 300:
126
+ return response.text
127
+ if response.status_code == 403:
128
+ return response.text
129
+ except Exception as e:
130
+ logger.warning(f"curl_cffi fetch failed for {url}: {e}. Trying standard requests fallback...")
 
 
 
 
 
 
 
 
 
 
 
 
 
 
131
 
132
+ # Fallback to standard requests (handles servers where curl_cffi TLS/HTTP2 hangs)
133
+ try:
134
+ import requests as std_requests
135
+ headers = {
136
+ "User-Agent": self.ua.random,
137
+ "Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8",
138
+ "Accept-Language": "en-US,en;q=0.9",
139
+ }
140
+ resp = std_requests.get(url, headers=headers, timeout=12, allow_redirects=True)
141
+ if 200 <= resp.status_code < 300:
142
+ return resp.text
143
+ if resp.status_code == 403:
144
+ return resp.text
145
+ except Exception as e2:
146
+ logger.warning(f"Standard requests fallback failed for {url}: {e2}")
147
 
 
 
 
 
 
148
  return None
149
 
150
  def _fetch_playwright(self, url):