feat: use Stooq as primary data source (yfinance heavily rate-limited on HF IPs)

fetcher.py

@@ -1,9 +1,9 @@
 """
-fetcher.py — Yahoo Finance OHLCV fetcher for Kronos Analyzer.
+fetcher.py — OHLCV fetcher for Kronos Analyzer (HF Space version).
 
-On HF Spaces the IP is shared with many users and yfinance tends to hit
-rate limits. This version uses curl_cffi browser impersonation + per-ticker
-retries + random jitter delays.
+Primary source: Stooq (via pandas-datareader). Free, no API key, and doesn't
+rate-limit cloud provider IPs the way Yahoo does.
+Fallback: yfinance.
 """
 from __future__ import annotations
 
@@ -14,31 +14,9 @@ from dataclasses import dataclass
 from datetime import datetime, timedelta
 
 import pandas as pd
-import yfinance as yf
 
 logger = logging.getLogger(__name__)
 
-# Shared session across requests; curl_cffi impersonates a real browser
-_SESSION = None
-
-
-def _get_session():
-    global _SESSION
-    if _SESSION is not None:
-        return _SESSION
-    try:
-        from curl_cffi import requests as curl_requests
-        _SESSION = curl_requests.Session(impersonate="chrome")
-    except Exception:
-        import requests as py_requests
-        _SESSION = py_requests.Session()
-        _SESSION.headers.update({
-            "User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) "
-                          "AppleWebKit/537.36 (KHTML, like Gecko) "
-                          "Chrome/130.0.0.0 Safari/537.36"
-        })
-    return _SESSION
-
 
 @dataclass
 class TickerData:
@@ -47,57 +25,92 @@ class TickerData:
     timestamps: pd.Series
 
 
-def _download_one_once(ticker: str, start, end, interval: str) -> pd.DataFrame | None:
-    """Single attempt, no retry."""
-    session = _get_session()
+def _stooq_download(ticker: str, start, end) -> pd.DataFrame | None:
+    """Fetch from Stooq via pandas-datareader."""
+    try:
+        from pandas_datareader import data as pdr
+        df = pdr.DataReader(ticker, "stooq", start, end)
+        if df is None or df.empty:
+            return None
+        # Stooq returns descending order (newest first); sort ascending
+        df = df.sort_index(ascending=True)
+        # Normalize column names
+        df.columns = [str(c).lower() for c in df.columns]
+        needed = ["open", "high", "low", "close", "volume"]
+        for col in needed:
+            if col not in df.columns:
+                logger.warning(f"[{ticker}] stooq missing column {col}")
+                return None
+        df = df[needed].dropna()
+        if df.empty:
+            return None
+        return df
+    except Exception as exc:
+        logger.warning(f"[{ticker}] stooq error: {exc}")
+        return None
+
+
+def _yfinance_download(ticker: str, start, end, interval: str = "1d") -> pd.DataFrame | None:
+    """Fallback: yfinance direct."""
     try:
-        tk = yf.Ticker(ticker, session=session)
+        import yfinance as yf
+        tk = yf.Ticker(ticker)
         df = tk.history(
             start=start.date().isoformat(),
             end=end.date().isoformat(),
             interval=interval,
             auto_adjust=True,
         )
-    except Exception as exc:
-        logger.warning(f"[{ticker}] history() error: {exc}")
-        return None
-
-    if df is None or df.empty:
-        return None
-
-    # Normalize columns
-    df.columns = [str(c).lower() for c in df.columns]
-    needed = ["open", "high", "low", "close", "volume"]
-    for col in needed:
-        if col not in df.columns:
-            logger.warning(f"[{ticker}] missing column {col}; skipping")
+        if df is None or df.empty:
             return None
-    df = df[needed].dropna()
-    if df.empty:
+        df.columns = [str(c).lower() for c in df.columns]
+        needed = ["open", "high", "low", "close", "volume"]
+        for col in needed:
+            if col not in df.columns:
+                return None
+        df = df[needed].dropna()
+        if df.empty:
+            return None
+        return df
+    except Exception as exc:
+        logger.warning(f"[{ticker}] yfinance error: {exc}")
         return None
-    return df
 
 
 def _download_one(ticker: str, lookback_days: int, interval: str = "1d") -> pd.DataFrame | None:
-    """Download with retry + exponential backoff."""
+    """Try Stooq first (cloud-friendly), fall back to yfinance."""
     multiplier = 1.6 if interval == "1d" else 2.0
     period_days = int(lookback_days * multiplier) + 60
     end = datetime.utcnow()
     start = end - timedelta(days=period_days)
 
-    max_retries = 4
-    for attempt in range(max_retries):
-        df = _download_one_once(ticker, start, end, interval)
+    # Stooq needs ticker uppercase. For US stocks on Stooq, append `.US` if missing.
+    stooq_ticker = ticker.upper()
+    if "." not in stooq_ticker:
+        stooq_ticker_us = f"{stooq_ticker}.US"
+    else:
+        stooq_ticker_us = stooq_ticker
+
+    # Try Stooq with retry
+    for attempt in range(3):
+        df = _stooq_download(stooq_ticker_us, start, end)
         if df is not None and not df.empty:
+            logger.info(f"[{ticker}] ✓ stooq ({len(df)} bars)")
             return df
+        if attempt < 2:
+            time.sleep(random.uniform(0.5, 1.5))
 
-        if attempt < max_retries - 1:
-            # Exponential backoff with jitter
-            delay = (2 ** attempt) + random.uniform(0.5, 1.5)
-            logger.warning(f"[{ticker}] retry {attempt + 1}/{max_retries - 1} after {delay:.1f}s")
-            time.sleep(delay)
+    # Fallback to yfinance
+    logger.info(f"[{ticker}] stooq failed, trying yfinance")
+    for attempt in range(3):
+        df = _yfinance_download(ticker, start, end, interval)
+        if df is not None and not df.empty:
+            logger.info(f"[{ticker}] ✓ yfinance ({len(df)} bars)")
+            return df
+        if attempt < 2:
+            time.sleep(2 ** attempt + random.uniform(0.5, 1.5))
 
-    logger.warning(f"[{ticker}] failed after {max_retries} attempts")
+    logger.warning(f"[{ticker}] both sources failed")
     return None
 
 
@@ -109,9 +122,8 @@ def fetch_aligned(
     raw = {}
     failed: list[str] = []
     for i, t in enumerate(tickers):
-        # Small jitter between requests to avoid bursty patterns
         if i > 0:
-            time.sleep(random.uniform(0.3, 0.8))
+            time.sleep(random.uniform(0.2, 0.5))
         df = _download_one(t, lookback_days=lookback, interval=interval)
         if df is None or len(df) < lookback:
             failed.append(t)

requirements.txt

@@ -6,5 +6,8 @@ safetensors==0.6.2
 tqdm==4.67.1
 pandas==2.2.2
 numpy<2.0
-# Pin yfinance to version before it added the websocket-based live module
+# Stooq as primary source (no API key, no cloud IP rate limit)
+pandas-datareader>=0.10.0
+# yfinance as fallback (often rate-limited on cloud IPs)
 yfinance==0.2.52
+websockets>=12.0