""" fetcher.py — OHLCV fetcher (HF Space version). Uses Yahoo Finance v8 chart JSON API directly, with browser-like headers. This is the same endpoint kronos-filter.vercel.app backend code (Next.js) can also call client-side, so it's proven to work from browsers. """ from __future__ import annotations import logging import random import time from dataclasses import dataclass from datetime import datetime, timedelta import pandas as pd import requests logger = logging.getLogger(__name__) UA = ( "Mozilla/5.0 (Windows NT 10.0; Win64; x64) " "AppleWebKit/537.36 (KHTML, like Gecko) " "Chrome/130.0.0.0 Safari/537.36" ) @dataclass class TickerData: ticker: str df: pd.DataFrame timestamps: pd.Series def _yahoo_v8(ticker: str, period1: int, period2: int, interval: str = "1d") -> pd.DataFrame | None: """Yahoo v8 chart JSON endpoint — different from v7 download that yfinance uses.""" url = f"https://query1.finance.yahoo.com/v8/finance/chart/{ticker}" params = { "period1": period1, "period2": period2, "interval": interval, "events": "history", "includeAdjustedClose": "true", } headers = { "User-Agent": UA, "Accept": "application/json", "Accept-Language": "en-US,en;q=0.9", } try: r = requests.get(url, params=params, headers=headers, timeout=15) if r.status_code != 200: logger.warning(f"[{ticker}] yahoo v8 HTTP {r.status_code}") return None data = r.json() result = data.get("chart", {}).get("result") if not result: err = data.get("chart", {}).get("error") logger.warning(f"[{ticker}] yahoo v8 no result: {err}") return None r0 = result[0] ts = r0.get("timestamp", []) indicators = r0.get("indicators", {}) quote = (indicators.get("quote") or [{}])[0] adj = (indicators.get("adjclose") or [{}])[0] opens = quote.get("open", []) highs = quote.get("high", []) lows = quote.get("low", []) closes = quote.get("close", []) volumes = quote.get("volume", []) adj_closes = adj.get("adjclose", []) if adj else closes if not ts or not opens: return None df = pd.DataFrame({ "date": pd.to_datetime(ts, unit="s"), "open": opens, "high": highs, "low": lows, "close": adj_closes if len(adj_closes) == len(ts) else closes, "volume": volumes, }) df.set_index("date", inplace=True) df = df.dropna(subset=["open", "high", "low", "close"]) if df.empty: return None # Volume can be None for some bars; fill with 0 df["volume"] = df["volume"].fillna(0) return df except Exception as exc: logger.warning(f"[{ticker}] yahoo v8 error: {exc}") return None def _download_one(ticker: str, lookback_days: int, interval: str = "1d") -> pd.DataFrame | None: multiplier = 1.6 if interval == "1d" else 2.0 period_days = int(lookback_days * multiplier) + 60 end_dt = datetime.utcnow() start_dt = end_dt - timedelta(days=period_days) p1 = int(start_dt.timestamp()) p2 = int(end_dt.timestamp()) for attempt in range(4): df = _yahoo_v8(ticker, p1, p2, interval) if df is not None and not df.empty: logger.info(f"[{ticker}] ✓ ({len(df)} bars)") return df if attempt < 3: delay = (2 ** attempt) + random.uniform(0.5, 1.5) logger.info(f"[{ticker}] retry {attempt + 1}/3 after {delay:.1f}s") time.sleep(delay) logger.warning(f"[{ticker}] all attempts failed") return None def fetch_aligned( tickers: list[str], lookback: int, interval: str = "1d", ) -> tuple[list[TickerData], list[str]]: raw = {} failed: list[str] = [] for i, t in enumerate(tickers): if i > 0: time.sleep(random.uniform(0.2, 0.4)) df = _download_one(t, lookback_days=lookback, interval=interval) if df is None or len(df) < lookback: failed.append(t) continue raw[t] = df if not raw: return [], failed common_idx = None for df in raw.values(): common_idx = df.index if common_idx is None else common_idx.intersection(df.index) if common_idx is None or len(common_idx) < lookback: logger.warning( f"Common date intersection too short ({0 if common_idx is None else len(common_idx)} < {lookback})" ) if common_idx is None: return [], tickers common_idx = common_idx.sort_values() common_idx = common_idx[-lookback:] result: list[TickerData] = [] for t, df in raw.items(): aligned = df.loc[df.index.intersection(common_idx)].sort_index() if len(aligned) < lookback: failed.append(t) continue aligned = aligned.tail(lookback).copy() aligned.reset_index(inplace=True) ts_col = aligned.columns[0] timestamps = pd.to_datetime(aligned[ts_col]) aligned = aligned[["open", "high", "low", "close", "volume"]] result.append(TickerData(ticker=t, df=aligned, timestamps=timestamps)) return result, failed def build_future_timestamps(last_ts: pd.Timestamp, pred_len: int, interval: str = "1d") -> pd.Series: if interval == "1d": future = pd.bdate_range(start=last_ts + pd.Timedelta(days=1), periods=pred_len) return pd.Series(future) freq_map = {"1h": "H", "30m": "30min", "15m": "15min", "5m": "5min", "1wk": "W"} freq = freq_map.get(interval, "D") future = pd.date_range(start=last_ts + pd.Timedelta(minutes=1), periods=pred_len, freq=freq) return pd.Series(future)