kronos-filter / fetcher.py
az30667176-sudo
feat: use Yahoo v8 chart JSON API directly (drops yfinance/stooq deps)
9d9d6d0
Raw History Blame Contribute Delete
5.85 kB
"""
fetcher.py — OHLCV fetcher (HF Space version).
Uses Yahoo Finance v8 chart JSON API directly, with browser-like headers.
This is the same endpoint kronos-filter.vercel.app backend code (Next.js)
can also call client-side, so it's proven to work from browsers.
"""
from __future__ import annotations
import logging
import random
import time
from dataclasses import dataclass
from datetime import datetime, timedelta
import pandas as pd
import requests
logger = logging.getLogger(__name__)
UA = (
"Mozilla/5.0 (Windows NT 10.0; Win64; x64) "
"AppleWebKit/537.36 (KHTML, like Gecko) "
"Chrome/130.0.0.0 Safari/537.36"
)
@dataclass
class TickerData:
ticker: str
df: pd.DataFrame
timestamps: pd.Series
def _yahoo_v8(ticker: str, period1: int, period2: int, interval: str = "1d") -> pd.DataFrame | None:
"""Yahoo v8 chart JSON endpoint — different from v7 download that yfinance uses."""
url = f"https://query1.finance.yahoo.com/v8/finance/chart/{ticker}"
params = {
"period1": period1,
"period2": period2,
"interval": interval,
"events": "history",
"includeAdjustedClose": "true",
}
headers = {
"User-Agent": UA,
"Accept": "application/json",
"Accept-Language": "en-US,en;q=0.9",
}
try:
r = requests.get(url, params=params, headers=headers, timeout=15)
if r.status_code != 200:
logger.warning(f"[{ticker}] yahoo v8 HTTP {r.status_code}")
return None
data = r.json()
result = data.get("chart", {}).get("result")
if not result:
err = data.get("chart", {}).get("error")
logger.warning(f"[{ticker}] yahoo v8 no result: {err}")
return None
r0 = result[0]
ts = r0.get("timestamp", [])
indicators = r0.get("indicators", {})
quote = (indicators.get("quote") or [{}])[0]
adj = (indicators.get("adjclose") or [{}])[0]
opens = quote.get("open", [])
highs = quote.get("high", [])
lows = quote.get("low", [])
closes = quote.get("close", [])
volumes = quote.get("volume", [])
adj_closes = adj.get("adjclose", []) if adj else closes
if not ts or not opens:
return None
df = pd.DataFrame({
"date": pd.to_datetime(ts, unit="s"),
"open": opens,
"high": highs,
"low": lows,
"close": adj_closes if len(adj_closes) == len(ts) else closes,
"volume": volumes,
})
df.set_index("date", inplace=True)
df = df.dropna(subset=["open", "high", "low", "close"])
if df.empty:
return None
# Volume can be None for some bars; fill with 0
df["volume"] = df["volume"].fillna(0)
return df
except Exception as exc:
logger.warning(f"[{ticker}] yahoo v8 error: {exc}")
return None
def _download_one(ticker: str, lookback_days: int, interval: str = "1d") -> pd.DataFrame | None:
multiplier = 1.6 if interval == "1d" else 2.0
period_days = int(lookback_days * multiplier) + 60
end_dt = datetime.utcnow()
start_dt = end_dt - timedelta(days=period_days)
p1 = int(start_dt.timestamp())
p2 = int(end_dt.timestamp())
for attempt in range(4):
df = _yahoo_v8(ticker, p1, p2, interval)
if df is not None and not df.empty:
logger.info(f"[{ticker}] ✓ ({len(df)} bars)")
return df
if attempt < 3:
delay = (2 ** attempt) + random.uniform(0.5, 1.5)
logger.info(f"[{ticker}] retry {attempt + 1}/3 after {delay:.1f}s")
time.sleep(delay)
logger.warning(f"[{ticker}] all attempts failed")
return None
def fetch_aligned(
tickers: list[str],
lookback: int,
interval: str = "1d",
) -> tuple[list[TickerData], list[str]]:
raw = {}
failed: list[str] = []
for i, t in enumerate(tickers):
if i > 0:
time.sleep(random.uniform(0.2, 0.4))
df = _download_one(t, lookback_days=lookback, interval=interval)
if df is None or len(df) < lookback:
failed.append(t)
continue
raw[t] = df
if not raw:
return [], failed
common_idx = None
for df in raw.values():
common_idx = df.index if common_idx is None else common_idx.intersection(df.index)
if common_idx is None or len(common_idx) < lookback:
logger.warning(
f"Common date intersection too short ({0 if common_idx is None else len(common_idx)} < {lookback})"
)
if common_idx is None:
return [], tickers
common_idx = common_idx.sort_values()
common_idx = common_idx[-lookback:]
result: list[TickerData] = []
for t, df in raw.items():
aligned = df.loc[df.index.intersection(common_idx)].sort_index()
if len(aligned) < lookback:
failed.append(t)
continue
aligned = aligned.tail(lookback).copy()
aligned.reset_index(inplace=True)
ts_col = aligned.columns[0]
timestamps = pd.to_datetime(aligned[ts_col])
aligned = aligned[["open", "high", "low", "close", "volume"]]
result.append(TickerData(ticker=t, df=aligned, timestamps=timestamps))
return result, failed
def build_future_timestamps(last_ts: pd.Timestamp, pred_len: int, interval: str = "1d") -> pd.Series:
if interval == "1d":
future = pd.bdate_range(start=last_ts + pd.Timedelta(days=1), periods=pred_len)
return pd.Series(future)
freq_map = {"1h": "H", "30m": "30min", "15m": "15min", "5m": "5min", "1wk": "W"}
freq = freq_map.get(interval, "D")
future = pd.date_range(start=last_ts + pd.Timedelta(minutes=1), periods=pred_len, freq=freq)
return pd.Series(future)