Spaces:
Sleeping
Sleeping
Download fetcher.py from Kurtobe/kronos-filter: direct link, hf CLI and curl.
- Browser
- Download file 5.85 kB
-
https://huggingface.co/spaces/Kurtobe/kronos-filter/resolve/main/fetcher.py
- Command line
-
hf download hf://spaces/Kurtobe/kronos-filter/fetcher.py
-
curl -L -o fetcher.py https://huggingface.co/spaces/Kurtobe/kronos-filter/resolve/main/fetcher.py
5.85 kB
| """ | |
| fetcher.py — OHLCV fetcher (HF Space version). | |
| Uses Yahoo Finance v8 chart JSON API directly, with browser-like headers. | |
| This is the same endpoint kronos-filter.vercel.app backend code (Next.js) | |
| can also call client-side, so it's proven to work from browsers. | |
| """ | |
| from __future__ import annotations | |
| import logging | |
| import random | |
| import time | |
| from dataclasses import dataclass | |
| from datetime import datetime, timedelta | |
| import pandas as pd | |
| import requests | |
| logger = logging.getLogger(__name__) | |
| UA = ( | |
| "Mozilla/5.0 (Windows NT 10.0; Win64; x64) " | |
| "AppleWebKit/537.36 (KHTML, like Gecko) " | |
| "Chrome/130.0.0.0 Safari/537.36" | |
| ) | |
| class TickerData: | |
| ticker: str | |
| df: pd.DataFrame | |
| timestamps: pd.Series | |
| def _yahoo_v8(ticker: str, period1: int, period2: int, interval: str = "1d") -> pd.DataFrame | None: | |
| """Yahoo v8 chart JSON endpoint — different from v7 download that yfinance uses.""" | |
| url = f"https://query1.finance.yahoo.com/v8/finance/chart/{ticker}" | |
| params = { | |
| "period1": period1, | |
| "period2": period2, | |
| "interval": interval, | |
| "events": "history", | |
| "includeAdjustedClose": "true", | |
| } | |
| headers = { | |
| "User-Agent": UA, | |
| "Accept": "application/json", | |
| "Accept-Language": "en-US,en;q=0.9", | |
| } | |
| try: | |
| r = requests.get(url, params=params, headers=headers, timeout=15) | |
| if r.status_code != 200: | |
| logger.warning(f"[{ticker}] yahoo v8 HTTP {r.status_code}") | |
| return None | |
| data = r.json() | |
| result = data.get("chart", {}).get("result") | |
| if not result: | |
| err = data.get("chart", {}).get("error") | |
| logger.warning(f"[{ticker}] yahoo v8 no result: {err}") | |
| return None | |
| r0 = result[0] | |
| ts = r0.get("timestamp", []) | |
| indicators = r0.get("indicators", {}) | |
| quote = (indicators.get("quote") or [{}])[0] | |
| adj = (indicators.get("adjclose") or [{}])[0] | |
| opens = quote.get("open", []) | |
| highs = quote.get("high", []) | |
| lows = quote.get("low", []) | |
| closes = quote.get("close", []) | |
| volumes = quote.get("volume", []) | |
| adj_closes = adj.get("adjclose", []) if adj else closes | |
| if not ts or not opens: | |
| return None | |
| df = pd.DataFrame({ | |
| "date": pd.to_datetime(ts, unit="s"), | |
| "open": opens, | |
| "high": highs, | |
| "low": lows, | |
| "close": adj_closes if len(adj_closes) == len(ts) else closes, | |
| "volume": volumes, | |
| }) | |
| df.set_index("date", inplace=True) | |
| df = df.dropna(subset=["open", "high", "low", "close"]) | |
| if df.empty: | |
| return None | |
| # Volume can be None for some bars; fill with 0 | |
| df["volume"] = df["volume"].fillna(0) | |
| return df | |
| except Exception as exc: | |
| logger.warning(f"[{ticker}] yahoo v8 error: {exc}") | |
| return None | |
| def _download_one(ticker: str, lookback_days: int, interval: str = "1d") -> pd.DataFrame | None: | |
| multiplier = 1.6 if interval == "1d" else 2.0 | |
| period_days = int(lookback_days * multiplier) + 60 | |
| end_dt = datetime.utcnow() | |
| start_dt = end_dt - timedelta(days=period_days) | |
| p1 = int(start_dt.timestamp()) | |
| p2 = int(end_dt.timestamp()) | |
| for attempt in range(4): | |
| df = _yahoo_v8(ticker, p1, p2, interval) | |
| if df is not None and not df.empty: | |
| logger.info(f"[{ticker}] ✓ ({len(df)} bars)") | |
| return df | |
| if attempt < 3: | |
| delay = (2 ** attempt) + random.uniform(0.5, 1.5) | |
| logger.info(f"[{ticker}] retry {attempt + 1}/3 after {delay:.1f}s") | |
| time.sleep(delay) | |
| logger.warning(f"[{ticker}] all attempts failed") | |
| return None | |
| def fetch_aligned( | |
| tickers: list[str], | |
| lookback: int, | |
| interval: str = "1d", | |
| ) -> tuple[list[TickerData], list[str]]: | |
| raw = {} | |
| failed: list[str] = [] | |
| for i, t in enumerate(tickers): | |
| if i > 0: | |
| time.sleep(random.uniform(0.2, 0.4)) | |
| df = _download_one(t, lookback_days=lookback, interval=interval) | |
| if df is None or len(df) < lookback: | |
| failed.append(t) | |
| continue | |
| raw[t] = df | |
| if not raw: | |
| return [], failed | |
| common_idx = None | |
| for df in raw.values(): | |
| common_idx = df.index if common_idx is None else common_idx.intersection(df.index) | |
| if common_idx is None or len(common_idx) < lookback: | |
| logger.warning( | |
| f"Common date intersection too short ({0 if common_idx is None else len(common_idx)} < {lookback})" | |
| ) | |
| if common_idx is None: | |
| return [], tickers | |
| common_idx = common_idx.sort_values() | |
| common_idx = common_idx[-lookback:] | |
| result: list[TickerData] = [] | |
| for t, df in raw.items(): | |
| aligned = df.loc[df.index.intersection(common_idx)].sort_index() | |
| if len(aligned) < lookback: | |
| failed.append(t) | |
| continue | |
| aligned = aligned.tail(lookback).copy() | |
| aligned.reset_index(inplace=True) | |
| ts_col = aligned.columns[0] | |
| timestamps = pd.to_datetime(aligned[ts_col]) | |
| aligned = aligned[["open", "high", "low", "close", "volume"]] | |
| result.append(TickerData(ticker=t, df=aligned, timestamps=timestamps)) | |
| return result, failed | |
| def build_future_timestamps(last_ts: pd.Timestamp, pred_len: int, interval: str = "1d") -> pd.Series: | |
| if interval == "1d": | |
| future = pd.bdate_range(start=last_ts + pd.Timedelta(days=1), periods=pred_len) | |
| return pd.Series(future) | |
| freq_map = {"1h": "H", "30m": "30min", "15m": "15min", "5m": "5min", "1wk": "W"} | |
| freq = freq_map.get(interval, "D") | |
| future = pd.date_range(start=last_ts + pd.Timedelta(minutes=1), periods=pred_len, freq=freq) | |
| return pd.Series(future) | |