GoshawkVortexAI commited on
Commit
c7949b3
·
verified ·
1 Parent(s): 7058d76

Delete data_fetcher.py

Browse files
Files changed (1) hide show
  1. data_fetcher.py +0 -113
data_fetcher.py DELETED
@@ -1,113 +0,0 @@
1
- import time
2
- import logging
3
- from typing import Dict, List, Optional
4
-
5
- import requests
6
- import numpy as np
7
- import pandas as pd
8
-
9
- from config import (
10
- OKX_BASE_URL,
11
- TIMEFRAME,
12
- CANDLE_LIMIT,
13
- REQUEST_DELAY,
14
- REQUEST_RETRIES,
15
- REQUEST_TIMEOUT,
16
- )
17
-
18
- logger = logging.getLogger(__name__)
19
-
20
-
21
- def _parse_candle(bar: list) -> dict:
22
- return {
23
- "timestamp": pd.to_datetime(int(bar[0]), unit="ms", utc=True),
24
- "open": float(bar[1]),
25
- "high": float(bar[2]),
26
- "low": float(bar[3]),
27
- "close": float(bar[4]),
28
- "volume": float(bar[5]),
29
- "quote_volume": float(bar[6]) if len(bar) > 6 else np.nan,
30
- }
31
-
32
-
33
- def fetch_ohlcv(
34
- symbol: str,
35
- timeframe: str = TIMEFRAME,
36
- limit: int = CANDLE_LIMIT,
37
- ) -> Optional[pd.DataFrame]:
38
- url = f"{OKX_BASE_URL}/api/v5/market/candles"
39
- params = {"instId": symbol, "bar": timeframe, "limit": str(limit)}
40
-
41
- for attempt in range(REQUEST_RETRIES):
42
- try:
43
- resp = requests.get(url, params=params, timeout=REQUEST_TIMEOUT)
44
- resp.raise_for_status()
45
- payload = resp.json()
46
-
47
- if payload.get("code") != "0":
48
- logger.warning(f"{symbol}: API error — {payload.get('msg', 'unknown')}")
49
- return None
50
-
51
- raw = payload.get("data")
52
- if not raw:
53
- logger.warning(f"{symbol}: empty data returned")
54
- return None
55
-
56
- records = [_parse_candle(bar) for bar in reversed(raw)]
57
- df = pd.DataFrame(records).set_index("timestamp").sort_index()
58
- time.sleep(REQUEST_DELAY)
59
- return df
60
-
61
- except requests.exceptions.Timeout:
62
- logger.warning(f"{symbol}: timeout on attempt {attempt + 1}")
63
- except requests.exceptions.RequestException as exc:
64
- logger.warning(f"{symbol}: request error on attempt {attempt + 1}: {exc}")
65
-
66
- time.sleep(REQUEST_DELAY * (attempt + 2))
67
-
68
- logger.error(f"{symbol}: all {REQUEST_RETRIES} fetch attempts failed")
69
- return None
70
-
71
-
72
- def fetch_instruments(inst_type: str = "SPOT") -> List[str]:
73
- url = f"{OKX_BASE_URL}/api/v5/public/instruments"
74
- params = {"instType": inst_type}
75
- try:
76
- resp = requests.get(url, params=params, timeout=REQUEST_TIMEOUT)
77
- resp.raise_for_status()
78
- payload = resp.json()
79
- if payload.get("code") != "0":
80
- return []
81
- return [
82
- item["instId"]
83
- for item in payload.get("data", [])
84
- if item.get("quoteCcy") == "USDT" and item.get("state") == "live"
85
- ]
86
- except requests.exceptions.RequestException as exc:
87
- logger.error(f"Failed to fetch instruments: {exc}")
88
- return []
89
-
90
-
91
- def fetch_multiple(
92
- symbols: List[str],
93
- timeframe: str = TIMEFRAME,
94
- limit: int = CANDLE_LIMIT,
95
- min_bars: int = 50,
96
- progress_callback=None,
97
- ) -> Dict[str, pd.DataFrame]:
98
- results: Dict[str, pd.DataFrame] = {}
99
- total = len(symbols)
100
-
101
- for i, sym in enumerate(symbols, 1):
102
- if progress_callback:
103
- progress_callback(i, total, sym)
104
- else:
105
- logger.info(f"Fetching {sym} ({i}/{total})")
106
-
107
- df = fetch_ohlcv(sym, timeframe=timeframe, limit=limit)
108
- if df is not None and len(df) >= min_bars:
109
- results[sym] = df
110
- else:
111
- logger.warning(f"{sym}: skipped (insufficient data)")
112
-
113
- return results