Karan6124 commited on
Commit
dbc57ea
·
1 Parent(s): e8b7080

perf: optimize ingestion worker polling by migrating to bulk yfinance download queries

Browse files
Files changed (1) hide show
  1. worker/worker.py +54 -22
worker/worker.py CHANGED
@@ -29,40 +29,71 @@ aggregation_buffer: Dict[str, Dict[str, List]] = {}
29
  COOLDOWN_UNTIL = None
30
  COOLDOWN_BACKOFF_SEC = 60
31
 
32
- async def get_latest_stock_data(ticker: str) -> Optional[dict]:
33
  """
34
- Fetch current stock price and cumulative volume using yfinance fast_info.
35
  Respects global rate-limiting cooldown if Yahoo Finance blocks the IP.
36
  """
37
  global COOLDOWN_UNTIL, COOLDOWN_BACKOFF_SEC
38
 
 
 
 
39
  now_utc = datetime.datetime.now(datetime.timezone.utc)
40
  if COOLDOWN_UNTIL and now_utc < COOLDOWN_UNTIL:
41
- return None
42
 
43
  try:
 
44
  # Run synchronous yfinance operations in a thread pool to avoid blocking the event loop
45
  loop = asyncio.get_event_loop()
46
- yf_ticker = yf.Ticker(ticker)
47
-
48
- # fast_info retrieves basic metrics quickly
49
- info = await loop.run_in_executor(None, lambda: yf_ticker.fast_info)
50
 
51
- price = info.last_price
52
- volume = info.last_volume
 
 
 
53
 
54
- if price is None or volume is None:
55
- return None
 
 
 
 
56
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
57
  # Reset backoff on success
58
  COOLDOWN_BACKOFF_SEC = 60
59
  COOLDOWN_UNTIL = None
60
 
61
- return {
62
- "price": float(price),
63
- "volume": int(volume),
64
- "timestamp": now_utc
65
- }
66
  except Exception as e:
67
  err_str = str(e).lower()
68
  if "too many requests" in err_str or "rate limit" in err_str or "429" in err_str:
@@ -70,8 +101,9 @@ async def get_latest_stock_data(ticker: str) -> Optional[dict]:
70
  print(f"Worker rate limited. Entering yfinance cooldown for {COOLDOWN_BACKOFF_SEC}s. Error: {e}")
71
  COOLDOWN_BACKOFF_SEC = min(COOLDOWN_BACKOFF_SEC * 2, 900) # Double backoff up to 15 mins
72
  else:
73
- print(f"Error fetching data for {ticker}: {str(e)}")
74
- return None
 
75
 
76
 
77
  async def aggregate_and_save_candle(db, ticker: str, timestamp: datetime.datetime):
@@ -160,8 +192,11 @@ async def main():
160
  if t not in aggregation_buffer:
161
  aggregation_buffer[t]= {"prices": [], "volumes": []}
162
 
 
 
 
163
  for ticker in dynamic_tickers:
164
- tick = await get_latest_stock_data(ticker)
165
 
166
  if tick:
167
  aggregation_buffer[ticker]["prices"].append(tick["price"])
@@ -179,9 +214,6 @@ async def main():
179
 
180
  if minute_changed:
181
  await aggregate_and_save_candle(db, ticker, now)
182
-
183
- # Stagger calls to yfinance to prevent rate limiting
184
- await asyncio.sleep(1.5)
185
 
186
  if minute_changed:
187
  last_minute = now.minute
 
29
  COOLDOWN_UNTIL = None
30
  COOLDOWN_BACKOFF_SEC = 60
31
 
32
+ async def get_latest_stock_data_bulk(tickers: List[str]) -> Dict[str, dict]:
33
  """
34
+ Fetch current stock price and cumulative volume for multiple tickers in a single bulk request.
35
  Respects global rate-limiting cooldown if Yahoo Finance blocks the IP.
36
  """
37
  global COOLDOWN_UNTIL, COOLDOWN_BACKOFF_SEC
38
 
39
+ if not tickers:
40
+ return {}
41
+
42
  now_utc = datetime.datetime.now(datetime.timezone.utc)
43
  if COOLDOWN_UNTIL and now_utc < COOLDOWN_UNTIL:
44
+ return {}
45
 
46
  try:
47
+ import pandas as pd
48
  # Run synchronous yfinance operations in a thread pool to avoid blocking the event loop
49
  loop = asyncio.get_event_loop()
 
 
 
 
50
 
51
+ # Download 1d of 2m interval candles for all tickers in a single bulk request
52
+ df = await loop.run_in_executor(
53
+ None,
54
+ lambda: yf.download(tickers=tickers, period="1d", interval="2m", group_by="ticker", progress=False)
55
+ )
56
 
57
+ if df.empty:
58
+ return {}
59
+
60
+ results = {}
61
+ for ticker in tickers:
62
+ ticker_upper = ticker.upper()
63
 
64
+ # Check if columns are a MultiIndex (standard for group_by="ticker")
65
+ if isinstance(df.columns, pd.MultiIndex):
66
+ # Ensure the ticker level exists in the columns
67
+ if ticker_upper not in df.columns.levels[0]:
68
+ continue
69
+ ticker_data = df[ticker_upper]
70
+ else:
71
+ ticker_data = df
72
+
73
+ if ticker_data.empty:
74
+ continue
75
+
76
+ # Find the last valid row (contains Close price)
77
+ valid_rows = ticker_data.dropna(subset=["Close"])
78
+ if valid_rows.empty:
79
+ continue
80
+
81
+ last_row = valid_rows.iloc[-1]
82
+ price = last_row["Close"]
83
+ volume = last_row["Volume"]
84
+
85
+ if price is not None and volume is not None:
86
+ results[ticker] = {
87
+ "price": float(price),
88
+ "volume": int(volume),
89
+ "timestamp": now_utc
90
+ }
91
+
92
  # Reset backoff on success
93
  COOLDOWN_BACKOFF_SEC = 60
94
  COOLDOWN_UNTIL = None
95
 
96
+ return results
 
 
 
 
97
  except Exception as e:
98
  err_str = str(e).lower()
99
  if "too many requests" in err_str or "rate limit" in err_str or "429" in err_str:
 
101
  print(f"Worker rate limited. Entering yfinance cooldown for {COOLDOWN_BACKOFF_SEC}s. Error: {e}")
102
  COOLDOWN_BACKOFF_SEC = min(COOLDOWN_BACKOFF_SEC * 2, 900) # Double backoff up to 15 mins
103
  else:
104
+ print(f"Error fetching bulk data for tickers {tickers}: {str(e)}")
105
+ return {}
106
+
107
 
108
 
109
  async def aggregate_and_save_candle(db, ticker: str, timestamp: datetime.datetime):
 
192
  if t not in aggregation_buffer:
193
  aggregation_buffer[t]= {"prices": [], "volumes": []}
194
 
195
+ # Fetch data in bulk for all watchlist tickers
196
+ bulk_ticks = await get_latest_stock_data_bulk(dynamic_tickers)
197
+
198
  for ticker in dynamic_tickers:
199
+ tick = bulk_ticks.get(ticker)
200
 
201
  if tick:
202
  aggregation_buffer[ticker]["prices"].append(tick["price"])
 
214
 
215
  if minute_changed:
216
  await aggregate_and_save_candle(db, ticker, now)
 
 
 
217
 
218
  if minute_changed:
219
  last_minute = now.minute