perf: optimize ingestion worker polling by migrating to bulk yfinance download queries
Browse files- worker/worker.py +54 -22
worker/worker.py
CHANGED
|
@@ -29,40 +29,71 @@ aggregation_buffer: Dict[str, Dict[str, List]] = {}
|
|
| 29 |
COOLDOWN_UNTIL = None
|
| 30 |
COOLDOWN_BACKOFF_SEC = 60
|
| 31 |
|
| 32 |
-
async def
|
| 33 |
"""
|
| 34 |
-
Fetch current stock price and cumulative volume
|
| 35 |
Respects global rate-limiting cooldown if Yahoo Finance blocks the IP.
|
| 36 |
"""
|
| 37 |
global COOLDOWN_UNTIL, COOLDOWN_BACKOFF_SEC
|
| 38 |
|
|
|
|
|
|
|
|
|
|
| 39 |
now_utc = datetime.datetime.now(datetime.timezone.utc)
|
| 40 |
if COOLDOWN_UNTIL and now_utc < COOLDOWN_UNTIL:
|
| 41 |
-
return
|
| 42 |
|
| 43 |
try:
|
|
|
|
| 44 |
# Run synchronous yfinance operations in a thread pool to avoid blocking the event loop
|
| 45 |
loop = asyncio.get_event_loop()
|
| 46 |
-
yf_ticker = yf.Ticker(ticker)
|
| 47 |
-
|
| 48 |
-
# fast_info retrieves basic metrics quickly
|
| 49 |
-
info = await loop.run_in_executor(None, lambda: yf_ticker.fast_info)
|
| 50 |
|
| 51 |
-
|
| 52 |
-
|
|
|
|
|
|
|
|
|
|
| 53 |
|
| 54 |
-
if
|
| 55 |
-
return
|
|
|
|
|
|
|
|
|
|
|
|
|
| 56 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 57 |
# Reset backoff on success
|
| 58 |
COOLDOWN_BACKOFF_SEC = 60
|
| 59 |
COOLDOWN_UNTIL = None
|
| 60 |
|
| 61 |
-
return
|
| 62 |
-
"price": float(price),
|
| 63 |
-
"volume": int(volume),
|
| 64 |
-
"timestamp": now_utc
|
| 65 |
-
}
|
| 66 |
except Exception as e:
|
| 67 |
err_str = str(e).lower()
|
| 68 |
if "too many requests" in err_str or "rate limit" in err_str or "429" in err_str:
|
|
@@ -70,8 +101,9 @@ async def get_latest_stock_data(ticker: str) -> Optional[dict]:
|
|
| 70 |
print(f"Worker rate limited. Entering yfinance cooldown for {COOLDOWN_BACKOFF_SEC}s. Error: {e}")
|
| 71 |
COOLDOWN_BACKOFF_SEC = min(COOLDOWN_BACKOFF_SEC * 2, 900) # Double backoff up to 15 mins
|
| 72 |
else:
|
| 73 |
-
print(f"Error fetching data for {
|
| 74 |
-
return
|
|
|
|
| 75 |
|
| 76 |
|
| 77 |
async def aggregate_and_save_candle(db, ticker: str, timestamp: datetime.datetime):
|
|
@@ -160,8 +192,11 @@ async def main():
|
|
| 160 |
if t not in aggregation_buffer:
|
| 161 |
aggregation_buffer[t]= {"prices": [], "volumes": []}
|
| 162 |
|
|
|
|
|
|
|
|
|
|
| 163 |
for ticker in dynamic_tickers:
|
| 164 |
-
tick =
|
| 165 |
|
| 166 |
if tick:
|
| 167 |
aggregation_buffer[ticker]["prices"].append(tick["price"])
|
|
@@ -179,9 +214,6 @@ async def main():
|
|
| 179 |
|
| 180 |
if minute_changed:
|
| 181 |
await aggregate_and_save_candle(db, ticker, now)
|
| 182 |
-
|
| 183 |
-
# Stagger calls to yfinance to prevent rate limiting
|
| 184 |
-
await asyncio.sleep(1.5)
|
| 185 |
|
| 186 |
if minute_changed:
|
| 187 |
last_minute = now.minute
|
|
|
|
| 29 |
COOLDOWN_UNTIL = None
|
| 30 |
COOLDOWN_BACKOFF_SEC = 60
|
| 31 |
|
| 32 |
+
async def get_latest_stock_data_bulk(tickers: List[str]) -> Dict[str, dict]:
|
| 33 |
"""
|
| 34 |
+
Fetch current stock price and cumulative volume for multiple tickers in a single bulk request.
|
| 35 |
Respects global rate-limiting cooldown if Yahoo Finance blocks the IP.
|
| 36 |
"""
|
| 37 |
global COOLDOWN_UNTIL, COOLDOWN_BACKOFF_SEC
|
| 38 |
|
| 39 |
+
if not tickers:
|
| 40 |
+
return {}
|
| 41 |
+
|
| 42 |
now_utc = datetime.datetime.now(datetime.timezone.utc)
|
| 43 |
if COOLDOWN_UNTIL and now_utc < COOLDOWN_UNTIL:
|
| 44 |
+
return {}
|
| 45 |
|
| 46 |
try:
|
| 47 |
+
import pandas as pd
|
| 48 |
# Run synchronous yfinance operations in a thread pool to avoid blocking the event loop
|
| 49 |
loop = asyncio.get_event_loop()
|
|
|
|
|
|
|
|
|
|
|
|
|
| 50 |
|
| 51 |
+
# Download 1d of 2m interval candles for all tickers in a single bulk request
|
| 52 |
+
df = await loop.run_in_executor(
|
| 53 |
+
None,
|
| 54 |
+
lambda: yf.download(tickers=tickers, period="1d", interval="2m", group_by="ticker", progress=False)
|
| 55 |
+
)
|
| 56 |
|
| 57 |
+
if df.empty:
|
| 58 |
+
return {}
|
| 59 |
+
|
| 60 |
+
results = {}
|
| 61 |
+
for ticker in tickers:
|
| 62 |
+
ticker_upper = ticker.upper()
|
| 63 |
|
| 64 |
+
# Check if columns are a MultiIndex (standard for group_by="ticker")
|
| 65 |
+
if isinstance(df.columns, pd.MultiIndex):
|
| 66 |
+
# Ensure the ticker level exists in the columns
|
| 67 |
+
if ticker_upper not in df.columns.levels[0]:
|
| 68 |
+
continue
|
| 69 |
+
ticker_data = df[ticker_upper]
|
| 70 |
+
else:
|
| 71 |
+
ticker_data = df
|
| 72 |
+
|
| 73 |
+
if ticker_data.empty:
|
| 74 |
+
continue
|
| 75 |
+
|
| 76 |
+
# Find the last valid row (contains Close price)
|
| 77 |
+
valid_rows = ticker_data.dropna(subset=["Close"])
|
| 78 |
+
if valid_rows.empty:
|
| 79 |
+
continue
|
| 80 |
+
|
| 81 |
+
last_row = valid_rows.iloc[-1]
|
| 82 |
+
price = last_row["Close"]
|
| 83 |
+
volume = last_row["Volume"]
|
| 84 |
+
|
| 85 |
+
if price is not None and volume is not None:
|
| 86 |
+
results[ticker] = {
|
| 87 |
+
"price": float(price),
|
| 88 |
+
"volume": int(volume),
|
| 89 |
+
"timestamp": now_utc
|
| 90 |
+
}
|
| 91 |
+
|
| 92 |
# Reset backoff on success
|
| 93 |
COOLDOWN_BACKOFF_SEC = 60
|
| 94 |
COOLDOWN_UNTIL = None
|
| 95 |
|
| 96 |
+
return results
|
|
|
|
|
|
|
|
|
|
|
|
|
| 97 |
except Exception as e:
|
| 98 |
err_str = str(e).lower()
|
| 99 |
if "too many requests" in err_str or "rate limit" in err_str or "429" in err_str:
|
|
|
|
| 101 |
print(f"Worker rate limited. Entering yfinance cooldown for {COOLDOWN_BACKOFF_SEC}s. Error: {e}")
|
| 102 |
COOLDOWN_BACKOFF_SEC = min(COOLDOWN_BACKOFF_SEC * 2, 900) # Double backoff up to 15 mins
|
| 103 |
else:
|
| 104 |
+
print(f"Error fetching bulk data for tickers {tickers}: {str(e)}")
|
| 105 |
+
return {}
|
| 106 |
+
|
| 107 |
|
| 108 |
|
| 109 |
async def aggregate_and_save_candle(db, ticker: str, timestamp: datetime.datetime):
|
|
|
|
| 192 |
if t not in aggregation_buffer:
|
| 193 |
aggregation_buffer[t]= {"prices": [], "volumes": []}
|
| 194 |
|
| 195 |
+
# Fetch data in bulk for all watchlist tickers
|
| 196 |
+
bulk_ticks = await get_latest_stock_data_bulk(dynamic_tickers)
|
| 197 |
+
|
| 198 |
for ticker in dynamic_tickers:
|
| 199 |
+
tick = bulk_ticks.get(ticker)
|
| 200 |
|
| 201 |
if tick:
|
| 202 |
aggregation_buffer[ticker]["prices"].append(tick["price"])
|
|
|
|
| 214 |
|
| 215 |
if minute_changed:
|
| 216 |
await aggregate_and_save_candle(db, ticker, now)
|
|
|
|
|
|
|
|
|
|
| 217 |
|
| 218 |
if minute_changed:
|
| 219 |
last_minute = now.minute
|