perf: implement centralized server market-data cache, staggered polling, and spot drift TTL

server.js

@@ -241,7 +241,7 @@ async function startAutopilot() {
   // 2. Subscribe to real-time Yahoo streams for all 6 symbols in the background
   // This keeps the server autopilot 100% active and rate-limit free!
   console.log('[Autopilot] Establishing background Yahoo monitoring feeds...');
-  for (const symbol of SYMBOLS) {
+  for (const [idx, symbol] of SYMBOLS.entries()) {
     const stream = new YahooStream();
     stream.subscribe(symbol, TIMEFRAME, {
       onTick: (price) => {
@@ -256,7 +256,7 @@ async function startAutopilot() {
           handleCandleUpdate(symbol, candle);
         }
       },
-    });
+    }, { pollMs: 10000, staggerMs: idx * 1500 });
     streams[symbol] = stream;
   }
 
@@ -295,6 +295,19 @@ app.get('/api/terminal-state', (req, res) => {
   });
 });
 
+// Cached market data: browsers read candles + live price from the server's own
+// poller cache instead of each hitting Yahoo. Reads in-memory state, no upstream call.
+app.get('/api/market-data', (req, res) => {
+  const symbol = req.query.symbol;
+  const candles = (symbol && historyMap[symbol]) ? historyMap[symbol].slice(-500) : [];
+  res.json({
+    symbol: symbol || null,
+    candles,
+    price: (symbol && livePrices[symbol] != null) ? livePrices[symbol] : null,
+    serverTime: Math.floor(Date.now() / 1000),
+  });
+});
+
 app.post('/api/take-trade', async (req, res) => {
   const { signal } = req.body;
   if (!signal) {

src/data/dataManager.js

@@ -8,6 +8,7 @@ import { fetchKlines, fetchTickerPrice }  from './binanceAPI.js';
 import { BinanceStream }              from './binanceWebSocket.js';
 import { fetchCandles as fetchYahooCandles, fetchQuote as fetchYahooQuote } from './yahooFinanceAPI.js';
 import { YahooStream }                from './yahooWebSocket.js';
+import { ServerStream }               from './serverStream.js';
 import { fetchCandles as fetchTwelveCandles, fetchQuote as fetchTwelveQuote } from './twelveDataAPI.js';
 import { TwelveDataStream }           from './twelveDataWebSocket.js';
 
@@ -57,6 +58,8 @@ export default class DataManager {
     this._binanceStream = new BinanceStream();
     /** @type {YahooStream} */
     this._yahooStream = new YahooStream();
+    /** @type {ServerStream} */
+    this._serverStream = new ServerStream();
     /** @type {TwelveDataStream} */
     this._twelveStream = new TwelveDataStream();
   }
@@ -96,6 +99,22 @@ export default class DataManager {
 
   /** @private */
   async _loadYahooHistory(symbol, timeframe) {
+    // For the server's native 15m timeframe, prefer the server's cached candles so
+    // the browser doesn't hit Yahoo directly. Fall back to Yahoo if the cache is cold.
+    if (timeframe === '15m') {
+      try {
+        const res = await fetch(`/api/market-data?symbol=${encodeURIComponent(symbol)}`);
+        if (res.ok) {
+          const data = await res.json();
+          if (data.candles && data.candles.length > 50) {
+            console.log(`[DataManager] Loaded ${data.candles.length} candles from server cache: ${symbol}`);
+            return data.candles;
+          }
+        }
+      } catch (err) {
+        console.warn(`[DataManager] Server cache unavailable for ${symbol}, falling back to Yahoo:`, err.message);
+      }
+    }
     console.log(`[DataManager] Loading Yahoo history: ${symbol} ${timeframe}`);
     return fetchYahooCandles(symbol, timeframe);
   }
@@ -146,6 +165,11 @@ export default class DataManager {
     if (config.provider === 'binance') {
       console.log(`[DataManager] Subscribing Binance stream: ${config.wsSymbol} ${tf.binance}`);
       this._binanceStream.subscribe(config.wsSymbol, tf.binance, callbacks);
+    } else if (timeframe === '15m') {
+      // Default timeframe served from the server's shared cache so each browser
+      // doesn't poll Yahoo directly (server is the single upstream poller).
+      console.log(`[DataManager] Subscribing server-cache stream: ${symbol} ${timeframe}`);
+      this._serverStream.subscribe(symbol, timeframe, callbacks);
     } else {
       console.log(`[DataManager] Subscribing Yahoo stream: ${symbol} ${timeframe}`);
       this._yahooStream.subscribe(symbol, timeframe, callbacks);
@@ -157,6 +181,7 @@ export default class DataManager {
     console.log('[DataManager] Unsubscribing all streams');
     this._binanceStream.unsubscribe();
     this._yahooStream.unsubscribe();
+    this._serverStream.unsubscribe();
     this._twelveStream.unsubscribe();
   }
 

src/data/serverStream.js

@@ -0,0 +1,103 @@
+/**
+ * serverStream.js
+ * Drop-in replacement for YahooStream that reads candles + live price from the
+ * backend's own cached market data (/api/market-data) instead of polling Yahoo
+ * directly. This makes the server the single upstream data client, so additional
+ * browser viewers cost only a cheap cache read rather than another Yahoo request.
+ */
+
+const TIMEFRAME_SECONDS = {
+  '1m':  60,
+  '5m':  300,
+  '15m': 900,
+  '1H':  3600,
+  '4H':  14400,
+  '1D':  86400,
+};
+
+export class ServerStream {
+  constructor() {
+    this._timer = null;
+    this._stopped = false;
+    this._symbol = null;
+    this._timeframe = null;
+    this._callbacks = null;
+    this._lastClosedTime = null;
+  }
+
+  /**
+   * @param {string} symbol
+   * @param {string} timeframe
+   * @param {{ onCandleUpdate?: (candle: object) => void, onTick?: (price: number) => void }} callbacks
+   * @param {{ pollMs?: number, maxPollMs?: number }} [options]
+   */
+  subscribe(symbol, timeframe, callbacks, options = {}) {
+    this.unsubscribe();
+    this._symbol = symbol;
+    this._timeframe = timeframe;
+    this._callbacks = callbacks;
+    this._stopped = false;
+    this._lastClosedTime = null;
+    this._baseMs = options.pollMs || 5000;
+    this._maxMs = options.maxPollMs || 30000;
+    this._currentMs = this._baseMs;
+    this._timer = setTimeout(() => this._loop(), 0);
+  }
+
+  unsubscribe() {
+    this._stopped = true;
+    if (this._timer) {
+      clearTimeout(this._timer);
+      this._timer = null;
+    }
+    this._symbol = null;
+    this._timeframe = null;
+    this._callbacks = null;
+  }
+
+  /** @private */
+  async _loop() {
+    if (this._stopped || !this._symbol) return;
+
+    let ok = false;
+    try {
+      const res = await fetch(`/api/market-data?symbol=${encodeURIComponent(this._symbol)}`);
+      if (res.ok) {
+        const data = await res.json();
+        const candles = data.candles || [];
+        if (candles.length > 0) {
+          ok = true;
+          const latest = candles[candles.length - 1];
+          const price = (data.price != null) ? data.price : latest.close;
+
+          if (this._callbacks?.onTick) {
+            this._callbacks.onTick(price);
+          }
+
+          if (this._callbacks?.onCandleUpdate) {
+            const duration = TIMEFRAME_SECONDS[this._timeframe] || 900;
+            const nowUnix = Math.floor(Date.now() / 1000);
+            // Emit closed=true only once per candle, not on every poll while the
+            // server is still serving the same bar as latest.
+            let isClosed = false;
+            if ((nowUnix - latest.time) >= duration && this._lastClosedTime !== latest.time) {
+              isClosed = true;
+              this._lastClosedTime = latest.time;
+            }
+            this._callbacks.onCandleUpdate({ ...latest, isClosed });
+          }
+        }
+      }
+    } catch (err) {
+      console.error(`[ServerStream] poll error for ${this._symbol}:`, err.message);
+    }
+
+    // Reset cadence on success, back off on failure up to the cap.
+    this._currentMs = ok ? this._baseMs : Math.min(this._maxMs, Math.round(this._currentMs * 1.8));
+    if (!this._stopped) {
+      this._timer = setTimeout(() => this._loop(), this._currentMs);
+    }
+  }
+}
+
+export default ServerStream;

src/data/yahooFinanceAPI.js

@@ -16,6 +16,8 @@ export const YAHOO_SYMBOL_MAP = {
 };
 
 let xauusdOffset = null;
+let xauusdOffsetTime = 0;
+const XAU_OFFSET_TTL_MS = 5 * 60 * 1000; // refresh the futures->spot offset every 5 min
 
 /**
  * Calculates or returns the cached offset between Swissquote Spot Gold and Yahoo Finance Futures (GC=F).
@@ -24,7 +26,11 @@ let xauusdOffset = null;
  * @returns {Promise<number>}
  */
 async function getXauusdOffset(latestYahooClose) {
-  if (xauusdOffset !== null) return xauusdOffset;
+  // Cache the offset but refresh periodically — futures vs spot basis drifts
+  // intraday, so a once-only offset goes stale over a session.
+  if (xauusdOffset !== null && (Date.now() - xauusdOffsetTime) < XAU_OFFSET_TTL_MS) {
+    return xauusdOffset;
+  }
   try {
     const res = await fetch('/api/swissquote');
     if (!res.ok) throw new Error(`HTTP ${res.status}`);
@@ -38,13 +44,15 @@ async function getXauusdOffset(latestYahooClose) {
       const spotPrice = (bid + ask) / 2;
       
       xauusdOffset = spotPrice - latestYahooClose;
+      xauusdOffsetTime = Date.now();
       console.log(`[yahooFinanceAPI] Calculated XAUUSD spot offset: ${xauusdOffset.toFixed(4)} (Spot: ${spotPrice.toFixed(2)}, Futures: ${latestYahooClose.toFixed(2)})`);
       return xauusdOffset;
     }
   } catch (err) {
     console.error('[yahooFinanceAPI] Failed to fetch Swissquote spot price for offset:', err);
   }
-  return 0; // Fallback to 0
+  // Keep the last known offset on failure rather than snapping back to 0.
+  return xauusdOffset ?? 0;
 }
 
 const TIMEFRAME_CONFIG = {
@@ -243,7 +251,7 @@ export async function fetchQuote(symbol) {
   let offset = 0;
   if (symbol === 'XAUUSD') {
     const rawClose = parseFloat(current);
-    offset = xauusdOffset !== null ? xauusdOffset : await getXauusdOffset(rawClose);
+    offset = await getXauusdOffset(rawClose);
   }
 
   return {

src/data/yahooWebSocket.js

@@ -36,73 +36,67 @@ export class YahooStream {
    * @param {string} timeframe - Local timeframe (1m, 5m, 15m, 1H, 4H, 1D)
    * @param {{ onCandleUpdate?: (candle: object) => void, onTick?: (price: number) => void }} callbacks
    */
-  subscribe(symbol, timeframe, callbacks) {
+  subscribe(symbol, timeframe, callbacks, options = {}) {
     this.unsubscribe();
 
     this._symbol = symbol;
     this._timeframe = timeframe;
     this._callbacks = callbacks;
-    this._isPolling = false;
+    this._stopped = false;
 
-    // Run first poll immediately, then poll every 6 seconds
-    this._poll();
-    this._intervalTimer = setInterval(() => {
-      this._poll();
-    }, 6000);
+    // Self-scheduling poll with exponential backoff: transient Yahoo errors / 429s
+    // slow the loop down instead of hammering. Optional stagger spreads multiple
+    // symbols so they don't all hit Yahoo on the same tick.
+    this._baseMs = options.pollMs || 6000;
+    this._maxMs = options.maxPollMs || 60000;
+    this._currentMs = this._baseMs;
+    this._timer = setTimeout(() => this._loop(), options.staggerMs || 0);
   }
 
   /** Disconnect the stream. */
   unsubscribe() {
-    if (this._intervalTimer) {
-      clearInterval(this._intervalTimer);
-      this._intervalTimer = null;
+    this._stopped = true;
+    if (this._timer) {
+      clearTimeout(this._timer);
+      this._timer = null;
     }
     this._symbol = null;
     this._timeframe = null;
     this._callbacks = null;
-    this._isPolling = false;
   }
 
   /** @private */
-  async _poll() {
-    // Prevent overlapping poll cycles
-    if (this._isPolling) return;
-    this._isPolling = true;
+  async _loop() {
+    if (this._stopped || !this._symbol || !this._timeframe) return;
 
+    let ok = false;
     try {
-      if (!this._symbol || !this._timeframe) {
-        this._isPolling = false;
-        return;
-      }
-
       const candles = await fetchCandles(this._symbol, this._timeframe);
 
       if (candles && candles.length > 0) {
+        ok = true;
         const latestCandle = candles[candles.length - 1];
 
-        // Emit tick for latest close price
         if (this._callbacks?.onTick) {
           this._callbacks.onTick(latestCandle.close);
         }
 
-        // Determine if this candle has expired/closed
         const duration = TIMEFRAME_SECONDS[this._timeframe] || 900;
         const nowUnix = Math.floor(Date.now() / 1000);
-        
-        // A candle is considered closed if the current time has moved past its period boundary
         const isClosed = (nowUnix - latestCandle.time) >= duration;
 
         if (this._callbacks?.onCandleUpdate) {
-          this._callbacks.onCandleUpdate({
-            ...latestCandle,
-            isClosed: isClosed,
-          });
+          this._callbacks.onCandleUpdate({ ...latestCandle, isClosed });
         }
       }
     } catch (err) {
-      console.error(`[YahooStream] Polling error for ${this._symbol}:`, err);
-    } finally {
-      this._isPolling = false;
+      console.error(`[YahooStream] Polling error for ${this._symbol}:`, err.message);
+    }
+
+    // Reset cadence on success, back off on failure up to the cap.
+    this._currentMs = ok ? this._baseMs : Math.min(this._maxMs, Math.round(this._currentMs * 1.8));
+    if (!this._stopped) {
+      this._timer = setTimeout(() => this._loop(), this._currentMs);
     }
   }
 }