feat: refresh capacity every five minutes and highlight recent availability

Dockerfile

@@ -10,7 +10,7 @@ USER user
 
 ENV HOST=0.0.0.0 \
     PORT=7860 \
-    AUTO_REFRESH_MINUTES=15 \
+    AUTO_REFRESH_MINUTES=5 \
     STATE_FILE=/data/state.json
 
 EXPOSE 7860

README.md

@@ -28,7 +28,7 @@ AWS_PROFILE=<profile> uv run gpu-capacity
 ```
 
 One representative (smallest) size per GPU family, plus `g7e.48xlarge`, is swept
-automatically when `AUTO_REFRESH_MINUTES` is set (the deployed Space uses 15);
+automatically when `AUTO_REFRESH_MINUTES` is set (the deployed Space uses 5);
 any other size can be probed on demand from the dashboard input or the API.
 Results are cached in `state.json` and shown with their age; they fade once older than
 `STALE_AFTER_SECONDS` (default 900).
@@ -81,6 +81,7 @@ the Space).
 |---|---|
 | `available` (n/m AZ) | reservation succeeded in n of m offered AZs |
 | `no capacity` | AWS returned `InsufficientInstanceCapacity` everywhere |
+| blue / `Seen <date>` | no capacity now, but at least one AZ had capacity within the past 10 days; the date is the latest positive observation |
 | `quota` / `denied` | account quota or IAM/SCP blocks the check (not a capacity signal) |
 | `n/s` | type not reservable via ODCR there (p5 → Capacity Blocks) |
 | `—` | type not offered in that region |
@@ -88,6 +89,11 @@ the Space).
 Note: results are account-specific (quotas, SCPs), which is what you want —
 "can *this* account get a G5 there".
 
+Blue is a historical hint, not current availability. Current availability stays
+green and quota/access failures stay orange. Open a cell to see the last positive
+observation and select the 10-day history range. This uses recorded observations
+and does not trigger additional probes.
+
 Latency: an `available` answer comes back in ~1 s per region; a `no capacity`
 answer takes 10–20 s because EC2 genuinely attempts placement before returning
 `InsufficientInstanceCapacity`. Probe creates run with retries disabled —

gpu_capacity/config.py

@@ -45,6 +45,9 @@ PROBE_END_DATE_MINUTES = 30
 # Cached probe results older than this are flagged stale in the UI.
 STALE_AFTER_SECONDS = int(os.environ.get("STALE_AFTER_SECONDS", "900"))
 
+# Highlight unavailable cells where capacity was seen within this window.
+RECENT_AVAILABILITY_DAYS = 10
+
 # Optional background refresh of all types/regions. 0 = disabled (probe on
 # demand only). Each full sweep creates+cancels one reservation per offered AZ.
 AUTO_REFRESH_MINUTES = int(os.environ.get("AUTO_REFRESH_MINUTES", "0"))

gpu_capacity/history.py

@@ -14,6 +14,8 @@ RECORDED = {"available", "unavailable", "quota", "forbidden"}
 
 # (instance_type, region) -> [[ts, available_azs, total_azs, status], ...]
 _hist: dict[tuple[str, str], list] = {}
+# Latest positive observation, indexed for the frequently polled state API.
+_last_available: dict[tuple[str, str], int] = {}
 # today's raw jsonl lines, kept whole: the day file is always rewritten in
 # full by a dedicated writer thread. Appending from the sweep thread is what
 # froze the sweeps — O_APPEND is unreliable on the FUSE bucket mount, and a
@@ -43,6 +45,9 @@ def load():
                 _hist.setdefault((r["type"], r["region"]), []).append(
                     [r["t"], r["a"], r["n"], r["s"]]
                 )
+                if r["s"] == "available" and r["a"] > 0:
+                    key = (r["type"], r["region"])
+                    _last_available[key] = max(_last_available.get(key, 0), r["t"])
                 rows += 1
                 if fn[:-6] == _today_day[0]:
                     _today.append(line.rstrip("\n"))
@@ -68,6 +73,9 @@ def record(instance_type: str, results: dict[str, dict]):
             _hist.setdefault((instance_type, region), []).append(
                 [ts, avail, len(azs), result["status"]]
             )
+            if result["status"] == "available" and avail > 0:
+                key = (instance_type, region)
+                _last_available[key] = max(_last_available.get(key, 0), ts)
             _queue.put(json.dumps({
                 "t": ts, "type": instance_type, "region": region,
                 "a": avail, "n": len(azs), "s": result["status"],
@@ -99,6 +107,17 @@ def _writer_loop():
             log.warning("history write failed: %s", e)
 
 
+def recent_availability(days: int) -> dict[str, dict[str, int]]:
+    now = time.time()
+    cutoff = now - days * 86400
+    recent: dict[str, dict[str, int]] = {}
+    with _lock:
+        for (instance_type, region), ts in _last_available.items():
+            if cutoff <= ts <= now:
+                recent.setdefault(instance_type, {})[region] = ts
+    return recent
+
+
 def series(instance_type: str, region: str, hours: float) -> list[dict]:
     cutoff = time.time() - hours * 3600
     with _lock:

gpu_capacity/main.py

@@ -100,6 +100,8 @@ def get_state():
         "instance_types": types,
         "regions": config.REGIONS,
         "stale_after_seconds": config.STALE_AFTER_SECONDS,
+        "recent_availability_days": config.RECENT_AVAILABILITY_DAYS,
+        "recent_availability": history.recent_availability(config.RECENT_AVAILABILITY_DAYS),
         "gpu_info": prober.gpu_info(types),
         "state": current,
     }

gpu_capacity/static/index.html

@@ -24,6 +24,8 @@
     --ok-soft: #e9f4ed;
     --ko: #b3453c;
     --ko-soft: #f7ecea;
+    --recent: #245fb5;
+    --recent-soft: #eaf1fc;
     --warn: #966916;
     --warn-soft: #f7f1e5;
     --na: #7d8782;
@@ -50,6 +52,8 @@
       --ok-soft: #18291f;
       --ko: #e18b83;
       --ko-soft: #2c201f;
+      --recent: #8ab7fb;
+      --recent-soft: #1b293d;
       --warn: #d9b468;
       --warn-soft: #2d281d;
       --na: #9ba59f;
@@ -412,6 +416,10 @@
     background: var(--ko);
   }
 
+  .signal.recent {
+    background: var(--recent);
+  }
+
   .signal.quota,
   .signal.forbidden {
     background: var(--warn);
@@ -621,6 +629,10 @@
     background: var(--ko-soft);
   }
 
+  .matrix-cell.recent .cell-button {
+    background: var(--recent-soft);
+  }
+
   .matrix-cell.quota .cell-button,
   .matrix-cell.forbidden .cell-button {
     background: var(--warn-soft);
@@ -654,6 +666,10 @@
     color: var(--ko);
   }
 
+  .matrix-cell.recent .cell-value {
+    color: var(--recent);
+  }
+
   .matrix-cell.quota .cell-value,
   .matrix-cell.forbidden .cell-value {
     color: var(--warn);
@@ -858,6 +874,12 @@
     font-size: 12px;
   }
 
+  #p-last-available {
+    color: var(--recent);
+    font-size: 12px;
+    line-height: 1.5;
+  }
+
   #p-summary {
     display: flex;
     align-items: center;
@@ -1146,6 +1168,7 @@
         <span class="legend-item"><span class="signal available"></span>Available</span>
         <span class="legend-item"><span class="signal partial"></span>Partial AZ coverage</span>
         <span class="legend-item"><span class="signal unavailable"></span>No capacity</span>
+        <span class="legend-item"><span class="signal recent"></span><span id="recent-legend">Seen in past 10d; unavailable now</span></span>
         <span class="legend-item"><span class="signal quota"></span>Quota or denied</span>
         <span class="legend-item"><span class="signal unsupported"></span>N.O. / n/s / stale</span>
       </div>
@@ -1173,6 +1196,7 @@
         <div id="p-age"></div>
       </div>
       <div id="p-summary"></div>
+      <p id="p-last-available" hidden></p>
       <div id="p-azs"></div>
       <button class="button primary" id="p-probe" type="button">Probe region now</button>
     </section>
@@ -1184,6 +1208,7 @@
           <button type="button" data-hours="24">24h</button>
           <button type="button" data-hours="48" class="active">48h</button>
           <button type="button" data-hours="168">7d</button>
+          <button type="button" data-hours="240">10d</button>
         </div>
       </div>
       <div id="chartwrap"><div id="tooltip"></div></div>
@@ -1250,7 +1275,14 @@ function humanStatus(status) {
   }[status] || status;
 }
 
-function cellModel(result) {
+function recentAvailabilityAt(type, region) {
+  const ts = ((META.recent_availability || {})[type] || {})[region];
+  const now = Date.now() / 1000;
+  const cutoff = now - META.recent_availability_days * 86400;
+  return Number.isFinite(ts) && cutoff <= ts && ts <= now ? ts : null;
+}
+
+function cellModel(result, lastAvailableAt = null) {
   if (!result) {
     return {
       cls: "never",
@@ -1264,11 +1296,14 @@ function cellModel(result) {
   }
 
   const counts = azCounts(result);
-  const status = result.status === "available" && counts.available < counts.total ? "partial" : result.status;
+  const status = result.status === "unavailable" && lastAvailableAt ? "recent"
+    : result.status === "available" && counts.available < counts.total ? "partial" : result.status;
+  const lastSeen = lastAvailableAt ? new Date(lastAvailableAt * 1000) : null;
   const label = {
     available: `${counts.available}/${counts.total}`,
     partial: `${counts.available}/${counts.total}`,
     unavailable: counts.total ? `0/${counts.total}` : "none",
+    recent: `Seen ${lastSeen?.toLocaleDateString([], { day: "numeric", month: "short" })}`,
     quota: "quota",
     forbidden: "denied",
     unsupported: "n/s",
@@ -1284,8 +1319,8 @@ function cellModel(result) {
     status,
     label,
     age: fmtAge(result.checked_at),
-    title,
-    detail: humanStatus(status),
+    title: lastSeen ? `${humanStatus(result.status)}. Last available: ${lastSeen.toLocaleString()}\n${title}` : title,
+    detail: status === "recent" ? `No capacity now; last available ${lastSeen.toLocaleString()}` : humanStatus(status),
     count,
   };
 }
@@ -1407,6 +1442,7 @@ function renderMatrix(data) {
   const empty = document.getElementById("empty-filter");
   const copy = document.getElementById("matrix-copy");
   copy.textContent = `${visibleTypes.length} of ${data.instance_types.length} instance types across ${data.regions.length} regions`;
+  document.getElementById("recent-legend").textContent = `Seen in past ${data.recent_availability_days}d; unavailable now`;
 
   const colgroup = `
     <colgroup>
@@ -1453,7 +1489,7 @@ function renderMatrix(data) {
         </th>
         ${displayRegions.map(region => {
           const result = (data.state[type] || {})[region];
-          const model = cellModel(result);
+          const model = cellModel(result, recentAvailabilityAt(type, region));
           const probing = isProbing(type, region);
           const aria = `${type} in ${region}: ${model.detail}${model.count ? `, ${model.count}` : ""}, ${model.age}`;
           return `<td class="matrix-cell ${esc(model.cls)}${probing ? " probing" : ""}" data-type="${esc(type)}" data-region="${esc(region)}">
@@ -1531,6 +1567,12 @@ function renderPanelStatus() {
   const age = document.getElementById("p-age");
   const summary = document.getElementById("p-summary");
   const box = document.getElementById("p-azs");
+  const lastAvailableAt = recentAvailabilityAt(PANEL.type, PANEL.region);
+  const historyNote = document.getElementById("p-last-available");
+  historyNote.hidden = !lastAvailableAt;
+  historyNote.textContent = lastAvailableAt
+    ? `Capacity last observed ${new Date(lastAvailableAt * 1000).toLocaleString()} (within the past ${META.recent_availability_days} days).`
+    : "";
   if (!result) {
     age.textContent = "";
     summary.innerHTML = `<span class="panel-status"><span class="signal never"></span>Never probed</span>`;

tests/test_recent_availability.py

@@ -0,0 +1,93 @@
+import json
+import queue
+import tempfile
+import unittest
+from pathlib import Path
+from unittest.mock import patch
+
+from gpu_capacity import history, main
+
+
+class RecentAvailabilityTests(unittest.TestCase):
+    def setUp(self):
+        self.now = 1791417600  # 2026-10-08 00:00 UTC
+        self.cutoff = self.now - 10 * 86400
+        for name in ("_hist", "_last_available"):
+            mock = patch.dict(getattr(history, name), {}, clear=True)
+            mock.start()
+            self.addCleanup(mock.stop)
+        for name, value in (("_queue", queue.Queue()), ("_today", []), ("_today_day", [""])):
+            mock = patch.object(history, name, value)
+            mock.start()
+            self.addCleanup(mock.stop)
+        clock = patch.object(history.time, "time", return_value=self.now)
+        clock.start()
+        self.addCleanup(clock.stop)
+
+    def result(self, ts, status="available", available=True):
+        return {
+            "checked_at": ts,
+            "status": status,
+            "azs": {"us-east-1a": {"status": "available" if available else "unavailable"}},
+        }
+
+    def test_positive_observation_survives_failures_and_out_of_order_results(self):
+        for result in (
+            self.result(self.now - 100),
+            self.result(self.now - 50, "unavailable", False),
+            self.result(self.now - 200),
+            self.result(self.now - 25, "quota", False),
+            self.result(self.now, "available", False),
+        ):
+            history.record("g7e.2xlarge", {"us-east-1": result})
+        self.assertEqual(
+            history.recent_availability(10),
+            {"g7e.2xlarge": {"us-east-1": self.now - 100}},
+        )
+
+    def test_window_boundary_and_expiration_without_new_probes(self):
+        history._last_available.update({
+            ("g7e.2xlarge", "us-east-1"): self.cutoff,
+            ("g7e.2xlarge", "us-east-2"): self.cutoff - 1,
+            ("g7e.48xlarge", "us-east-1"): self.now + 1,
+        })
+        self.assertEqual(
+            history.recent_availability(10),
+            {"g7e.2xlarge": {"us-east-1": self.cutoff}},
+        )
+        with patch.object(history.time, "time", return_value=self.now + 0.5):
+            self.assertEqual(history.recent_availability(10), {})
+
+    def test_restore_history_finds_latest_success_not_latest_probe(self):
+        observations = [
+            (self.now - 86400, 1, "available"),
+            (self.now - 2 * 86400, 2, "available"),
+            (self.now - 100, 0, "unavailable"),
+            (self.now - 50, 0, "available"),
+            (self.now, 1, "quota"),
+        ]
+        with tempfile.TemporaryDirectory() as directory:
+            path = Path(directory) / "2026-10-07.jsonl"
+            path.write_text("\n".join(json.dumps({
+                "t": ts, "a": available, "n": 2, "s": status,
+                "type": "g7e.2xlarge", "region": "us-east-1",
+            }) for ts, available, status in observations))
+            with patch.object(history.config, "HISTORY_DIR", directory):
+                history.load()
+        self.assertEqual(
+            history.recent_availability(10),
+            {"g7e.2xlarge": {"us-east-1": self.now - 86400}},
+        )
+
+    def test_state_api_exposes_history_without_changing_current_status(self):
+        history.record("g7e.2xlarge", {"us-east-1": self.result(self.now - 86400)})
+        current = {"g7e.2xlarge": {"us-east-1": self.result(self.now, "unavailable", False)}}
+        with patch.object(main.state, "get", return_value=current), patch.object(main.prober, "gpu_info", return_value={}):
+            response = main.get_state()
+        self.assertEqual(response["recent_availability_days"], 10)
+        self.assertEqual(response["recent_availability"]["g7e.2xlarge"]["us-east-1"], self.now - 86400)
+        self.assertEqual(response["state"]["g7e.2xlarge"]["us-east-1"]["status"], "unavailable")
+
+
+if __name__ == "__main__":
+    unittest.main()

tests/test_ui.cjs

@@ -0,0 +1,77 @@
+const assert = require("node:assert/strict");
+const fs = require("node:fs");
+const path = require("node:path");
+const test = require("node:test");
+const vm = require("node:vm");
+
+const html = fs.readFileSync(path.join(__dirname, "../gpu_capacity/static/index.html"), "utf8");
+const script = html.match(/<script>([\s\S]*?)<\/script>/)[1]
+  .split('document.getElementById("grid").addEventListener')[0];
+const now = Date.UTC(2026, 9, 8) / 1000;
+
+function setup(status = "unavailable", lastSeen = now - 86400) {
+  const elements = {};
+  const context = vm.createContext({
+    Date: class extends Date { static now() { return now * 1000; } },
+    document: { getElementById(id) {
+      return elements[id] ||= { classList: { toggle() {} }, style: {}, textContent: "", innerHTML: "" };
+    } },
+  });
+  vm.runInContext(script, context);
+  const data = {
+    instance_types: ["g7e.2xlarge"], regions: ["us-east-1"], gpu_info: {},
+    stale_after_seconds: 900, recent_availability_days: 10,
+    recent_availability: { "g7e.2xlarge": { "us-east-1": lastSeen } },
+    state: { "g7e.2xlarge": { "us-east-1": {
+      status, checked_at: now - 30,
+      azs: { "us-east-1a": { status }, "us-east-1b": { status } },
+    } } },
+  };
+  vm.runInContext(`META = ${JSON.stringify(data)}; PANEL.type = "g7e.2xlarge"; PANEL.region = "us-east-1";`, context);
+  return { context, elements, run: source => vm.runInContext(source, context) };
+}
+
+function model(ui) {
+  return ui.run('cellModel(META.state["g7e.2xlarge"]["us-east-1"], recentAvailabilityAt("g7e.2xlarge", "us-east-1"))');
+}
+
+test("unavailable cells with recent capacity turn blue and disclose the date", () => {
+  const ui = setup();
+  const cell = model(ui);
+  assert.equal(cell.status, "recent");
+  assert.match(cell.label, /^Seen /);
+  assert.match(cell.detail, /No capacity now; last available/);
+  ui.run("renderMatrix(META); renderPanelStatus();");
+  assert.match(ui.elements.grid.innerHTML, /matrix-cell recent/);
+  assert.match(ui.elements["p-summary"].innerHTML, /No capacity/);
+  assert.match(ui.elements["p-last-available"].textContent, /Capacity last observed/);
+  assert.equal(ui.elements["p-last-available"].hidden, false);
+  assert.match(ui.elements["recent-legend"].textContent, /10d/);
+});
+
+test("fresh capacity and quota failures retain their current colors", () => {
+  for (const status of ["available", "quota", "forbidden", "error", "unsupported", "not_offered"]) {
+    assert.equal(model(setup(status)).status, status);
+  }
+  const ui = setup("available");
+  ui.run('META.state["g7e.2xlarge"]["us-east-1"].azs["us-east-1b"].status = "unavailable"');
+  assert.equal(model(ui).status, "partial");
+});
+
+test("missing, future, or expired observations stay red, including after the time window expires", () => {
+  for (const timestamp of [null, now + 1, now - 10 * 86400 - 1]) {
+    assert.equal(model(setup("unavailable", timestamp)).status, "unavailable");
+  }
+  const ui = setup("unavailable", now - 10 * 86400);
+  assert.equal(model(ui).status, "recent");
+  ui.run(`Date.now = () => ${(now + 1) * 1000}`);
+  assert.equal(model(ui).status, "unavailable");
+});
+
+test("historical capacity is not counted or filtered as current availability", () => {
+  const ui = setup();
+  ui.run("VIEW.availableOnly = true;");
+  assert.equal(ui.run("filteredTypes(META).length"), 0);
+  assert.equal(ui.run('coverageSummary(META, "g7e.2xlarge").available'), 0);
+  assert.match(html, /data-hours="240">10d/);
+});