deploy demo space from af5c03ea1d8d

app.py

@@ -276,6 +276,7 @@ _CSS = """
 .cell-dot--green { background: #22c55e; box-shadow: 0 0 7px 1px #22c55e; animation: cell-blink 1.4s ease-in-out infinite; }
 .cell-dot--yellow { background: #eab308; box-shadow: 0 0 7px 1px #eab308; animation: cell-blink 1.0s ease-in-out infinite; }
 .cell-dot--red { background: #ef4444; box-shadow: 0 0 7px 1px #ef4444; animation: cell-blink 0.8s ease-in-out infinite; }
+.cell-dot--grey { background: #9ca3af; }
 @keyframes cell-blink { 0%, 100% { opacity: 1; } 50% { opacity: 0.25; } }
 """
 
@@ -288,6 +289,10 @@ def _cell_light_html(snap: dict, task: config.Task) -> str:
             snap.get("cell_statuses") or {},
             aggregate=snap.get("cell_status", {}).get(task.slug, "offline"),
         )
+    return _light_html(color, caption)
+
+
+def _light_html(color: str, caption: str) -> str:
     return (
         '<div class="cell-status">'
         f'<span class="cell-dot cell-dot--{color}"></span>'
@@ -420,9 +425,9 @@ with gr.Blocks(title="armnet eval — evaluate a policy on real robots") as demo
                     "lerobot-eval runtime image ref) in the Space settings — see the README."
                 )
 
-            cell_light = gr.HTML(
-                _cell_light_html({"cell_status": {}}, _DEFAULT_TASK)
-            )
+            # Not "cells offline": a page that never gets its first refresh
+            # would then look exactly like a fleet that is down.
+            cell_light = gr.HTML(_light_html("grey", "checking cells…"))
 
             with gr.Row():
                 preset = gr.Dropdown(
@@ -518,8 +523,14 @@ with gr.Blocks(title="armnet eval — evaluate a policy on real robots") as demo
         inputs=[task_sel, preset, custom],
         outputs=[banner, status, run_btn, feed_sel, feed_caption, cell_light],
     )
+    # concurrency_limit=None on both: they share one Gradio concurrency group
+    # (keyed by the function), and its default limit of 1 would put every
+    # viewer's refresh behind a single worker. It only reads a locked snapshot.
     text_timer.tick(
-        _refresh_text, inputs=[task_sel, feed_sel], outputs=text_outputs
+        _refresh_text,
+        inputs=[task_sel, feed_sel],
+        outputs=text_outputs,
+        concurrency_limit=None,
     )
     feed_sel.change(
         _on_feed_change,
@@ -527,7 +538,12 @@ with gr.Blocks(title="armnet eval — evaluate a policy on real robots") as demo
         outputs=[status, rollouts, summary, events, feed_caption],
     )
 
-    demo.load(_refresh_text, inputs=[task_sel, feed_sel], outputs=text_outputs)
+    demo.load(
+        _refresh_text,
+        inputs=[task_sel, feed_sel],
+        outputs=text_outputs,
+        concurrency_limit=None,
+    )
     demo.load(
         _stream_video,
         outputs=video,

runner.py

@@ -24,7 +24,14 @@ from typing import Any, Optional
 import certifi
 import websocket
 
-from armnet_core import API_KEY_HEADER, Job, JobSpec, JobStatus, TerminalStatus
+from armnet_core import (
+    API_KEY_HEADER,
+    CellStatusSummary,
+    Job,
+    JobSpec,
+    JobStatus,
+    TerminalStatus,
+)
 from armnet_client._config import api_key, orchestrator_url
 from armnet_client.client import OrchestratorClient, OrchestratorError
 from armnet_client.execute import _ws_url
@@ -109,6 +116,33 @@ def fleet_availability(
     return "red", "cells offline"
 
 
+def _read_cell_status(
+    client: Any, *, embodiment: str, task: str
+) -> CellStatusSummary:
+    """Fetch cell status without each cell's ``device_config``.
+
+    The Space never reads the device config, and it is the one strict part of
+    the response: the Space installs armnet-core from PyPI, which lags the
+    deployed orchestrator, and a config field added since then fails the whole
+    response. Every poll then fails and the lights stay offline until the
+    Space is rebuilt against a newer armnet-core.
+
+    Test doubles without an HTTP session keep using ``get_cell_status``.
+    """
+    http = getattr(client, "_client", None)
+    if http is None:
+        return client.get_cell_status(embodiment=embodiment, task=task)
+    response = http.get(
+        "/cells/status", params={"embodiment": embodiment, "task": task}
+    )
+    response.raise_for_status()
+    payload = response.json()
+    for cell in payload.get("cells") or []:
+        if isinstance(cell, dict):
+            cell.pop("device_config", None)
+    return CellStatusSummary.model_validate(payload)
+
+
 @dataclass
 class RunSlot:
     """Independent live state for one BusyBox cell's current/recent eval."""
@@ -544,28 +578,64 @@ class DemoState:
         previous: dict[str, str] = {}
         while True:
             for task in config.TASKS.values():
-                status, reason, client, cells = self._poll_one(client, task)
+                # This is the only thread that updates the lights and the
+                # queue. If it dies they freeze for every viewer until the
+                # Space restarts.
+                try:
+                    status, reason, client, cells = self._poll_one(client, task)
+                    shown = self._apply_poll(task.slug, status, cells)
+                except Exception:  # noqa: BLE001
+                    logger.exception("%s status poll crashed", task.label)
+                    continue
                 # Logged on change rather than every poll: at a few seconds per
                 # poll this would otherwise be the loudest thing in the Space
                 # log, but without it a red light gives an operator nothing to
                 # act on.
-                shown = self._apply_poll(task.slug, status, cells)
                 if shown != previous.get(task.slug) and reason and shown == "offline":
                     logger.warning("%s shown as offline: %s", task.label, reason)
                 previous[task.slug] = shown
-            try:
-                self._refresh_queue(client)
-                self._adopt_dispatched_eval(client)
-            except Exception as exc:  # noqa: BLE001 - keep last good queue view
-                logger.warning("queue status poll failed: %r", exc)
+            client = self._poll_queue(client)
             time.sleep(config.CELL_STATUS_POLL_S)
 
+    def _poll_queue(
+        self, client: Optional[OrchestratorClient]
+    ) -> Optional[OrchestratorClient]:
+        """Refresh the queue view and adopt started evals; the client to reuse.
+
+        Runs even when the status polls dropped their client. Otherwise the
+        queued count only grows (submissions add to it locally), a policy that
+        already started stays "already queued", and a full count disables Run.
+        """
+        try:
+            if client is None:
+                client = OrchestratorClient(
+                    orchestrator_url(), timeout=_STATUS_CALL_TIMEOUT_S
+                )
+            _call_bounded(lambda: self._refresh_queue(client), _STATUS_CALL_TIMEOUT_S)
+            _call_bounded(
+                lambda: self._adopt_dispatched_eval(client), _STATUS_CALL_TIMEOUT_S
+            )
+            return client
+        except Exception as exc:  # noqa: BLE001 - keep last good queue view
+            logger.warning("queue status poll failed: %r", exc)
+            if client is not None:
+                try:
+                    client.close()
+                except Exception:  # noqa: BLE001
+                    pass
+            return None
+
     def _apply_poll(
         self, task_slug: str, status: str, cells: Optional[list[Any]]
     ) -> str:
-        """Record one poll. A failed poll keeps the last good fleet for a while."""
+        """Record one poll. A failed or empty poll keeps the last good fleet a while.
+
+        An empty fleet is debounced too: an orchestrator instance that has just
+        started (a deploy, or Cloud Run scaling out) answers with no cells until
+        the first heartbeats reach it a few seconds later.
+        """
         with self._lock:
-            if cells is None:
+            if not cells:
                 fails = self._status_failures.get(task_slug, 0) + 1
                 self._status_failures[task_slug] = fails
                 if fails < _STATUS_FAILURES_BEFORE_OFFLINE:
@@ -588,8 +658,8 @@ class DemoState:
                     orchestrator_url(), timeout=_STATUS_CALL_TIMEOUT_S
                 )
             summary = _call_bounded(
-                lambda: client.get_cell_status(
-                    embodiment=config.EMBODIMENT, task=task.slug
+                lambda: _read_cell_status(
+                    client, embodiment=config.EMBODIMENT, task=task.slug
                 ),
                 _STATUS_CALL_TIMEOUT_S,
             )