Supervised launcher: auto-respawn worker/MCP if they die

app.py

@@ -110,8 +110,42 @@ def _spawn(module: str, port: int, name: str):
     )
 
 
-_worker = _spawn("Agent.worker:app", 8700, "backend worker")
-_mcp = _spawn("Agent.mcp_server:app", 8701, "MCP tool server")
+_children = {}
+
+
+def _supervise(specs):
+    """Respawn dead background services forever.
+
+    The Spaces container can OOM-kill or crash the worker (it is the
+    heaviest process: torch + llama-index + chat stores). Without
+    supervision that meant "all queries fail until a manual restart";
+    with it, the launcher brings a fresh worker back within seconds.
+    """
+    while True:
+        for key, module, port, name in specs:
+            if _listening(port):
+                continue  # healthy (or handled by an older instance)
+            proc = _children.get(key)
+            if proc is not None and proc.poll() is None:
+                continue  # spawned and still starting up (heavy import)
+            print(f"[LAUNCHER] {name} is DOWN - respawning on :{port}", flush=True)
+            try:
+                _children[key] = _spawn(module, port, name)
+            except Exception as spawn_err:
+                print(f"[LAUNCHER] respawn of {name} failed: {spawn_err}", flush=True)
+        time.sleep(10)
+
+
+_supervise_specs = (
+    ("worker", "Agent.worker:app", 8700, "backend worker"),
+    ("mcp", "Agent.mcp_server:app", 8701, "MCP tool server"),
+)
+for _key, _module, _port, _name in _supervise_specs:
+    _children[_key] = _spawn(_module, _port, _name)
+
+import threading  # noqa: E402
+
+threading.Thread(target=_supervise, args=(_supervise_specs,), daemon=True).start()
 
 import gradio as gr  # noqa: E402  (after 'spaces' — ZeroGPU rule 1)
 from web.app import app as fastapi_app  # noqa: E402