fix: bound the waits that could freeze the learn loop (voyager_runner.py)

voyager_runner.py

@@ -5,6 +5,7 @@ import json
 import logging
 import shutil
 import sys
+import threading
 import time
 from pathlib import Path
 
@@ -314,10 +315,56 @@ def main() -> None:
     delay = float(env("VOYAGER_LEARN_BACKOFF_SECONDS", "15"))
     delay_cap = float(env("VOYAGER_LEARN_BACKOFF_CAP_SECONDS", "300"))
     attempt = 0
+
+    # A watchdog for the failure mode that retry-forever hides: the process is
+    # alive, every error counter is zero, and nothing progresses because
+    # learn() is parked on a wedged dependency. Retrying cannot help there --
+    # the loop is blocked, not failing -- so the only recovery is to exit and
+    # let the supervisor start a fresh runner. Measured: the bot sat in that
+    # state for 45+ minutes with zero model calls. Progress is read from the
+    # checkpoint files (not from a log line), so it is immune to the log being
+    # absent, and the watchdog only arms once a task has actually completed,
+    # so a cold start is never killed.
+    stall_seconds = float(env("VOYAGER_STALL_SECONDS", "900"))
+    completed_path = ckpt / "curriculum" / "completed_tasks.json"
+
+    def _progress_fingerprint():
+        try:
+            stat = completed_path.stat()
+            return (stat.st_mtime_ns, stat.st_size)
+        except OSError:
+            return None
+
+    def _watch_stall():
+        last_seen = _progress_fingerprint()
+        last_change = time.time()
+        while True:
+            time.sleep(60)
+            current = _progress_fingerprint()
+            if current != last_seen:
+                last_seen = current
+                last_change = time.time()
+                continue
+            idle = time.time() - last_change
+            if idle >= stall_seconds:
+                # os._exit is deliberate: the learning thread is blocked inside
+                # a call that will not return, so a cooperative shutdown would
+                # itself hang. Exiting hard lets the Space watchdog observe it.
+                print(
+                    f"[voyager] STALL WATCHDOG: no completed task in {idle:.0f}s "
+                    f"(limit {stall_seconds:.0f}s); exiting so the supervisor can "
+                    f"restart the runner",
+                    flush=True,
+                )
+                sys.stdout.flush()
+                sys.stderr.flush()
+                os._exit(75)
+
     try:
         quarantine_invalid_skills(voyager, ckpt)
         quarantine_template_skills(voyager, ckpt)
         quarantine_placeholder_skills(voyager, ckpt)
+        threading.Thread(target=_watch_stall, daemon=True, name="voyager-stall-watch").start()
         while True:
             attempt += 1
             try: