Upload sync_hf.py

scripts/sync_hf.py

@@ -21,6 +21,14 @@ Fixes for exit 78 (gateway startup gates):
 - Self-heal remaining 'requires migration' gates: park the file named in the
   startup error and restart the gateway (gates fire one at a time)
 - Surface doctor failures instead of ignoring the exit code
+
+Fixes for silent gateway startup hang (no healthz, no log output):
+- Run the exact 'openclaw agents add main ... --non-interactive' command that
+  doctor prints when it holds back an unverified agent database
+- Force OPENCLAW_LOG_LEVEL=debug + OPENCLAW_GATEWAY_STARTUP_TRACE on launch
+  so subsystem logs reach the console regardless of logging.consoleLevel
+- On health timeout: SIGUSR2 (--report-on-signal) to the gateway node tree,
+  then print /proc state and the Node diagnostic report javascriptStack
 """
 
 import os
@@ -1147,6 +1155,8 @@ class OpenClawFullSync:
         """Run a CLI command, streaming tagged output. Returns exit code or -1."""
         print(f"[SYNC] Running: {' '.join(cmd)} (timeout={timeout}s)")
         t0 = time.time()
+        self._last_cmd_output = ""
+        captured = []
         try:
             proc = subprocess.Popen(
                 cmd, cwd=str(APP_DIR), env=env,
@@ -1160,6 +1170,7 @@ class OpenClawFullSync:
 
         def pump():
             for line in proc.stdout:
+                captured.append(line)
                 print(f"[{tag} +{time.time() - t0:.0f}s] {line.rstrip()}", flush=True)
         th = threading.Thread(target=pump, daemon=True)
         th.start()
@@ -1171,8 +1182,10 @@ class OpenClawFullSync:
                 os.killpg(proc.pid, signal.SIGKILL)
             except Exception:
                 proc.kill()
+            self._last_cmd_output = "".join(captured)
             return -1
         th.join(timeout=5)
+        self._last_cmd_output = "".join(captured)
         print(f"[SYNC] {tag} exit code: {rc} ({time.time() - t0:.0f}s)")
         return rc
 
@@ -1188,8 +1201,32 @@ class OpenClawFullSync:
         if rc != 0:
             print(f"[SYNC] WARNING: doctor --fix failed (exit {rc}) — continuing; "
                   "session migration runs next and startup gates self-heal")
+        self._run_doctor_agent_restore_hint()
         return rc == 0
 
+    def _run_doctor_agent_restore_hint(self):
+        """Run the exact 'openclaw agents add ...' command doctor prints when it
+        holds back an unverified agent database (upstream's non-interactive fix)."""
+        text = getattr(self, "_last_cmd_output", "") or ""
+        entry = self._entry_cmd()
+        if not entry:
+            return False
+        phrases = re.findall(r"run (openclaw agents add main [^;\n]*?--non-interactive)", text)
+        if not phrases:
+            return False
+        ran = False
+        for phrase in list(dict.fromkeys(phrases))[:4]:
+            words = phrase.split()
+            if words and words[0] == "openclaw":
+                words = words[1:]
+            if not words or any(not re.fullmatch(r"[A-Za-z0-9_.:/~+-]+", w) for w in words):
+                continue
+            print("[SYNC] Doctor held back an agent database — running its suggested restore")
+            rc = self._run_logged_cmd(entry + words, self._doctor_env(), 180, tag="AGENT-ADD")
+            if rc == 0:
+                ran = True
+        return ran
+
     # ── Session store migration (exit 78 gate) ────────────────────────
 
     def _find_legacy_session_stores(self):
@@ -1387,6 +1424,103 @@ class OpenClawFullSync:
 
         return False
 
+    def _diagnose_hung_gateway(self, process):
+        """Capture why the gateway never became healthy: liveness, /proc state,
+        and Node diagnostic report javascriptStack (SIGUSR2 needs
+        --report-on-signal in NODE_OPTIONS). Best-effort, never raises."""
+        try:
+            if process.poll() is not None:
+                print(f"[HEALTH] Gateway process already exited: code={process.returncode}")
+                return
+            pid = process.pid
+            print(f"[HEALTH] Gateway process alive: pid={pid}")
+
+            def read_proc(p):
+                info = {}
+                try:
+                    with open(f"/proc/{p}/status") as f:
+                        for line in f:
+                            if line.startswith(("Name:", "State:", "Threads:",
+                                                "voluntary_ctxt_switches:",
+                                                "nonvoluntary_ctxt_switches:")):
+                                k, _, v = line.partition(":")
+                                info[k] = v.strip()
+                except Exception:
+                    pass
+                try:
+                    with open(f"/proc/{p}/wchan") as f:
+                        info["wchan"] = f.read().strip()
+                except Exception:
+                    pass
+                return info
+
+            def node_tree(p, depth=0, acc=None):
+                if acc is None:
+                    acc = []
+                if depth > 3:
+                    return acc
+                acc.append(p)
+                try:
+                    with open(f"/proc/{p}/task/{p}/children") as f:
+                        kids = [int(x) for x in f.read().split()]
+                except Exception:
+                    kids = []
+                for k in kids:
+                    node_tree(k, depth + 1, acc)
+                return acc
+
+            def is_node(p):
+                try:
+                    with open(f"/proc/{p}/comm") as f:
+                        return f.read().strip() == "node"
+                except Exception:
+                    return False
+
+            print(f"[HEALTH] proc: {read_proc(pid)}")
+            tree = [p for p in node_tree(pid) if is_node(p)]
+            print(f"[HEALTH] node process tree: {tree}")
+
+            if not hasattr(signal, "SIGUSR2"):
+                print("[HEALTH] SIGUSR2 unavailable on this platform")
+                return
+            for p in tree:
+                try:
+                    os.kill(p, signal.SIGUSR2)
+                    print(f"[HEALTH] Sent SIGUSR2 to node pid {p}")
+                except Exception as e:
+                    print(f"[HEALTH] SIGUSR2 pid {p}: {e}")
+            time.sleep(6)
+
+            found = False
+            now = time.time()
+            app = Path(APP_DIR)
+            for f in sorted(app.glob("*.json")):
+                try:
+                    if now - f.stat().st_mtime > 120:
+                        continue
+                    txt = f.read_text(encoding="utf-8", errors="replace")
+                    if '"javascriptStack"' not in txt:
+                        continue
+                    data = json.loads(txt)
+                except Exception:
+                    continue
+                found = True
+                print(f"[HEALTH] Node report: {f.name}")
+                print(f"[REPORT] header: {json.dumps(data.get('header'))[:400]}")
+                js = data.get("javascriptStack") or {}
+                for key, stack in js.items():
+                    if not isinstance(stack, list):
+                        continue
+                    limit = 80 if key == "main" else 12
+                    prefix = "[REPORT]" if key == "main" else f"[REPORT][{key}]"
+                    for ln in stack[:limit]:
+                        print(f"{prefix} {ln}")
+            if not found:
+                print("[HEALTH] No Node diagnostic report found in "
+                      f"{app} (child may not support SIGUSR2)")
+        except Exception as e:
+            print(f"[HEALTH] diagnose failed: {e}")
+
     # ── Application runner ────────────────────────────────────────────
 
     def run_openclaw(self):
@@ -1420,6 +1554,16 @@ class OpenClawFullSync:
         log_fh = open(log_file, "a")
         env = os.environ.copy()
 
+        # Startup diagnostics — force gateway visibility + Node crash reports.
+        # logging.consoleLevel in the dataset config may hide info-level lines;
+        # OPENCLAW_LOG_LEVEL=debug overrides it and the startup-trace step
+        # timings log at info (visible under debug). --report-on-signal lets the
+        # health-timeout path trigger a Node diagnostic report via SIGUSR2.
+        env.setdefault("OPENCLAW_LOG_LEVEL", "debug")
+        env.setdefault("OPENCLAW_GATEWAY_STARTUP_TRACE", "1")
+        if "--report-on-signal" not in env.get("NODE_OPTIONS", ""):
+            env["NODE_OPTIONS"] = (env.get("NODE_OPTIONS", "") + " --report-on-signal").strip()
+
         # Telegram probe
         if TELEGRAM_API_BASE:
             tg_root = TELEGRAM_API_BASE.rstrip("/")
@@ -1475,6 +1619,7 @@ class OpenClawFullSync:
             def health_worker():
                 healthy = self._wait_for_gateway_health(timeout=STARTUP_HEALTH_TIMEOUT)
                 if not healthy:
+                    self._diagnose_hung_gateway(process)
                     print("[HEALTH] ✗ Gateway unhealthy — dumping last 80 lines of startup.log")
                     try:
                         with open(log_file, "r") as f: