import { spawn, type ChildProcess } from "node:child_process"; import { once } from "node:events"; import fs from "node:fs"; import path from "node:path"; import { setTimeout as delay } from "node:timers/promises"; import { afterEach, describe, expect, it, vi } from "vitest"; import { withTestTimeout } from "../../../test/helpers/promise.js"; import { createTempDirTracker } from "../../../test/helpers/temp-dir.js"; import { resolveRuntimeWorkerUrl } from "../../infra/runtime-worker-url.js"; import { gatewayDirectStopEntrypoints } from "../cli-entrypoint.test-support.js"; const tempDirs = createTempDirTracker(); const children = new Map>(); const runLoopUrl = resolveRuntimeWorkerUrl(gatewayDirectStopEntrypoints.runLoop).href; const restartUrl = resolveRuntimeWorkerUrl(gatewayDirectStopEntrypoints.restartPolicy).href; const fileLogTransportUrl = resolveRuntimeWorkerUrl( gatewayDirectStopEntrypoints.fileLogTransport, ).href; const childScript = ` import fs from "node:fs"; import http from "node:http"; import { runGatewayLoop } from ${JSON.stringify(runLoopUrl)}; import { setGatewaySigusr1RestartPolicy } from ${JSON.stringify(restartUrl)}; import { fileLogTransport } from ${JSON.stringify(fileLogTransportUrl)}; const faultPath = process.argv[1]; const closeFailure = process.argv[2]; setGatewaySigusr1RestartPolicy({ allowExternal: true }); let starts = 0; try { await runGatewayLoop({ ownsProcessLifecycle: true, start: async () => { const attempt = ++starts; process.stdout.write("start:" + attempt + "\\n"); if (fs.existsSync(faultPath)) throw new Error("fixture startup refused"); const server = http.createServer((_request, response) => response.end("ready")); await new Promise((resolve, reject) => { server.once("error", reject); server.listen(0, "127.0.0.1", resolve); }); process.stdout.write("ready:" + attempt + "\\n"); return { getTailscaleIngressEndpoint: () => undefined, startupSettled: Promise.resolve(), close: () => { if (closeFailure === "pending") { fileLogTransport.setAppenderForTests(() => { process.stdout.write("append:pending\\n"); return new Promise(() => {}); }); process.stdout.write("close:pending\\n"); return new Promise(() => {}); } if (closeFailure) { const error = new TypeError("fixture close owner failed"); error.stack = "TypeError: fixture close owner failed\\n at closeOwner (fixture.js:12:3)"; if (closeFailure === "sync") throw error; return Promise.reject(error); } return new Promise((resolve, reject) => server.close(error => error ? reject(error) : resolve())) .then(() => process.stdout.write("closed:" + attempt + "\\n")); }, }; }, runtime: { log: () => {}, error: (...args) => console.error(...args), exit: code => process.exit(code), }, }); } catch (error) { console.error(error); process.exitCode = 1; } `; afterEach(async () => { for (const child of children.keys()) { if (child.exitCode === null && child.signalCode === null) { child.kill("SIGKILL"); } } const results = await withTestTimeout( Promise.allSettled(children.values()), 5_000, "restart liveness children did not close; retaining their fixture directories", ); const errors = results.flatMap((result) => (result.status === "rejected" ? [result.reason] : [])); if (errors.length) { throw new AggregateError(errors, "restart liveness cleanup failed"); } children.clear(); tempDirs.cleanup(); }); function startFixture(initialFailure = false, closeFailure = "") { const directory = tempDirs.make("openclaw-restart-liveness-"); const home = path.join(directory, "home"); fs.mkdirSync(home); const faultPath = path.join(directory, "startup-fault"); const logFile = path.join(directory, "gateway.jsonl"); const stateDir = path.join(directory, "state"); fs.writeFileSync( path.join(directory, "openclaw.json"), JSON.stringify({ logging: { level: "info", file: logFile } }), ); if (initialFailure) { fs.writeFileSync(faultPath, "refuse"); } const child = spawn( process.execPath, ["--import", "tsx", "--input-type=module", "--eval", childScript, faultPath, closeFailure], { env: { PATH: process.env.PATH, HOME: home, TMPDIR: directory, OPENCLAW_STATE_DIR: stateDir, OPENCLAW_CONFIG_PATH: path.join(directory, "openclaw.json"), OPENCLAW_NO_RESPAWN: "1", ...(closeFailure === "pending" ? { OPENCLAW_LAUNCHD_LABEL: "ai.openclaw.gateway.test" } : {}), NODE_DISABLE_COMPILE_CACHE: "1", TSX_DISABLE_CACHE: "1", ESBUILD_WORKER_THREADS: "0", }, stdio: ["ignore", "pipe", "pipe"], }, ); const closed = once(child, "close"); children.set(child, closed); void closed.catch(() => {}); let output = ""; child.stdout?.on("data", (chunk: Buffer) => (output += chunk.toString())); child.stderr?.on("data", (chunk: Buffer) => (output += chunk.toString())); const waitForOutput = (text: string) => vi.waitFor(() => expect(output).toContain(text), { timeout: 45_000, interval: 25 }); return { child, closed, faultPath, logFile, stateDir, waitForOutput, output: () => output }; } async function expectFailedRestartWaiting( fixture: ReturnType, attempt: number, ) { expect(fixture.child.kill("SIGUSR1")).toBe(true); await fixture.waitForOutput(`start:${attempt}`); await vi.waitFor( () => expect( fixture.output().split("Process will stay alive; fix the issue and restart.").length - 1, ).toBe(attempt - 1), { timeout: 5_000, interval: 25 }, ); // Only the parent observes an idle interval. A child timer or IPC channel // would hide the lost-process regression after its real listener closes. expect( await Promise.race([ fixture.closed.then((exit) => ({ exit })), delay(750).then(() => "waiting"), ]), fixture.output(), ).toBe("waiting"); } describe("runGatewayLoop failed-restart process lifetime", () => { const posixIt = process.platform === "win32" ? it.skip : it; it.skipIf(process.platform !== "darwin")( "exits before the hard watchdog when a shutdown-deadline log append stalls", async () => { const fixture = startFixture(false, "pending"); await fixture.waitForOutput("ready:1"); expect(fixture.child.kill("SIGTERM")).toBe(true); await fixture.waitForOutput("close:pending"); expect(await fixture.closed, fixture.output()).toEqual([0, null]); expect(fixture.output()).toContain("append:pending"); const bundleDir = path.join(fixture.stateDir, "logs", "stability"); const files = fs.readdirSync(bundleDir); expect(files).toHaveLength(1); expect(JSON.parse(fs.readFileSync(path.join(bundleDir, files[0]!), "utf8"))).toMatchObject({ reason: "gateway.stop_shutdown_timeout", }); }, 60_000, ); posixIt.each(["sync", "rejected"])( "persists %s close failures before a SIGUSR1 force-exit", async (mode) => { const fixture = startFixture(false, mode); await fixture.waitForOutput("ready:1"); expect(fixture.child.kill("SIGUSR1")).toBe(true); expect(await fixture.closed, fixture.output()).toEqual([1, null]); const bundleDir = path.join(fixture.stateDir, "logs", "stability"); const files = fs.readdirSync(bundleDir); expect(files).toHaveLength(1); const bundle = JSON.parse(fs.readFileSync(path.join(bundleDir, files[0]!), "utf8")); expect(bundle).toMatchObject({ reason: "gateway.restart_close_failed", error: { name: "TypeError", message: "fixture close owner failed" }, evidence: { shutdown: { step: "gateway-server-close", errors: [ { name: "TypeError", message: "fixture close owner failed", stack: "TypeError: fixture close owner failed\n at closeOwner (fixture.js:12:3)", }, ], }, }, }); expect(fs.readFileSync(fixture.logFile, "utf8")).toContain( "shutdown step failed (gateway server close): fixture close owner failed", ); expect(fixture.output()).not.toContain("start:2"); }, 60_000, ); posixIt( "recovers in the same process after repeated operator-triggered startup failures", async () => { const fixture = startFixture(); await fixture.waitForOutput("ready:1"); fs.writeFileSync(fixture.faultPath, "refuse"); await expectFailedRestartWaiting(fixture, 2); await expectFailedRestartWaiting(fixture, 3); fs.unlinkSync(fixture.faultPath); expect(fixture.child.kill("SIGUSR1")).toBe(true); await fixture.waitForOutput("ready:4"); expect(fixture.child.kill("SIGTERM")).toBe(true); expect(await fixture.closed, fixture.output()).toEqual([0, null]); }, 60_000, ); posixIt.each(["SIGTERM", "SIGINT"] as const)( "exits cleanly on %s while waiting after a failed restart", async (signal) => { const fixture = startFixture(); await fixture.waitForOutput("ready:1"); fs.writeFileSync(fixture.faultPath, "refuse"); await expectFailedRestartWaiting(fixture, 2); expect(fixture.child.kill(signal)).toBe(true); expect(await fixture.closed, fixture.output()).toEqual([0, null]); expect(fixture.output()).not.toContain("start:3"); }, 60_000, ); posixIt( "releases process ownership when initial startup rejects", async () => { const fixture = startFixture(true); expect(await fixture.closed, fixture.output()).toEqual([1, null]); expect(fixture.output()).toContain("fixture startup refused"); expect(fixture.output()).not.toContain("Process will stay alive"); }, 60_000, ); });