/** LaunchAgent bootstrap recovery plus start and restart lifecycle controls. */ import { spawnSync } from "node:child_process"; import { formatPortDiagnostics } from "../infra/ports-format.js"; import { inspectPortUsage } from "../infra/ports-inspect.js"; import { cleanStaleGatewayProcessesSync } from "../infra/restart-stale-pids.js"; import { isCurrentProcessInsideLaunchdService } from "./launchd-current-service.js"; import { execLaunchctl, formatLaunchctlResultDetail, isLaunchctlNotLoaded, } from "./launchd-exec.js"; import { resolveLaunchAgentLabel } from "./launchd-label.js"; import { scheduleDetachedLaunchdRestartHandoff } from "./launchd-restart-handoff.js"; import { bootstrapLaunchAgentOrThrow, isLaunchctlAlreadyLoaded, isUnsupportedGuiDomain, parseLaunchctlPrint, readLaunchAgentRuntime, resolveLaunchAgentGatewayContext, resolveLaunchAgentGuiDomain, } from "./launchd-runtime.js"; import { resolveLaunchAgentPlistPath, rewriteLaunchAgentPlistForRestart, } from "./launchd-service-files.js"; import { assertNoSystemLaunchDaemonOwnership, isSystemLaunchDaemonOwnershipError, } from "./launchd-system.js"; import { formatLine } from "./output.js"; import { createGatewayLifecycleMutationReporter } from "./service-mutation.js"; import { resolveServiceManagerEnv } from "./service-process-env.js"; import type { GatewayServiceControlArgs, GatewayServiceEnv, GatewayServiceRestartResult, } from "./service-types.js"; import { assertGatewayServiceUpdateCurrent, isUpdateOwnedGatewayServiceCommand, } from "./service-update-authority.js"; const LAUNCHCTL_PROTECTED_PID_TIMEOUT_MS = 2_000; function readLaunchAgentPidForCleanupSync(serviceTarget: string): number { const probe = spawnSync("launchctl", ["print", serviceTarget], { env: resolveServiceManagerEnv(), encoding: "utf8", timeout: LAUNCHCTL_PROTECTED_PID_TIMEOUT_MS, }); const result = { stdout: probe.stdout ?? "", stderr: probe.error?.message ?? probe.stderr ?? "", code: probe.error ? 1 : (probe.status ?? 1), }; if (result.code !== 0) { throw new Error(`launchctl print failed: ${formatLaunchctlResultDetail(result)}`); } const pid = parseLaunchctlPrint(result.stdout || result.stderr || "").pid; if (pid === undefined) { throw new Error("launchctl print did not report a running pid"); } return pid; } type LaunchAgentBootstrapRepairResult = | { ok: true; status: "repaired" | "already-loaded" } | { ok: false; status: "bootstrap-failed" | "kickstart-failed"; detail?: string; } | { ok: false; status: "system-launchdaemon-conflict" | "system-launchdaemon-unverifiable"; detail: string; } | { ok: false; status: "gui-session-unavailable"; detail: string; domain: string }; export async function repairLaunchAgentBootstrap(args: { env?: Record; warn?: (message: string) => void; }): Promise { const env = args.env ?? (process.env as Record); const domain = resolveLaunchAgentGuiDomain(); const label = resolveLaunchAgentLabel(env); const plistPath = resolveLaunchAgentPlistPath(env); const serviceTarget = `${domain}/${label}`; try { await assertNoSystemLaunchDaemonOwnership(label); } catch (error) { if (!isSystemLaunchDaemonOwnershipError(error)) { throw error; } return { ok: false, status: error.ownership.status === "unverifiable" ? "system-launchdaemon-unverifiable" : "system-launchdaemon-conflict", detail: error.message, }; } // Rewrite first so legacy inline environment secrets move into the private // env file before the plist becomes world-readable for launchd. const warn = args.warn ?? ((message: string) => console.warn(formatLine("Warning", message))); await rewriteLaunchAgentPlistForRestart({ env, label, plistPath, warn }); await execLaunchctl(["enable", serviceTarget]); const boot = await execLaunchctl(["bootstrap", domain, plistPath]); let repairStatus: "repaired" | "already-loaded" = "repaired"; if (boot.code !== 0) { const detail = (boot.stderr || boot.stdout).trim(); if (isUnsupportedGuiDomain(detail)) { return { ok: false, status: "gui-session-unavailable", detail, domain, }; } if (!isLaunchctlAlreadyLoaded(boot)) { return { ok: false, status: "bootstrap-failed", detail: detail || undefined }; } repairStatus = "already-loaded"; } if (repairStatus === "repaired") { return { ok: true, status: repairStatus }; } // Service is already bootstrapped. Only kickstart if it is not actively running — // kickstarting a healthy running service causes unnecessary session disconnects. const runtime = await readLaunchAgentRuntime(env); if (runtime.status === "running") { return { ok: true, status: repairStatus }; } const kick = await execLaunchctl(["kickstart", serviceTarget]); if (kick.code !== 0) { return { ok: false, status: "kickstart-failed", detail: (kick.stderr || kick.stdout).trim() || undefined, }; } return { ok: true, status: repairStatus }; } type LaunchAgentRestoreResult = { loaded: true } | { loaded: false; detail: string }; function writeLaunchAgentActionLine( stdout: NodeJS.WritableStream, label: string, value: string, ): void { try { stdout.write(`${formatLine(label, value)}\n`); } catch (err: unknown) { if ((err as NodeJS.ErrnoException)?.code !== "EPIPE") { throw err; } } } async function ensureLaunchAgentLoadedAfterFailure(params: { domain: string; serviceTarget: string; plistPath: string; onMutation?: (mode: "enable" | "bootstrap") => void; }): Promise { const probe = await execLaunchctl(["print", params.serviceTarget]); if (probe.code === 0) { return { loaded: true }; } try { await bootstrapLaunchAgentOrThrow({ domain: params.domain, serviceTarget: params.serviceTarget, plistPath: params.plistPath, actionHint: "openclaw gateway start", onMutation: params.onMutation, }); return { loaded: true }; } catch (error) { // A failed restore is not recoverable by launchd: the label is gone, so // KeepAlive has nothing to respawn. Report it instead of dropping it. return { loaded: false, detail: error instanceof Error ? error.message : String(error) }; } } function formatLaunchAgentLeftUnloadedError(params: { domain: string; serviceTarget: string; plistPath: string; failure: string; restoreDetail: string; }): string { return [ params.failure, `LaunchAgent ${params.serviceTarget} is not loaded and could not be restored: ${params.restoreDetail}`, "The gateway is down and launchd has no job left to respawn it.", `Fix: run \`openclaw gateway start\`, or \`launchctl bootstrap ${params.domain} ${params.plistPath}\`.`, ].join("\n"); } export async function startLaunchAgent({ stdout, env, onMutation, assertCurrent, preserveAutoStart, }: GatewayServiceControlArgs): Promise { const serviceEnv = env ?? (process.env as GatewayServiceEnv); const domain = resolveLaunchAgentGuiDomain(); const label = resolveLaunchAgentLabel(serviceEnv); const plistPath = resolveLaunchAgentPlistPath(serviceEnv); const serviceTarget = `${domain}/${label}`; const reportMutation = createGatewayLifecycleMutationReporter(onMutation); await assertNoSystemLaunchDaemonOwnership(label); // Enable is an independent mutation; audit it even if the later launch fails. let enabled = false; if (!preserveAutoStart) { assertCurrent?.(); const enable = await execLaunchctl(["enable", serviceTarget]); enabled = enable.code === 0; if (enabled) { reportMutation("enable"); } } assertCurrent?.(); let start = await execLaunchctl(["kickstart", serviceTarget]); if (isLaunchctlNotLoaded(start)) { await bootstrapLaunchAgentOrThrow({ domain, serviceTarget, plistPath, actionHint: "openclaw gateway start", onMutation: reportMutation, skipEnable: preserveAutoStart || enabled, assertCurrent, }); // Loading does not start demand-only jobs. Without -k, an auto-started job is left running. assertCurrent?.(); start = await execLaunchctl(["kickstart", serviceTarget]); } if (start.code !== 0) { throw new Error(`launchctl kickstart failed: ${start.stderr || start.stdout}`.trim()); } reportMutation("kickstart"); writeLaunchAgentActionLine(stdout, "Started LaunchAgent", serviceTarget); } export async function restartLaunchAgent({ preserveDefinition, stdout, env, warn, onMutation, }: GatewayServiceControlArgs): Promise { const serviceEnv = env ?? (process.env as GatewayServiceEnv); const domain = resolveLaunchAgentGuiDomain(); const label = resolveLaunchAgentLabel(serviceEnv); const plistPath = resolveLaunchAgentPlistPath(serviceEnv); const serviceTarget = `${domain}/${label}`; const reportMutation = createGatewayLifecycleMutationReporter(onMutation); await assertNoSystemLaunchDaemonOwnership(label); const detached = await isCurrentProcessInsideLaunchdService(label); if (!detached) { const { env: cleanupEnv, port: cleanupPort, probeHosts, } = await resolveLaunchAgentGatewayContext(serviceEnv); if (cleanupPort !== null) { assertGatewayServiceUpdateCurrent(); cleanStaleGatewayProcessesSync(cleanupPort, { env: cleanupEnv, assertCurrent: assertGatewayServiceUpdateCurrent, // Resolve after lsof captures its listener snapshot. A KeepAlive respawn // during enumeration must be protected before candidate filtering/signals. resolveProtectedPid: () => { const pid = readLaunchAgentPidForCleanupSync(serviceTarget); assertGatewayServiceUpdateCurrent(); return pid; }, }); const diagnostics = await inspectPortUsage(cleanupPort, { probeHosts, }).catch(() => null); if (diagnostics?.status === "busy") { const runtime = await readLaunchAgentRuntime(serviceEnv); const managedPid = runtime.pid; // Only the current supervised PID may keep the port busy before a // disruptive restart. Re-read after cleanup to close over a concurrent // launchd respawn rather than trusting the protected pre-cleanup PID. const ownedByLaunchAgent = managedPid !== undefined && diagnostics.listeners.length > 0 && diagnostics.listeners.every((listener) => listener.pid === managedPid); if (!ownedByLaunchAgent) { throw new Error( [ `gateway port ${cleanupPort} is busy but is not verifiably owned by LaunchAgent ${label}`, ...formatPortDiagnostics(diagnostics), ].join("\n"), ); } } } } // Preservation permits native activation only, including detached handoffs. const plistReloadNeeded = !preserveDefinition && (await rewriteLaunchAgentPlistForRestart({ env: serviceEnv, label, plistPath, stdout, warn, })); // Restart requests issued from inside the managed gateway process tree need a // detached handoff. A direct `kickstart -k` would terminate the caller before // it can finish the restart command. if (detached) { if (isUpdateOwnedGatewayServiceCommand()) { throw new Error( "UPDATE_NATIVE_AUTHORITY: update-owned native restart requires an external executor, not a detached service handoff.", ); } const handoff = scheduleDetachedLaunchdRestartHandoff({ env: serviceEnv, mode: plistReloadNeeded ? "reload" : "kickstart", waitForPid: process.pid, }); if (!handoff.ok) { throw new Error(`launchd restart handoff failed: ${handoff.error}`); } reportMutation(plistReloadNeeded ? "handoff-reload" : "handoff-kickstart"); writeLaunchAgentActionLine(stdout, "Scheduled LaunchAgent restart", serviceTarget); return { outcome: "scheduled" }; } // `openclaw gateway restart` is an explicit operator request to bring the // LaunchAgent back, so clear any persisted disabled state before restart. const enable = await execLaunchctl(["enable", serviceTarget]); if (enable.code === 0) { reportMutation("enable"); } if (plistReloadNeeded) { const bootout = await execLaunchctl(["bootout", serviceTarget]); if (bootout.code !== 0 && !isLaunchctlNotLoaded(bootout)) { throw new Error(`launchctl bootout failed: ${formatLaunchctlResultDetail(bootout)}`); } if (bootout.code === 0) { reportMutation("bootout"); } try { await bootstrapLaunchAgentOrThrow({ domain, serviceTarget, plistPath, actionHint: "openclaw gateway restart", onMutation: reportMutation, retryPendingTeardown: true, }); } catch (error) { // bootout already removed the job from the domain, so a failed bootstrap // leaves the gateway down with no KeepAlive respawn to recover it. Restore // the job before surfacing the original failure, as the kickstart path does. const restored = await ensureLaunchAgentLoadedAfterFailure({ domain, serviceTarget, plistPath, onMutation: reportMutation, }); if (restored.loaded) { throw error; } throw new Error( formatLaunchAgentLeftUnloadedError({ domain, serviceTarget, plistPath, failure: error instanceof Error ? error.message : String(error), restoreDetail: restored.detail, }), { cause: error }, ); } writeLaunchAgentActionLine(stdout, "Restarted LaunchAgent", serviceTarget); return { outcome: "completed" }; } const start = await execLaunchctl(["kickstart", "-k", serviceTarget]); if (start.code === 0) { reportMutation("kickstart"); writeLaunchAgentActionLine(stdout, "Restarted LaunchAgent", serviceTarget); return { outcome: "completed" }; } if (!isLaunchctlNotLoaded(start)) { const restored = await ensureLaunchAgentLoadedAfterFailure({ domain, serviceTarget, plistPath, onMutation: reportMutation, }); const failure = `launchctl kickstart failed: ${start.stderr || start.stdout}`.trim(); if (restored.loaded) { throw new Error(failure); } throw new Error( formatLaunchAgentLeftUnloadedError({ domain, serviceTarget, plistPath, failure, restoreDetail: restored.detail, }), ); } // A preserved plist may be demand-only; bootstrap alone only registers it. await bootstrapLaunchAgentOrThrow({ domain, serviceTarget, plistPath, actionHint: "openclaw gateway restart", onMutation: reportMutation, }); if (preserveDefinition) { const kick = await execLaunchctl(["kickstart", serviceTarget]); if (kick.code !== 0) { throw new Error(`launchctl kickstart failed: ${kick.stderr || kick.stdout}`.trim()); } reportMutation("kickstart"); } writeLaunchAgentActionLine(stdout, "Restarted LaunchAgent", serviceTarget); return { outcome: "completed" }; }