File size: 8,535 Bytes
d197cf3
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
/** LaunchAgent stop semantics and in-service maintenance parking. */
import { formatPortDiagnostics } from "../infra/ports-format.js";
import { inspectPortUsage } from "../infra/ports-inspect.js";
import { probePortUsage } from "../infra/ports-probe.js";
import { cleanStaleGatewayProcessesSync } from "../infra/restart-stale-pids.js";
import { sleep } from "../utils.js";
import { isCurrentProcessInsideLaunchdService } from "./launchd-current-service.js";
import {
  execLaunchctl,
  formatLaunchctlResultDetail,
  isLaunchctlNotLoaded,
} from "./launchd-exec.js";
import { resolveLaunchAgentLabel } from "./launchd-label.js";
import { LAUNCH_AGENT_EXIT_TIMEOUT_SECONDS } from "./launchd-plist.js";
import { scheduleDetachedLaunchdMaintenancePark } from "./launchd-restart-handoff.js";
import {
  resolveLaunchAgentGatewayContext,
  resolveLaunchAgentGuiDomain,
  waitForLaunchAgentStopped,
} from "./launchd-runtime.js";
import { formatLine } from "./output.js";
import { createGatewayLifecycleMutationReporter } from "./service-mutation.js";
import type { GatewayServiceControlArgs, GatewayServiceEnv } from "./service-types.js";
import { assertGatewayServiceUpdateCurrent } from "./service-update-authority.js";

const LAUNCH_AGENT_STOP_PORT_RELEASE_TIMEOUT_MS = LAUNCH_AGENT_EXIT_TIMEOUT_SECONDS * 1_000;
const LAUNCH_AGENT_STOP_PORT_RELEASE_POLL_MS = 100;
async function bootoutLaunchAgentOrThrow(params: {
  serviceTarget: string;
  warning: string;
  stdout: NodeJS.WritableStream;
  onMutation?: () => void;
  assertCurrent?: () => void;
}): Promise<void> {
  params.assertCurrent?.();
  const bootout = await execLaunchctl(["bootout", params.serviceTarget]);
  if (bootout.code !== 0 && !isLaunchctlNotLoaded(bootout)) {
    throw new Error(
      `${params.warning}; launchctl bootout failed: ${formatLaunchctlResultDetail(bootout)}`,
    );
  }
  params.onMutation?.();
  params.stdout.write(`${formatLine("Warning", params.warning)}\n`);
}
async function waitForGatewayPortRelease(
  port: number,
  probeHosts: readonly string[],
): Promise<boolean> {
  const deadline = Date.now() + LAUNCH_AGENT_STOP_PORT_RELEASE_TIMEOUT_MS;
  while (Date.now() < deadline) {
    await sleep(Math.min(LAUNCH_AGENT_STOP_PORT_RELEASE_POLL_MS, deadline - Date.now()));
    const status = await probePortUsage(port, probeHosts);
    if (status === "free") {
      return true;
    }
  }
  return false;
}

async function assertGatewayPortReleasedAfterStop(
  env: GatewayServiceEnv,
  assertCurrent?: () => void,
): Promise<void> {
  const { env: cleanupEnv, port, probeHosts } = await resolveLaunchAgentGatewayContext(env);
  if (port === null) {
    return;
  }
  assertCurrent?.();
  assertGatewayServiceUpdateCurrent();
  cleanStaleGatewayProcessesSync(port, {
    env: cleanupEnv,
    assertCurrent: assertGatewayServiceUpdateCurrent,
  });
  const diagnostics = await inspectPortUsage(port, {
    probeHosts,
  }).catch(() => null);
  if (diagnostics?.status !== "busy") {
    return;
  }
  if (await waitForGatewayPortRelease(port, probeHosts)) {
    return;
  }
  throw new Error(
    [
      `gateway port ${port} is still busy after LaunchAgent stop`,
      ...formatPortDiagnostics(diagnostics),
    ].join("\n"),
  );
}

export async function stopLaunchAgent({
  stdout,
  env,
  disable: persistDisable,
  onMutation,
  assertCurrent,
}: GatewayServiceControlArgs): Promise<void> {
  const serviceEnv = env ?? (process.env as GatewayServiceEnv);
  const domain = resolveLaunchAgentGuiDomain();
  const label = resolveLaunchAgentLabel(serviceEnv);
  const serviceTarget = `${domain}/${label}`;
  const reportMutation = createGatewayLifecycleMutationReporter(onMutation);

  if (await isCurrentProcessInsideLaunchdService(label, process.env)) {
    throw new Error(
      `Refusing to stop LaunchAgent ${label} from inside the same launchd service; run this command from an external shell.`,
    );
  }

  assertCurrent?.();
  if (!persistDisable) {
    // Default: bootout only. Removes the job from the current launchd domain without
    // persisting a disable, so KeepAlive auto-recovery survives future crashes and
    // `openclaw gateway start` re-enables cleanly without a manual `launchctl enable`.
    const bootout = await execLaunchctl(["bootout", serviceTarget]);
    if (bootout.code !== 0 && !isLaunchctlNotLoaded(bootout)) {
      throw new Error(`launchctl bootout failed: ${formatLaunchctlResultDetail(bootout)}`);
    }
    reportMutation("bootout");
    await assertGatewayPortReleasedAfterStop(serviceEnv, assertCurrent);
    stdout.write(`${formatLine("Stopped LaunchAgent", serviceTarget)}\n`);
    return;
  }

  // --disable: persistently suppress KeepAlive/RunAtLoad before stopping.
  // Without this, launchd can relaunch the process as soon as `stop` exits.
  const disableResult = await execLaunchctl(["disable", serviceTarget]);
  if (disableResult.code !== 0) {
    await bootoutLaunchAgentOrThrow({
      serviceTarget,
      assertCurrent,
      stdout,
      warning: `launchctl disable failed; used bootout fallback and left service unloaded: ${formatLaunchctlResultDetail(disableResult)}`,
      onMutation: () => reportMutation("disable-bootout"),
    });
    await assertGatewayPortReleasedAfterStop(serviceEnv, assertCurrent);
    stdout.write(`${formatLine("Stopped LaunchAgent (degraded)", serviceTarget)}\n`);
    return;
  }
  reportMutation("disable");

  // `launchctl stop` targets the plain label (not the fully-qualified service target).
  assertCurrent?.();
  const stop = await execLaunchctl(["stop", label]);
  if (stop.code !== 0 && !isLaunchctlNotLoaded(stop)) {
    await bootoutLaunchAgentOrThrow({
      serviceTarget,
      assertCurrent,
      stdout,
      warning: `launchctl stop failed; used bootout fallback and left service unloaded: ${formatLaunchctlResultDetail(stop)}`,
      onMutation: () => reportMutation("disable-bootout"),
    });
    await assertGatewayPortReleasedAfterStop(serviceEnv, assertCurrent);
    stdout.write(`${formatLine("Stopped LaunchAgent (degraded)", serviceTarget)}\n`);
    return;
  }

  reportMutation("disable-stop");

  const stopState = await waitForLaunchAgentStopped(serviceTarget);
  if (stopState.state !== "stopped" && stopState.state !== "not-loaded") {
    const warning =
      stopState.state === "unknown"
        ? `launchctl print could not confirm stop; used bootout fallback and left service unloaded: ${stopState.detail ?? "unknown error"}`
        : "launchctl stop did not fully stop the service; used bootout fallback and left service unloaded";
    await bootoutLaunchAgentOrThrow({
      serviceTarget,
      assertCurrent,
      stdout,
      warning,
      onMutation: () => reportMutation("disable-bootout"),
    });
    await assertGatewayPortReleasedAfterStop(serviceEnv, assertCurrent);
    stdout.write(`${formatLine("Stopped LaunchAgent (degraded)", serviceTarget)}\n`);
    return;
  }

  await assertGatewayPortReleasedAfterStop(serviceEnv, assertCurrent);
  stdout.write(`${formatLine("Stopped LaunchAgent", serviceTarget)}\n`);
}

export async function parkCurrentLaunchAgentForMaintenance(
  params: {
    env?: GatewayServiceEnv;
  } = {},
): Promise<boolean> {
  const serviceEnv = params.env ?? (process.env as GatewayServiceEnv);
  const domain = resolveLaunchAgentGuiDomain();
  const label = resolveLaunchAgentLabel(serviceEnv);
  if (!(await isCurrentProcessInsideLaunchdService(label, process.env))) {
    return false;
  }
  const serviceTarget = `${domain}/${label}`;
  // Disable before exit so KeepAlive cannot spawn a replacement before the
  // detached handoff can boot the current job out of launchd.
  const disable = await execLaunchctl(["disable", serviceTarget]);
  if (disable.code !== 0) {
    throw new Error(
      `launchctl disable failed while parking ${serviceTarget}: ${formatLaunchctlResultDetail(disable)}`,
    );
  }
  const handoff = scheduleDetachedLaunchdMaintenancePark({
    env: serviceEnv,
    waitForPid: process.pid,
  });
  const handoffError = !handoff.ok
    ? handoff.error
    : (await handoff.value)
      ? undefined
      : "helper failed to spawn";
  if (handoffError) {
    const rollback = await execLaunchctl(["enable", serviceTarget]);
    const rollbackDetail =
      rollback.code === 0
        ? "restored launchd enable state"
        : `launchctl enable rollback failed: ${formatLaunchctlResultDetail(rollback)}`;
    throw new Error(`launchd maintenance park handoff failed: ${handoffError}; ${rollbackDetail}`);
  }
  return true;
}