import type { ChildProcess } from "node:child_process"; import { EventEmitter } from "node:events"; import fs from "node:fs/promises"; import { beforeEach, describe, expect, it, vi } from "vitest"; const spawnMock = vi.hoisted(() => vi.fn()); vi.mock("node:child_process", () => ({ spawn: spawnMock, })); const profileMocks = vi.hoisted(() => ({ buildCellRunArgs: vi.fn((_profile: unknown, options: { environmentFile: string }) => [ "run", "--env-file", options.environmentFile, "cell-image", ]), buildCellCreateArgs: vi.fn((_profile: unknown, options: { environmentFile: string }) => [ "create", "--env-file", options.environmentFile, "cell-image", ]), validateCellContainerProfile: vi.fn(), validateFleetImage: vi.fn((image: string) => image), })); vi.mock("./cell-profile.js", () => profileMocks); import type { CellContainerProfile } from "./cell-profile.js"; import { createRedactingStreamWriter } from "./containers.redaction.js"; import { createFleetContainerRuntime } from "./containers.runtime.js"; type FleetContainerCommandExecutor = NonNullable[0]>; type FleetContainerStreamExecutor = NonNullable[1]>; function successfulExecutor() { return vi.fn(async () => ({ stdout: "", stderr: "", code: 0, })); } function createStreamChild(pid?: number): ChildProcess { const stdout = Object.assign(new EventEmitter(), { pause: vi.fn(), resume: vi.fn(), }); const stderr = Object.assign(new EventEmitter(), { pause: vi.fn(), resume: vi.fn(), }); return Object.assign(new EventEmitter(), { pid, stdout, stderr, kill: vi.fn(() => true), }) as unknown as ChildProcess; } describe("fleet container runtime", () => { beforeEach(() => { vi.clearAllMocks(); }); it("accepts only local Docker endpoints", async () => { const localExecutor = vi.fn(async () => ({ stdout: JSON.stringify([ { Endpoints: { docker: { Host: "unix:///run/user/1000/docker.sock" } } }, ]), stderr: "", code: 0, })); await expect( createFleetContainerRuntime(localExecutor).assertLocal("docker"), ).resolves.toBeUndefined(); expect(localExecutor).toHaveBeenCalledWith("docker", ["context", "inspect"], {}); const remoteExecutor = vi.fn(async () => ({ stdout: JSON.stringify([ { Endpoints: { docker: { Host: "tcp://remote.example.invalid:2376" } } }, ]), stderr: "", code: 0, })); await expect(createFleetContainerRuntime(remoteExecutor).assertLocal("docker")).rejects.toThrow( /local Docker endpoint.*remote cells/iu, ); expect(remoteExecutor).toHaveBeenCalledWith("docker", ["context", "inspect"], {}); }); it.each([ [false, true], [true, false], ] as const)("classifies Podman serviceIsRemote=%s", async (serviceIsRemote, accepted) => { const executor = vi.fn(async () => ({ stdout: JSON.stringify({ host: { serviceIsRemote } }), stderr: "", code: 0, })); const check = createFleetContainerRuntime(executor).assertLocal("podman"); if (accepted) { await expect(check).resolves.toBeUndefined(); } else { await expect(check).rejects.toThrow(/local Podman.*remote cells/iu); } expect(executor).toHaveBeenCalledWith("podman", ["info", "--format", "json"], {}); }); it("fails closed on malformed runtime-locality output", async () => { const executor = vi.fn(async () => ({ stdout: "{}", stderr: "", code: 0, })); await expect(createFleetContainerRuntime(executor).assertLocal("docker")).rejects.toThrow( /invalid response/iu, ); await expect(createFleetContainerRuntime(executor).assertLocal("podman")).rejects.toThrow( /invalid response/iu, ); }); it("uses run or create args according to the requested start mode", async () => { const environmentFiles: string[] = []; const executor = vi.fn(async (_runtime, args) => { const environmentFile = args[args.indexOf("--env-file") + 1]; if (!environmentFile) { throw new Error("missing environment file"); } environmentFiles.push(environmentFile); await expect(fs.readFile(environmentFile, "utf8")).resolves.toBe( "AAA_FEATURE=synthetic-first\nOPENCLAW_GATEWAY_TOKEN=fake-value\nZZZ_FEATURE=synthetic-last\n", ); expect(args.join(" ")).not.toContain("fake-value"); expect(args.join(" ")).not.toContain("synthetic-first"); expect(args.join(" ")).not.toContain("synthetic-last"); return { stdout: "", stderr: "", code: 0 }; }); const runtime = createFleetContainerRuntime(executor); const profile = { runtime: "podman", environment: { ZZZ_FEATURE: "synthetic-last", OPENCLAW_GATEWAY_TOKEN: "fake-value", AAA_FEATURE: "synthetic-first", }, } as unknown as CellContainerProfile; await runtime.run(profile, true); await runtime.run(profile, false); expect(profileMocks.buildCellRunArgs).toHaveBeenCalledWith(profile, { environmentFile: environmentFiles[0], }); expect(profileMocks.buildCellCreateArgs).toHaveBeenCalledWith(profile, { environmentFile: environmentFiles[1], }); expect(executor).toHaveBeenNthCalledWith( 1, "podman", ["run", "--env-file", environmentFiles[0], "cell-image"], { redactValues: ["synthetic-last", "fake-value", "synthetic-first"] }, ); expect(executor).toHaveBeenNthCalledWith( 2, "podman", ["create", "--env-file", environmentFiles[1], "cell-image"], { redactValues: ["synthetic-last", "fake-value", "synthetic-first"], }, ); await Promise.all( environmentFiles.map(async (environmentFile) => { await expect(fs.stat(environmentFile)).rejects.toMatchObject({ code: "ENOENT" }); }), ); }); it("preserves fleet profile validation errors before staging an invalid environment", async () => { const executor = successfulExecutor(); const failure = new Error("Invalid fleet cell environment entry: BAD KEY"); profileMocks.validateCellContainerProfile.mockImplementationOnce(() => { throw failure; }); const profile = { runtime: "docker", environment: { "BAD KEY": "synthetic-invalid-value" }, } as unknown as CellContainerProfile; await expect(createFleetContainerRuntime(executor).run(profile, true)).rejects.toBe(failure); expect(executor).not.toHaveBeenCalled(); }); it("normalizes Docker and Podman inspect output", async () => { const executor = vi.fn(async () => ({ stdout: JSON.stringify([ { Id: "container-id", Image: "sha256:old-image-id", State: { Status: "running", Running: true }, Config: { Env: ["OPENCLAW_GATEWAY_TOKEN=test-auth-token", "FEATURE=a=b"], Image: "ghcr.io/openclaw/openclaw:latest", Labels: { "openclaw.fleet.tenant": "acme" }, User: "1000:1000", }, HostConfig: { Memory: 2_147_483_648, NanoCpus: 2_000_000_000, PidsLimit: 512, UsernsMode: "keep-id", }, }, ]), stderr: "", code: 0, })); await expect( createFleetContainerRuntime(executor).inspect("podman", "cell-acme"), ).resolves.toEqual({ kind: "ok", containerId: "container-id", state: "running", running: true, labels: { "openclaw.fleet.tenant": "acme" }, environment: { OPENCLAW_GATEWAY_TOKEN: "test-auth-token", FEATURE: "a=b" }, imageId: "sha256:old-image-id", memory: "2147483648", cpus: "2", pidsLimit: 512, storageOpt: {}, capDrop: [], securityOpt: [], init: undefined, restartPolicy: undefined, portBindings: [], user: "1000:1000", usernsMode: "keep-id", }); expect(executor).toHaveBeenCalledWith("podman", ["container", "inspect", "cell-acme"], { allowFailure: true, }); }); it("detects a rootless Docker daemon from security options", async () => { const executor = vi.fn(async () => ({ stdout: JSON.stringify(["name=seccomp,profile=builtin", "name=rootless"]), stderr: "", code: 0, })); await expect(createFleetContainerRuntime(executor).isDockerRootless()).resolves.toBe(true); expect(executor).toHaveBeenCalledWith( "docker", ["info", "--format", "{{json .SecurityOptions}}"], {}, ); }); it("distinguishes missing containers from unavailable runtimes", async () => { const missingExecutor = vi.fn(async () => ({ stdout: "", stderr: "Error: no such container: cell-missing", code: 1, })); const unavailableExecutor = vi.fn(async () => ({ stdout: "", stderr: "Cannot connect to the Docker daemon", code: 1, })); await expect( createFleetContainerRuntime(missingExecutor).inspect("docker", "cell-missing"), ).resolves.toEqual({ kind: "missing", state: "missing" }); await expect( createFleetContainerRuntime(unavailableExecutor).inspect("docker", "cell-acme"), ).resolves.toEqual({ kind: "unavailable", state: "unknown", error: "Cannot connect to the Docker daemon", }); }); it.each([ ["docker", "Labels", "Containers", "Name"], ["podman", "labels", "containers", "name"], ] as const)( "normalizes %s network labels and attached containers", async (runtimeName, labelsField, containersField, nameField) => { const executor = vi.fn(async () => ({ stdout: JSON.stringify([ { [labelsField]: { "openclaw.fleet.tenant": "acme", "openclaw.fleet.owner": "owner-id", }, [containersField]: { "container-b": { [nameField]: "peer-b" }, "container-a": { [nameField]: "peer-a" }, }, }, ]), stderr: "", code: 0, })); await expect( createFleetContainerRuntime(executor).inspectNetwork(runtimeName, "openclaw-cell-acme-net"), ).resolves.toEqual({ kind: "ok", labels: { "openclaw.fleet.tenant": "acme", "openclaw.fleet.owner": "owner-id", }, attachedContainers: [ { id: "container-a", name: "peer-a" }, { id: "container-b", name: "peer-b" }, ], internal: false, }); expect(executor).toHaveBeenCalledWith( runtimeName, ["network", "inspect", "openclaw-cell-acme-net"], { allowFailure: true }, ); }, ); it("normalizes a network with no attached containers", async () => { const executor = vi.fn(async () => ({ stdout: JSON.stringify([{ Labels: {}, Containers: {} }]), stderr: "", code: 0, })); await expect( createFleetContainerRuntime(executor).inspectNetwork("docker", "openclaw-cell-acme-net"), ).resolves.toEqual({ kind: "ok", labels: {}, attachedContainers: [], internal: false }); }); it("distinguishes missing networks from unavailable runtimes", async () => { const missingExecutor = vi.fn(async () => ({ stdout: "", stderr: "Error: network openclaw-cell-missing-net not found", code: 1, })); const unavailableExecutor = vi.fn(async () => ({ stdout: "", stderr: "Cannot connect to the Docker daemon", code: 1, })); await expect( createFleetContainerRuntime(missingExecutor).inspectNetwork( "docker", "openclaw-cell-missing-net", ), ).resolves.toEqual({ kind: "missing" }); await expect( createFleetContainerRuntime(unavailableExecutor).inspectNetwork( "docker", "openclaw-cell-acme-net", ), ).resolves.toEqual({ kind: "unavailable", error: "Cannot connect to the Docker daemon", }); }); it("treats malformed network inspect JSON as unavailable", async () => { const executor = vi.fn(async () => ({ stdout: JSON.stringify([{ Labels: { "openclaw.fleet.tenant": 42 } }]), stderr: "", code: 0, })); await expect( createFleetContainerRuntime(executor).inspectNetwork("docker", "openclaw-cell-acme-net"), ).resolves.toEqual({ kind: "unavailable", error: "network inspect returned an invalid response", }); }); it("treats malformed inspect JSON as unavailable without echoing its output", async () => { const executor = vi.fn(async () => ({ stdout: 'not-json OPENCLAW_GATEWAY_TOKEN="secret"', stderr: "", code: 0, })); const result = await createFleetContainerRuntime(executor).inspect("docker", "cell-acme"); expect(result).toEqual({ kind: "unavailable", state: "unknown", error: "container inspect returned an invalid response", }); expect(JSON.stringify(result)).not.toContain("secret"); }); it("redacts environment values from command failures", async () => { const executor = vi.fn(async () => { throw new Error("runtime rejected fake-value"); }); const runtime = createFleetContainerRuntime(executor); const profile = { runtime: "docker", environment: { OPENCLAW_GATEWAY_TOKEN: "fake-value" }, } as unknown as CellContainerProfile; let failure: unknown; try { await runtime.run(profile, true); } catch (error) { failure = error; } expect(failure).toBeInstanceOf(Error); expect((failure as Error).message).toContain(""); expect((failure as Error).message).not.toContain("fake-value"); }); it("executes lifecycle commands with the selected runtime", async () => { const executor = successfulExecutor(); const runtime = createFleetContainerRuntime(executor); await runtime.pull("docker", "image:v2"); await runtime.start("docker", "cell-acme"); await runtime.stop("docker", "cell-acme"); await runtime.restart("docker", "cell-acme"); await runtime.remove("docker", "cell-acme", true); await runtime.remove("docker", "cell-acme", false); expect(executor.mock.calls.map(([, args]) => args)).toEqual([ ["pull", "image:v2"], ["start", "cell-acme"], ["stop", "cell-acme"], ["restart", "cell-acme"], ["rm", "--force", "cell-acme"], ["rm", "cell-acme"], ]); }); it.each([ [{}, ["logs", "cell-acme"]], [{ follow: true }, ["logs", "--follow", "cell-acme"]], [{ timestamps: true }, ["logs", "--timestamps", "cell-acme"]], [{ tail: 200 }, ["logs", "--tail", "200", "cell-acme"]], [{ since: "10m" }, ["logs", "--since", "10m", "cell-acme"]], [ { follow: true, timestamps: true, tail: 100, since: "2026-07-11T10:00:00Z" }, [ "logs", "--follow", "--timestamps", "--tail", "100", "--since", "2026-07-11T10:00:00Z", "cell-acme", ], ], ] as const)("streams logs with exact argv for %o", async (options, expectedArgs) => { const stream = vi.fn(async () => ({ code: 0, signal: null })); const runtime = createFleetContainerRuntime(successfulExecutor(), stream); await expect( runtime.logs("podman", "cell-acme", { ...options, redactValues: ["secret"] }), ).resolves.toBeUndefined(); expect(stream).toHaveBeenCalledWith("podman", expectedArgs, { redactValues: ["secret"] }); }); it.each(["", "-1h", "10 m", "10m\n"])("rejects unsafe --since value %o", async (since) => { const stream = vi.fn(async () => ({ code: 0, signal: null })); const runtime = createFleetContainerRuntime(successfulExecutor(), stream); await expect(runtime.logs("docker", "cell-acme", { since, redactValues: [] })).rejects.toThrow( /--since/iu, ); expect(stream).not.toHaveBeenCalled(); }); it("reports stream failures but accepts operator interrupt while following", async () => { const stream = vi.fn(); const runtime = createFleetContainerRuntime(successfulExecutor(), stream); stream .mockResolvedValueOnce({ code: 7, signal: null }) .mockResolvedValueOnce({ code: 130, signal: null }) .mockResolvedValueOnce({ code: 130, signal: null }) .mockResolvedValueOnce({ code: null, signal: "SIGTERM" }) .mockResolvedValueOnce({ code: null, signal: "SIGTERM" }); await expect(runtime.logs("podman", "cell-acme", { redactValues: [] })).rejects.toThrow( /podman logs failed with exit code 7/iu, ); await expect( runtime.logs("podman", "cell-acme", { follow: true, redactValues: [] }), ).resolves.toBeUndefined(); await expect(runtime.logs("podman", "cell-acme", { redactValues: [] })).rejects.toThrow( /podman logs failed with exit code 130/iu, ); // A forwarded termination signal ends a follow stream cleanly but fails a bounded read. await expect( runtime.logs("podman", "cell-acme", { follow: true, redactValues: [] }), ).resolves.toBeUndefined(); await expect(runtime.logs("podman", "cell-acme", { redactValues: [] })).rejects.toThrow( /podman logs failed with signal SIGTERM/iu, ); // Crash signals are never masked as operator stops, even while following. stream.mockResolvedValueOnce({ code: null, signal: "SIGSEGV" }); await expect( runtime.logs("podman", "cell-acme", { follow: true, redactValues: [] }), ).rejects.toThrow(/podman logs failed with signal SIGSEGV/iu); }); it("rejects a log stream spawn error when the child has no pid", async () => { const child = createStreamChild(); spawnMock.mockReturnValue(child); const runtime = createFleetContainerRuntime(successfulExecutor()); const logs = runtime.logs("podman", "cell-acme", { follow: true, redactValues: [] }); child.emit("error", new Error("spawn failed")); await expect(logs).rejects.toThrow("spawn failed"); child.emit("close", -2, null); }); it("waits for log stream close across repeated operational errors", async () => { const child = createStreamChild(4242); spawnMock.mockReturnValue(child); const runtime = createFleetContainerRuntime(successfulExecutor()); let settled = false; const logs = runtime .logs("podman", "cell-acme", { follow: true, redactValues: [] }) .finally(() => { settled = true; }); child.emit("error", new Error("first signal delivery failed")); child.emit("error", new Error("second signal delivery failed")); await Promise.resolve(); expect(settled).toBe(false); child.emit("close", 0, null); await expect(logs).resolves.toBeUndefined(); }); it("creates and removes a labeled per-cell network", async () => { const executor = successfulExecutor(); const runtime = createFleetContainerRuntime(executor); await runtime.createNetwork( "podman", "openclaw-cell-acme-net", { "openclaw.fleet.tenant": "acme", "openclaw.fleet.attempt": "attempt-id", "openclaw.fleet.owner": "owner-id", }, { internal: false }, ); await runtime.removeNetwork("podman", "openclaw-cell-acme-net"); expect(executor.mock.calls.map(([, args]) => args)).toEqual([ [ "network", "create", "--driver", "bridge", "--label", "openclaw.fleet.attempt=attempt-id", "--label", "openclaw.fleet.owner=owner-id", "--label", "openclaw.fleet.tenant=acme", "openclaw-cell-acme-net", ], ["network", "rm", "openclaw-cell-acme-net"], ]); }); it.each([ [true, true], [false, false], ] as const)("sets internal=%s on network create", async (internal, expected) => { const executor = successfulExecutor(); const runtime = createFleetContainerRuntime(executor); await runtime.createNetwork("podman", "openclaw-cell-acme-net", {}, { internal }); expect(executor.mock.calls[0]?.[1].includes("--internal")).toBe(expected); }); it("redacts secrets from streamed log output, including across chunk boundaries", () => { const written: string[] = []; const target = { write: (text: string) => written.push(text) } as unknown as NodeJS.WriteStream; const writer = createRedactingStreamWriter(target, ["gw-secret-token"]); writer.write(Buffer.from("boot ok\ntoken=gw-sec")); writer.write(Buffer.from("ret-token done\ntail without newline")); writer.flush(); const output = written.join(""); expect(output).toContain("token= done"); expect(output).toContain("tail without newline"); expect(output).not.toContain("gw-secret-token"); }); it("never splits a secret across a forced long-line flush", () => { const written: string[] = []; const target = { write: (text: string) => written.push(text) } as unknown as NodeJS.WriteStream; const writer = createRedactingStreamWriter(target, ["gw-secret-token"]); // An unterminated line ending exactly in a secret prefix at the flush point. writer.write(Buffer.from(`${"x".repeat(64 * 1024)}gw-sec`)); writer.write(Buffer.from("ret-token trailing")); writer.flush(); const output = written.join(""); expect(output).toContain(" trailing"); expect(output).not.toContain("gw-secret-token"); }); it("parses hardened inspect fields and Docker network internal state", async () => { const executor = vi.fn(async (_runtime, args) => args[0] === "network" ? { stdout: JSON.stringify([{ Labels: {}, Containers: {}, Internal: true }]), stderr: "", code: 0, } : { stdout: JSON.stringify([ { Id: "container-id", Image: "sha256:image", State: { Status: "running", Running: true }, Config: { Env: [], Labels: {} }, // Podman reports null EffectiveCaps when every capability is dropped. EffectiveCaps: null, HostConfig: { Memory: 1, NanoCpus: 1_000_000_000, PidsLimit: 1, StorageOpt: { size: "10g" }, CapDrop: ["ALL"], SecurityOpt: ["no-new-privileges"], Init: true, RestartPolicy: { Name: "unless-stopped" }, PortBindings: { "18789/tcp": [{ HostIp: "127.0.0.1", HostPort: "19100" }] }, }, }, ]), stderr: "", code: 0, }, ); const runtime = createFleetContainerRuntime(executor); await expect(runtime.inspect("docker", "cell-acme")).resolves.toMatchObject({ storageOpt: { size: "10g" }, capDrop: ["ALL"], effectiveCaps: [], securityOpt: ["no-new-privileges"], init: true, restartPolicy: "unless-stopped", portBindings: [{ containerPort: "18789/tcp", hostIp: "127.0.0.1", hostPort: "19100" }], }); await expect(runtime.inspectNetwork("docker", "net-acme")).resolves.toMatchObject({ internal: true, }); }); it("fails closed on invalid hardened inspect field shapes", async () => { const executor = vi.fn(async () => ({ stdout: JSON.stringify([ { Image: "sha256:image", State: { Status: "running", Running: true }, Config: { Env: [], Labels: {} }, HostConfig: { Memory: 1, NanoCpus: 1, PidsLimit: 1, StorageOpt: { size: 10 } }, }, ]), stderr: "", code: 0, })); await expect( createFleetContainerRuntime(executor).inspect("docker", "cell-acme"), ).resolves.toEqual({ kind: "unavailable", state: "unknown", error: "container inspect returned an invalid response", }); }); });