Blame for src/cron/cron-execution-diagnostics.e2e.test.ts

import { createServer, type Server, type AddressInfo } from "node:net";
import { afterAll, beforeEach, describe, expect, it, vi } from "vitest";
import { FailoverError } from "../agents/failover-error.js";
import {
  createAgentRunDirectAbortError,
  createAgentRunRestartAbortError,
  createAgentRunSupersededAbortError,
} from "../agents/run-termination.js";
import type { OpenClawConfig } from "../config/config.js";
import { createAgentRunStaleLifecycleError } from "../infra/agent-lifecycle-error.js";
import { resetTaskRegistryForTests } from "../tasks/task-runtime.test-helpers.js";
import { withOpenClawTestState } from "../test-utils/openclaw-test-state.js";
import {
  loadRunCronIsolatedAgentTurn,
  dispatchCronDeliveryMock,
  mockRunCronFallbackPassthrough,
  resetRunCronIsolatedAgentTurnHarness,
  resolveAllowedModelRefMock,
  resolveConfiguredModelRefMock,
  runEmbeddedAgentMock,
  runWithModelFallbackMock,
} from "./isolated-agent/run.test-harness.js";
import { CronService, type CronEvent } from "./service.js";
import { createNoopLogger } from "./service.test-harness.js";
import { cronStoreKey } from "./store/key.js";
import { readCronTaskRunHistoryPage } from "./task-run-history.js";

vi.doUnmock("./isolated-agent/model-preflight.runtime.js");

const runCronIsolatedAgentTurn = await loadRunCronIsolatedAgentTurn();
const { resetCronModelProviderPreflightCacheForTest } =
  await import("./isolated-agent/model-preflight.runtime.js");

type ModelRef = { provider: string; model: string };

function makeCommandLaneTaskTimeoutError(lane: string, timeoutMs: number): Error {
  const error = new Error(`Command lane "${lane}" task timed out after ${timeoutMs}ms`);
  error.name = "CommandLaneTaskTimeoutError";
  return error;
}

async function listenForBrokenHttpConnections(): Promise<{
  baseUrl: string;
  server: Server;
}> {
  const server = createServer((socket) => socket.destroy());
  await new Promise<void>((resolve, reject) => {
    server.once("error", reject);
    server.listen(0, "127.0.0.1", resolve);
  });
  const address = server.address() as AddressInfo;
  return { baseUrl: `http://127.0.0.1:${address.port}`, server };
}

async function closeServer(server: Server): Promise<void> {
  await new Promise<void>((resolve, reject) => {
    server.close((error) => (error ? reject(error) : resolve()));
  });
}

function configFor(modelRef: ModelRef, localBaseUrl?: string): OpenClawConfig {
  return {
    agents: {
      defaults: {
        model: `${modelRef.provider}/${modelRef.model}`,
      },
    },
    ...(localBaseUrl
      ? {
          models: {
            providers: {
              [modelRef.provider]: {
                api: "ollama",
                baseUrl: localBaseUrl,
                models: [],
              },
            },
          },
        }
      : {}),
  };
}

async function runPersistedDiagnosticCase(params: {
  cfg: OpenClawConfig;
  modelRef: ModelRef;
  name: string;
}) {
  return await withOpenClawTestState(
    { layout: "state-only", prefix: "openclaw-cron-execution-diagnostics-" },
    async (state) => {
      resetTaskRegistryForTests();
      const events: CronEvent[] = [];
      const storePath = state.path("cron", "jobs.json");
      const cron = new CronService({
        storePath,
        cronEnabled: true,
        cronConfig: { triggers: { enabled: true } },
        log: createNoopLogger(),
        enqueueSystemEvent: vi.fn(),
        requestHeartbeat: vi.fn(),
        onEvent: (event) => events.push(structuredClone(event)),
        runIsolatedAgentJob: async (runParams) =>
          await runCronIsolatedAgentTurn({
            ...runParams,
            cfg: params.cfg,
            deps: {},
            sessionKey: `cron:${runParams.job.id}`,
          }),
      });
      try {
        await cron.start();
        const job = await cron.add({
          name: params.name,
          enabled: true,
          schedule: { kind: "every", everyMs: 60_000 },
          sessionTarget: "isolated",
          wakeMode: "next-heartbeat",
          payload: {
            kind: "agentTurn",
            message: `run ${params.name}`,
            model: `${params.modelRef.provider}/${params.modelRef.model}`,
          },
          delivery: { mode: "none" },
        });
        await expect(cron.run(job.id, "force")).resolves.toEqual({ ok: true, ran: true });

        const finished = events.find(
          (event) => event.action === "finished" && event.jobId === job.id,
        );
        const history = readCronTaskRunHistoryPage({
          storeKey: cronStoreKey(storePath),
          jobId: job.id,
          limit: 1,
        }).entries[0];
        expect(finished).toBeDefined();
        expect(history).toBeDefined();
        return {
          finished: finished!,
          history: history!,
          lastError: cron.getJob(job.id)?.state.lastError,
          lastErrorReason: cron.getJob(job.id)?.state.lastErrorReason,
        };
      } finally {
        cron.stop();
        resetTaskRegistryForTests({ persist: false });
      }
    },
  );
}

describe("cron execution diagnostics", { concurrent: false }, () => {
  const servers: Server[] = [];

  beforeEach(() => {
    resetRunCronIsolatedAgentTurnHarness();
    resetCronModelProviderPreflightCacheForTest();
  });

  afterAll(async () => {
    await Promise.all(servers.map(closeServer));
  });

  it("persists and emits a real isolated-run local-provider preflight failure", async () => {
    const endpoint = await listenForBrokenHttpConnections();
    servers.push(endpoint.server);
    const modelRef = { provider: "ollama", model: "diagnostic-model" };
    resolveConfiguredModelRefMock.mockReturnValue(modelRef);
    resolveAllowedModelRefMock.mockReturnValue({ ref: modelRef });

    const { finished, history } = await runPersistedDiagnosticCase({
      cfg: configFor(modelRef, endpoint.baseUrl),
      modelRef,
      name: "unreachable provider",
    });

    expect(runWithModelFallbackMock).not.toHaveBeenCalled();
    expect(runEmbeddedAgentMock).not.toHaveBeenCalled();
    for (const outcome of [finished, history]) {
      expect(outcome).toMatchObject({
        status: "skipped",
        provider: "ollama",
        model: "diagnostic-model",
        error: expect.stringContaining("unavailable"),
        diagnostics: {
          entries: expect.arrayContaining([
            expect.objectContaining({
              source: "model-preflight",
              severity: "warn",
              message: expect.stringContaining("unavailable"),
            }),
          ]),
        },
      });
    }
  });

  it("persists and emits the canonical terminal timeout diagnostic", async () => {
    const modelRef = { provider: "openai", model: "gpt-5.4" };
    resolveConfiguredModelRefMock.mockReturnValue(modelRef);
    runWithModelFallbackMock.mockRejectedValueOnce(
      makeCommandLaneTaskTimeoutError("cron-nested", 330_000),
    );

    const { finished, history } = await runPersistedDiagnosticCase({
      cfg: configFor(modelRef),
      modelRef,
      name: "nested lane timeout",
    });

    for (const outcome of [finished, history]) {
      expect(outcome).toMatchObject({
        status: "error",
        provider: "openai",
        model: "gpt-5.4",
        error: "cron: job execution timed out",
        diagnostics: {
          entries: expect.arrayContaining([
            expect.objectContaining({
              source: "cron-setup",
              severity: "error",
              message: "cron: job execution timed out",
            }),
          ]),
        },
      });
    }
  });

  it("persists provider failures without internal class names", async () => {
    const message = "Saved selection requires an update.";
    const modelRef = { provider: "openai", model: "not-a-real-model" };
    resolveConfiguredModelRefMock.mockReturnValue(modelRef);
    resolveAllowedModelRefMock.mockReturnValue({ ref: modelRef });
    runWithModelFallbackMock.mockRejectedValueOnce(
      new FailoverError(message, {
        reason: "model_not_found",
        provider: modelRef.provider,
        model: modelRef.model,
      }),
    );

    const { finished, history, lastError, lastErrorReason } = await runPersistedDiagnosticCase({
      cfg: configFor(modelRef),
      modelRef,
      name: "missing provider model",
    });

    for (const outcome of [finished, history]) {
      expect(outcome).toMatchObject({
        status: "error",
        provider: modelRef.provider,
        model: modelRef.model,
        error: message,
        diagnostics: { summary: message },
      });
      expect(outcome.error).not.toContain("FailoverError");
    }
    expect(lastError).toBe(message);
    expect(lastErrorReason).toBe("model_not_found");
    expect(history.errorReason).toBe("model_not_found");
  });

  it.each([
    ["direct abort", createAgentRunDirectAbortError],
    ["gateway restart", createAgentRunRestartAbortError],
    ["superseded run", createAgentRunSupersededAbortError],
    ["stale gateway lifecycle", createAgentRunStaleLifecycleError],
  ])("persists the coded %s reason instead of reporting a timeout", async (name, createError) => {
    const modelRef = { provider: "openai", model: "gpt-5.4" };
    resolveConfiguredModelRefMock.mockReturnValue(modelRef);
    const rejection = createError() as Error & { code: string };
    runWithModelFallbackMock.mockRejectedValueOnce(rejection);

    const { finished, history, lastError } = await runPersistedDiagnosticCase({
      cfg: configFor(modelRef),
      modelRef,
      name,
    });
    const expected = `${rejection.message} | ${rejection.code}`;

    for (const outcome of [finished, history]) {
      expect(outcome).toMatchObject({
        status: "error",
        error: expected,
      });
      expect(outcome.error).not.toContain("timed out");
    }
    expect(lastError).toBe(expected);
  });

  it("persists and emits a fatal execution-denial diagnostic", async () => {
    const modelRef = { provider: "openai", model: "gpt-5.4" };
    resolveConfiguredModelRefMock.mockReturnValue(modelRef);
    mockRunCronFallbackPassthrough();
    runEmbeddedAgentMock.mockResolvedValueOnce({
      payloads: [],
      meta: {
        agentMeta: {},
        failureSignal: {
          kind: "execution_denied",
          source: "tool",
          toolName: "exec",
          code: "SYSTEM_RUN_DENIED",
          message: "SYSTEM_RUN_DENIED: approval required",
          fatalForCron: true,
        },
      },
    });

    const { finished, history } = await runPersistedDiagnosticCase({
      cfg: configFor(modelRef),
      modelRef,
      name: "execution denied",
    });

    for (const outcome of [finished, history]) {
      expect(outcome).toMatchObject({
        status: "error",
        provider: "openai",
        model: "gpt-5.4",
        error: "SYSTEM_RUN_DENIED: approval required",
        summary: "SYSTEM_RUN_DENIED: approval required",
        diagnostics: {
          entries: expect.arrayContaining([
            expect.objectContaining({
              source: "tool",
              severity: "error",
              message: "SYSTEM_RUN_DENIED: approval required",
            }),
          ]),
        },
      });
    }
  });

  it("persists and emits terminal tool detail while keeping the payload generic", async () => {
    const modelRef = { provider: "openai", model: "gpt-5.4" };
    resolveConfiguredModelRefMock.mockReturnValue(modelRef);
    mockRunCronFallbackPassthrough();
    runEmbeddedAgentMock.mockResolvedValueOnce({
      payloads: [{ text: "⚠️ Exec failed", isError: true, toolName: "exec" }],
      meta: {
        agentMeta: {},
        terminalToolFailure: {
          source: "tool",
          toolName: "exec",
          code: "UNKNOWN_TOOL_ID",
        },
      },
    });

    const { finished, history } = await runPersistedDiagnosticCase({
      cfg: configFor(modelRef),
      modelRef,
      name: "unknown tool id",
    });

    expect(dispatchCronDeliveryMock).toHaveBeenCalledWith(
      expect.objectContaining({
        deliveryPayloads: [{ text: "cron isolated run returned an error payload", isError: true }],
        outputText: "cron isolated run returned an error payload",
        summary: "Code Mode could not resolve a configured MCP tool.",
      }),
    );

    for (const outcome of [finished, history]) {
      expect(outcome).toMatchObject({
        status: "error",
        provider: "openai",
        model: "gpt-5.4",
        summary: "Code Mode could not resolve a configured MCP tool.",
        diagnostics: {
          summary: "Code Mode could not resolve a configured MCP tool.",
          entries: expect.arrayContaining([
            expect.objectContaining({
              source: "tool",
              severity: "error",
              message: "Code Mode could not resolve a configured MCP tool.",
              toolName: "exec",
            }),
          ]),
        },
      });
    }
  });
});