| import { createServer, type Server, type AddressInfo } from "node:net"; |
| import { afterAll, beforeEach, describe, expect, it, vi } from "vitest"; |
| import { FailoverError } from "../agents/failover-error.js"; |
| import { |
| createAgentRunDirectAbortError, |
| createAgentRunRestartAbortError, |
| createAgentRunSupersededAbortError, |
| } from "../agents/run-termination.js"; |
| import type { OpenClawConfig } from "../config/config.js"; |
| import { createAgentRunStaleLifecycleError } from "../infra/agent-lifecycle-error.js"; |
| import { resetTaskRegistryForTests } from "../tasks/task-runtime.test-helpers.js"; |
| import { withOpenClawTestState } from "../test-utils/openclaw-test-state.js"; |
| import { |
| loadRunCronIsolatedAgentTurn, |
| dispatchCronDeliveryMock, |
| mockRunCronFallbackPassthrough, |
| resetRunCronIsolatedAgentTurnHarness, |
| resolveAllowedModelRefMock, |
| resolveConfiguredModelRefMock, |
| runEmbeddedAgentMock, |
| runWithModelFallbackMock, |
| } from "./isolated-agent/run.test-harness.js"; |
| import { CronService, type CronEvent } from "./service.js"; |
| import { createNoopLogger } from "./service.test-harness.js"; |
| import { cronStoreKey } from "./store/key.js"; |
| import { readCronTaskRunHistoryPage } from "./task-run-history.js"; |
|
|
| vi.doUnmock("./isolated-agent/model-preflight.runtime.js"); |
|
|
| const runCronIsolatedAgentTurn = await loadRunCronIsolatedAgentTurn(); |
| const { resetCronModelProviderPreflightCacheForTest } = |
| await import("./isolated-agent/model-preflight.runtime.js"); |
|
|
| type ModelRef = { provider: string; model: string }; |
|
|
| function makeCommandLaneTaskTimeoutError(lane: string, timeoutMs: number): Error { |
| const error = new Error(`Command lane "${lane}" task timed out after ${timeoutMs}ms`); |
| error.name = "CommandLaneTaskTimeoutError"; |
| return error; |
| } |
|
|
| async function listenForBrokenHttpConnections(): Promise<{ |
| baseUrl: string; |
| server: Server; |
| }> { |
| const server = createServer((socket) => socket.destroy()); |
| await new Promise<void>((resolve, reject) => { |
| server.once("error", reject); |
| server.listen(0, "127.0.0.1", resolve); |
| }); |
| const address = server.address() as AddressInfo; |
| return { baseUrl: `http://127.0.0.1:${address.port}`, server }; |
| } |
|
|
| async function closeServer(server: Server): Promise<void> { |
| await new Promise<void>((resolve, reject) => { |
| server.close((error) => (error ? reject(error) : resolve())); |
| }); |
| } |
|
|
| function configFor(modelRef: ModelRef, localBaseUrl?: string): OpenClawConfig { |
| return { |
| agents: { |
| defaults: { |
| model: `${modelRef.provider}/${modelRef.model}`, |
| }, |
| }, |
| ...(localBaseUrl |
| ? { |
| models: { |
| providers: { |
| [modelRef.provider]: { |
| api: "ollama", |
| baseUrl: localBaseUrl, |
| models: [], |
| }, |
| }, |
| }, |
| } |
| : {}), |
| }; |
| } |
|
|
| async function runPersistedDiagnosticCase(params: { |
| cfg: OpenClawConfig; |
| modelRef: ModelRef; |
| name: string; |
| }) { |
| return await withOpenClawTestState( |
| { layout: "state-only", prefix: "openclaw-cron-execution-diagnostics-" }, |
| async (state) => { |
| resetTaskRegistryForTests(); |
| const events: CronEvent[] = []; |
| const storePath = state.path("cron", "jobs.json"); |
| const cron = new CronService({ |
| storePath, |
| cronEnabled: true, |
| cronConfig: { triggers: { enabled: true } }, |
| log: createNoopLogger(), |
| enqueueSystemEvent: vi.fn(), |
| requestHeartbeat: vi.fn(), |
| onEvent: (event) => events.push(structuredClone(event)), |
| runIsolatedAgentJob: async (runParams) => |
| await runCronIsolatedAgentTurn({ |
| ...runParams, |
| cfg: params.cfg, |
| deps: {}, |
| sessionKey: `cron:${runParams.job.id}`, |
| }), |
| }); |
| try { |
| await cron.start(); |
| const job = await cron.add({ |
| name: params.name, |
| enabled: true, |
| schedule: { kind: "every", everyMs: 60_000 }, |
| sessionTarget: "isolated", |
| wakeMode: "next-heartbeat", |
| payload: { |
| kind: "agentTurn", |
| message: `run ${params.name}`, |
| model: `${params.modelRef.provider}/${params.modelRef.model}`, |
| }, |
| delivery: { mode: "none" }, |
| }); |
| await expect(cron.run(job.id, "force")).resolves.toEqual({ ok: true, ran: true }); |
|
|
| const finished = events.find( |
| (event) => event.action === "finished" && event.jobId === job.id, |
| ); |
| const history = readCronTaskRunHistoryPage({ |
| storeKey: cronStoreKey(storePath), |
| jobId: job.id, |
| limit: 1, |
| }).entries[0]; |
| expect(finished).toBeDefined(); |
| expect(history).toBeDefined(); |
| return { |
| finished: finished!, |
| history: history!, |
| lastError: cron.getJob(job.id)?.state.lastError, |
| lastErrorReason: cron.getJob(job.id)?.state.lastErrorReason, |
| }; |
| } finally { |
| cron.stop(); |
| resetTaskRegistryForTests({ persist: false }); |
| } |
| }, |
| ); |
| } |
|
|
| describe("cron execution diagnostics", { concurrent: false }, () => { |
| const servers: Server[] = []; |
|
|
| beforeEach(() => { |
| resetRunCronIsolatedAgentTurnHarness(); |
| resetCronModelProviderPreflightCacheForTest(); |
| }); |
|
|
| afterAll(async () => { |
| await Promise.all(servers.map(closeServer)); |
| }); |
|
|
| it("persists and emits a real isolated-run local-provider preflight failure", async () => { |
| const endpoint = await listenForBrokenHttpConnections(); |
| servers.push(endpoint.server); |
| const modelRef = { provider: "ollama", model: "diagnostic-model" }; |
| resolveConfiguredModelRefMock.mockReturnValue(modelRef); |
| resolveAllowedModelRefMock.mockReturnValue({ ref: modelRef }); |
|
|
| const { finished, history } = await runPersistedDiagnosticCase({ |
| cfg: configFor(modelRef, endpoint.baseUrl), |
| modelRef, |
| name: "unreachable provider", |
| }); |
|
|
| expect(runWithModelFallbackMock).not.toHaveBeenCalled(); |
| expect(runEmbeddedAgentMock).not.toHaveBeenCalled(); |
| for (const outcome of [finished, history]) { |
| expect(outcome).toMatchObject({ |
| status: "skipped", |
| provider: "ollama", |
| model: "diagnostic-model", |
| error: expect.stringContaining("unavailable"), |
| diagnostics: { |
| entries: expect.arrayContaining([ |
| expect.objectContaining({ |
| source: "model-preflight", |
| severity: "warn", |
| message: expect.stringContaining("unavailable"), |
| }), |
| ]), |
| }, |
| }); |
| } |
| }); |
|
|
| it("persists and emits the canonical terminal timeout diagnostic", async () => { |
| const modelRef = { provider: "openai", model: "gpt-5.4" }; |
| resolveConfiguredModelRefMock.mockReturnValue(modelRef); |
| runWithModelFallbackMock.mockRejectedValueOnce( |
| makeCommandLaneTaskTimeoutError("cron-nested", 330_000), |
| ); |
|
|
| const { finished, history } = await runPersistedDiagnosticCase({ |
| cfg: configFor(modelRef), |
| modelRef, |
| name: "nested lane timeout", |
| }); |
|
|
| for (const outcome of [finished, history]) { |
| expect(outcome).toMatchObject({ |
| status: "error", |
| provider: "openai", |
| model: "gpt-5.4", |
| error: "cron: job execution timed out", |
| diagnostics: { |
| entries: expect.arrayContaining([ |
| expect.objectContaining({ |
| source: "cron-setup", |
| severity: "error", |
| message: "cron: job execution timed out", |
| }), |
| ]), |
| }, |
| }); |
| } |
| }); |
|
|
| it("persists provider failures without internal class names", async () => { |
| const message = "Saved selection requires an update."; |
| const modelRef = { provider: "openai", model: "not-a-real-model" }; |
| resolveConfiguredModelRefMock.mockReturnValue(modelRef); |
| resolveAllowedModelRefMock.mockReturnValue({ ref: modelRef }); |
| runWithModelFallbackMock.mockRejectedValueOnce( |
| new FailoverError(message, { |
| reason: "model_not_found", |
| provider: modelRef.provider, |
| model: modelRef.model, |
| }), |
| ); |
|
|
| const { finished, history, lastError, lastErrorReason } = await runPersistedDiagnosticCase({ |
| cfg: configFor(modelRef), |
| modelRef, |
| name: "missing provider model", |
| }); |
|
|
| for (const outcome of [finished, history]) { |
| expect(outcome).toMatchObject({ |
| status: "error", |
| provider: modelRef.provider, |
| model: modelRef.model, |
| error: message, |
| diagnostics: { summary: message }, |
| }); |
| expect(outcome.error).not.toContain("FailoverError"); |
| } |
| expect(lastError).toBe(message); |
| expect(lastErrorReason).toBe("model_not_found"); |
| expect(history.errorReason).toBe("model_not_found"); |
| }); |
|
|
| it.each([ |
| ["direct abort", createAgentRunDirectAbortError], |
| ["gateway restart", createAgentRunRestartAbortError], |
| ["superseded run", createAgentRunSupersededAbortError], |
| ["stale gateway lifecycle", createAgentRunStaleLifecycleError], |
| ])("persists the coded %s reason instead of reporting a timeout", async (name, createError) => { |
| const modelRef = { provider: "openai", model: "gpt-5.4" }; |
| resolveConfiguredModelRefMock.mockReturnValue(modelRef); |
| const rejection = createError() as Error & { code: string }; |
| runWithModelFallbackMock.mockRejectedValueOnce(rejection); |
|
|
| const { finished, history, lastError } = await runPersistedDiagnosticCase({ |
| cfg: configFor(modelRef), |
| modelRef, |
| name, |
| }); |
| const expected = `${rejection.message} | ${rejection.code}`; |
|
|
| for (const outcome of [finished, history]) { |
| expect(outcome).toMatchObject({ |
| status: "error", |
| error: expected, |
| }); |
| expect(outcome.error).not.toContain("timed out"); |
| } |
| expect(lastError).toBe(expected); |
| }); |
|
|
| it("persists and emits a fatal execution-denial diagnostic", async () => { |
| const modelRef = { provider: "openai", model: "gpt-5.4" }; |
| resolveConfiguredModelRefMock.mockReturnValue(modelRef); |
| mockRunCronFallbackPassthrough(); |
| runEmbeddedAgentMock.mockResolvedValueOnce({ |
| payloads: [], |
| meta: { |
| agentMeta: {}, |
| failureSignal: { |
| kind: "execution_denied", |
| source: "tool", |
| toolName: "exec", |
| code: "SYSTEM_RUN_DENIED", |
| message: "SYSTEM_RUN_DENIED: approval required", |
| fatalForCron: true, |
| }, |
| }, |
| }); |
|
|
| const { finished, history } = await runPersistedDiagnosticCase({ |
| cfg: configFor(modelRef), |
| modelRef, |
| name: "execution denied", |
| }); |
|
|
| for (const outcome of [finished, history]) { |
| expect(outcome).toMatchObject({ |
| status: "error", |
| provider: "openai", |
| model: "gpt-5.4", |
| error: "SYSTEM_RUN_DENIED: approval required", |
| summary: "SYSTEM_RUN_DENIED: approval required", |
| diagnostics: { |
| entries: expect.arrayContaining([ |
| expect.objectContaining({ |
| source: "tool", |
| severity: "error", |
| message: "SYSTEM_RUN_DENIED: approval required", |
| }), |
| ]), |
| }, |
| }); |
| } |
| }); |
|
|
| it("persists and emits terminal tool detail while keeping the payload generic", async () => { |
| const modelRef = { provider: "openai", model: "gpt-5.4" }; |
| resolveConfiguredModelRefMock.mockReturnValue(modelRef); |
| mockRunCronFallbackPassthrough(); |
| runEmbeddedAgentMock.mockResolvedValueOnce({ |
| payloads: [{ text: "⚠️ Exec failed", isError: true, toolName: "exec" }], |
| meta: { |
| agentMeta: {}, |
| terminalToolFailure: { |
| source: "tool", |
| toolName: "exec", |
| code: "UNKNOWN_TOOL_ID", |
| }, |
| }, |
| }); |
|
|
| const { finished, history } = await runPersistedDiagnosticCase({ |
| cfg: configFor(modelRef), |
| modelRef, |
| name: "unknown tool id", |
| }); |
|
|
| expect(dispatchCronDeliveryMock).toHaveBeenCalledWith( |
| expect.objectContaining({ |
| deliveryPayloads: [{ text: "cron isolated run returned an error payload", isError: true }], |
| outputText: "cron isolated run returned an error payload", |
| summary: "Code Mode could not resolve a configured MCP tool.", |
| }), |
| ); |
|
|
| for (const outcome of [finished, history]) { |
| expect(outcome).toMatchObject({ |
| status: "error", |
| provider: "openai", |
| model: "gpt-5.4", |
| summary: "Code Mode could not resolve a configured MCP tool.", |
| diagnostics: { |
| summary: "Code Mode could not resolve a configured MCP tool.", |
| entries: expect.arrayContaining([ |
| expect.objectContaining({ |
| source: "tool", |
| severity: "error", |
| message: "Code Mode could not resolve a configured MCP tool.", |
| toolName: "exec", |
| }), |
| ]), |
| }, |
| }); |
| } |
| }); |
| }); |
|
|
