src/media-understanding/image.runtime-timeout.test.ts

// Image runtime tests cover model-backed image routing, auth/profile handling,
// provider payload transforms, and MiniMax/Copilot special paths.
import { expectDefined } from "@openclaw/normalization-core/expect";
import { MAX_TIMER_TIMEOUT_MS } from "@openclaw/normalization-core/number-coercion";
import { describe, expect, it, vi } from "vitest";
import { createDeferred } from "../../test/helpers/promise.js";
import { createEmptyPluginMetadataSnapshot } from "../agents/test-helpers/embedded-agent-runner-e2e-mocks.js";
import { AsyncWorkScope } from "../shared/async-work-scope.js";
import {
  SET_RUNTIME_API_KEY_FIELD,
  imageRuntimeMocks,
  installImageRuntimeTestHooks,
  preparedAuthStorage,
} from "./image.test-support.js";

const {
  completeMock,
  acquireAgentRunPreparedModelRuntimeMock,
  shouldPreferProviderRuntimeResolvedModelMock,
  getApiKeyForModelMock,
  prepareProviderRuntimeAuthMock,
  setRuntimeApiKeyMock,
  discoverModelsMock,
  releasePreparedModelRuntimeMock,
  resolveModelAsyncMock,
  resolveModelWithRegistryMock,
} = imageRuntimeMocks;

const { describeImageWithModelCore } = await import("./image.js");
const imageModelRuntime = await import("./image-model-runtime.js");

describe("describeImageWithModelCore", () => {
  installImageRuntimeTestHooks();

  it("reports the resolved model input when an image model is text-only", async () => {
    discoverModelsMock.mockReturnValue({
      find: vi.fn(() => ({
        provider: "lmstudio",
        id: "text-only",
        api: "openai-completions",
        input: ["text"],
        baseUrl: "http://127.0.0.1:1234",
      })),
    });

    await expect(
      describeImageWithModelCore({
        cfg: {},
        agentDir: "/tmp/openclaw-agent",
        provider: "lmstudio",
        model: "text-only",
        buffer: Buffer.from("png-bytes"),
        fileName: "image.png",
        mime: "image/png",
        prompt: "Describe the image.",
        timeoutMs: 1000,
      }),
    ).rejects.toThrow(
      "Model does not support images: lmstudio/text-only (resolved lmstudio/text-only input: text)",
    );
    expect(completeMock).not.toHaveBeenCalled();
  });

  it("passes image prompt as system instructions for codex image requests", async () => {
    discoverModelsMock.mockReturnValue({
      find: vi.fn(() => ({
        provider: "openai",
        id: "gpt-5.4",
        input: ["text", "image"],
        baseUrl: "https://chatgpt.com/backend-api",
      })),
    });
    completeMock.mockResolvedValue({
      role: "assistant",
      api: "openai-chatgpt-responses",
      provider: "openai",
      model: "gpt-5.4",
      stopReason: "stop",
      timestamp: Date.now(),
      content: [{ type: "text", text: "codex ok" }],
    });

    const result = await describeImageWithModelCore({
      cfg: {},
      agentDir: "/tmp/openclaw-agent",
      provider: "openai",
      model: "gpt-5.4",
      buffer: Buffer.from("png-bytes"),
      fileName: "image.png",
      mime: "image/png",
      prompt: "Describe the image.",
      timeoutMs: 1000,
    });

    expect(result).toEqual({
      text: "codex ok",
      model: "gpt-5.4",
    });
    expect(completeMock).toHaveBeenCalledOnce();
    const firstCall = expectDefined(completeMock.mock.calls[0], "image completion call 0");
    const [completionModel, context, options] = firstCall;
    expect(completionModel).toEqual(
      expect.objectContaining({
        provider: "openai",
        id: "gpt-5.4",
        input: ["text", "image"],
        baseUrl: "https://chatgpt.com/backend-api",
      }),
    );
    expect(context.systemPrompt).toBe("Describe the image.");
    expect(context.messages).toHaveLength(1);
    expect(Object.keys(options).toSorted()).toEqual(["apiKey", "maxTokens", "signal", "timeoutMs"]);
    expect(options.apiKey).toBe("test-token");
    expect(options.maxTokens).toBe(4096);
    expect(options.signal).toBeInstanceOf(AbortSignal);
    expect(options.timeoutMs).toBeGreaterThan(0);
    expect(options.timeoutMs).toBeLessThanOrEqual(1000);
    const userMessage = context.messages[0];
    if (!userMessage) {
      throw new Error("expected image completion user message");
    }
    expect(userMessage.role).toBe("user");
    expect(userMessage.content).toHaveLength(1);
    expect(userMessage.content[0]).toEqual({
      type: "image",
      data: Buffer.from("png-bytes").toString("base64"),
      mimeType: "image/png",
    });
  });

  it("clamps oversized image description timeouts before scheduling", async () => {
    const setTimeoutSpy = vi.spyOn(globalThis, "setTimeout");
    discoverModelsMock.mockReturnValue({
      find: vi.fn(() => ({
        provider: "openai",
        id: "gpt-5.4",
        input: ["text", "image"],
        baseUrl: "https://chatgpt.com/backend-api",
      })),
    });
    completeMock.mockResolvedValue({
      role: "assistant",
      api: "openai-chatgpt-responses",
      provider: "openai",
      model: "gpt-5.4",
      stopReason: "stop",
      timestamp: Date.now(),
      content: [{ type: "text", text: "codex ok" }],
    });

    const result = await describeImageWithModelCore({
      cfg: {},
      agentDir: "/tmp/openclaw-agent",
      provider: "openai",
      model: "gpt-5.4",
      buffer: Buffer.from("png-bytes"),
      fileName: "image.png",
      mime: "image/png",
      prompt: "Describe the image.",
      timeoutMs: Number.MAX_SAFE_INTEGER,
    });

    expect(result).toEqual({
      text: "codex ok",
      model: "gpt-5.4",
    });
    expect(setTimeoutSpy).toHaveBeenCalledWith(expect.any(Function), MAX_TIMER_TIMEOUT_MS);
    const firstCall = expectDefined(completeMock.mock.calls[0], "image completion call 0");
    expect(firstCall[2].timeoutMs).toBe(MAX_TIMER_TIMEOUT_MS);
  });

  it("places OpenRouter image prompts in user content before images", async () => {
    discoverModelsMock.mockReturnValue({
      find: vi.fn(() => ({
        api: "openai-completions",
        provider: "openrouter",
        id: "google/gemini-2.5-flash",
        input: ["text", "image"],
        baseUrl: "https://openrouter.ai/api/v1",
      })),
    });
    completeMock.mockResolvedValue({
      role: "assistant",
      api: "openai-completions",
      provider: "openrouter",
      model: "google/gemini-2.5-flash",
      stopReason: "stop",
      timestamp: Date.now(),
      content: [{ type: "text", text: "openrouter ok" }],
    });

    const result = await describeImageWithModelCore({
      cfg: {},
      agentDir: "/tmp/openclaw-agent",
      provider: "openrouter",
      model: "google/gemini-2.5-flash",
      buffer: Buffer.from("png-bytes"),
      fileName: "image.png",
      mime: "image/png",
      prompt: "Describe the image.",
      timeoutMs: 1000,
    });

    expect(result).toEqual({
      text: "openrouter ok",
      model: "google/gemini-2.5-flash",
    });
    const firstCall = expectDefined(
      completeMock.mock.calls[0],
      "OpenRouter image completion call 0",
    );
    const [, context] = firstCall;
    expect(context.systemPrompt).toBeUndefined();
    const userMessage = context.messages[0];
    if (!userMessage) {
      throw new Error("expected OpenRouter image completion user message");
    }
    expect(userMessage.content).toEqual([
      { type: "text", text: "Describe the image." },
      {
        type: "image",
        data: Buffer.from("png-bytes").toString("base64"),
        mimeType: "image/png",
      },
    ]);
  });

  it("places DashScope image prompts in user content before images", async () => {
    discoverModelsMock.mockReturnValue({
      find: vi.fn(() => ({
        api: "openai-completions",
        provider: "qwen",
        id: "qwen3.6-plus",
        input: ["text", "image"],
        baseUrl: "https://dashscope.aliyuncs.com/compatible-mode/v1",
      })),
    });
    completeMock.mockResolvedValue({
      role: "assistant",
      api: "openai-completions",
      provider: "qwen",
      model: "qwen3.6-plus",
      stopReason: "stop",
      timestamp: Date.now(),
      content: [{ type: "text", text: "dashscope ok" }],
    });

    const result = await describeImageWithModelCore({
      cfg: {},
      agentDir: "/tmp/openclaw-agent",
      provider: "qwen",
      model: "qwen3.6-plus",
      buffer: Buffer.from("png-bytes"),
      fileName: "image.png",
      mime: "image/png",
      prompt: "Describe the image.",
      timeoutMs: 1000,
    });

    expect(result).toEqual({
      text: "dashscope ok",
      model: "qwen3.6-plus",
    });
    const firstCall = expectDefined(
      completeMock.mock.calls[0],
      "DashScope image completion call 0",
    );
    const [, context] = firstCall;
    expect(context.systemPrompt).toBeUndefined();
    const userMessage = context.messages[0];
    if (!userMessage) {
      throw new Error("expected DashScope image completion user message");
    }
    expect(userMessage.content).toEqual([
      { type: "text", text: "Describe the image." },
      {
        type: "image",
        data: Buffer.from("png-bytes").toString("base64"),
        mimeType: "image/png",
      },
    ]);
  });

  it.each([
    {
      name: "direct OpenAI Responses baseUrl",
      provider: "openai",
      model: {
        api: "openai-responses",
        provider: "openai",
        id: "gpt-5.4-mini",
        input: ["text", "image"],
        baseUrl: "https://api.openai.com/v1",
      },
      expectedRetryPayload: {
        reasoning: { effort: "none" },
      },
    },
    {
      name: "default OpenAI Responses route without explicit baseUrl",
      provider: "openai",
      model: {
        api: "openai-responses",
        provider: "openai",
        id: "gpt-5.4-mini",
        input: ["text", "image"],
      },
      expectedRetryPayload: {
        reasoning: { effort: "none" },
      },
    },
    {
      name: "azure-openai provider using openai-responses api",
      provider: "azure-openai",
      model: {
        api: "openai-responses",
        provider: "azure-openai",
        id: "gpt-5.4-mini",
        input: ["text", "image"],
        baseUrl: "https://myresource.openai.azure.com/openai/v1",
      },
      expectedRetryPayload: {
        reasoning: { effort: "none" },
      },
    },
    {
      name: "proxy-like openai-responses route",
      provider: "openai",
      model: {
        api: "openai-responses",
        provider: "openai",
        id: "gpt-5.4-mini",
        input: ["text", "image"],
        baseUrl: "https://proxy.example.com/v1",
      },
      expectedRetryPayload: {},
    },
  ])(
    "retries reasoning-only image responses with reasoning disabled for $name",
    async ({ provider, model, expectedRetryPayload }) => {
      discoverModelsMock.mockReturnValue({
        find: vi.fn(() => model),
      });
      completeMock
        .mockResolvedValueOnce({
          role: "assistant",
          api: model.api,
          provider: model.provider,
          model: model.id,
          stopReason: "stop",
          timestamp: Date.now(),
          content: [
            {
              type: "thinking",
              thinking: "internal image reasoning",
              thinkingSignature: "reasoning_content",
            },
          ],
        })
        .mockResolvedValueOnce({
          role: "assistant",
          api: model.api,
          provider: model.provider,
          model: model.id,
          stopReason: "stop",
          timestamp: Date.now(),
          content: [{ type: "text", text: "retry ok" }],
        });

      const result = await describeImageWithModelCore({
        cfg: {},
        agentDir: "/tmp/openclaw-agent",
        provider,
        model: model.id,
        buffer: Buffer.from("png-bytes"),
        fileName: "image.png",
        mime: "image/png",
        prompt: "Describe the image.",
        timeoutMs: 1000,
      });

      expect(result).toEqual({
        text: "retry ok",
        model: model.id,
      });
      expect(completeMock).toHaveBeenCalledTimes(2);
      const retryCall = expectDefined(completeMock.mock.calls[1], "retry image completion call 1");
      const [retryModel, , retryOptions] = retryCall;
      if (!retryOptions?.onPayload) {
        throw new Error("expected retry payload mapper");
      }
      const retryPayload = await retryOptions.onPayload(
        {
          reasoning: { effort: "high", summary: "auto" },
          reasoning_effort: "high",
          include: ["reasoning.encrypted_content"],
        },
        retryModel,
      );
      expect(retryPayload).toEqual(expectedRetryPayload);
    },
  );

  it("does not start the reasoning-only retry after caller cancellation", async () => {
    const controller = new AbortController();
    discoverModelsMock.mockReturnValue({
      find: vi.fn(() => ({
        api: "openai-responses",
        provider: "openai",
        id: "gpt-5.4-mini",
        input: ["text", "image"],
        baseUrl: "https://api.openai.com/v1",
      })),
    });
    completeMock.mockImplementationOnce(async () => {
      controller.abort(new Error("caller cancelled image description"));
      return {
        role: "assistant",
        api: "openai-responses",
        provider: "openai",
        model: "gpt-5.4-mini",
        stopReason: "stop",
        timestamp: Date.now(),
        content: [{ type: "thinking", thinking: "internal", thinkingSignature: "reasoning" }],
      };
    });

    await expect(
      describeImageWithModelCore({
        cfg: {},
        agentDir: "/tmp/openclaw-agent",
        provider: "openai",
        model: "gpt-5.4-mini",
        buffer: Buffer.from("png-bytes"),
        fileName: "image.png",
        mime: "image/png",
        prompt: "Describe the image.",
        timeoutMs: 1000,
        signal: controller.signal,
      }),
    ).rejects.toThrow("caller cancelled image description");

    expect(completeMock).toHaveBeenCalledOnce();
    const options = expectDefined(
      completeMock.mock.calls[0],
      "cancelled image completion call 0",
    )[2];
    expect(options?.signal?.aborted).toBe(true);
  });

  it("rejects when a generic image completion ignores the abort signal", async () => {
    vi.useFakeTimers();
    discoverModelsMock.mockReturnValue({
      find: vi.fn(() => ({
        api: "openai-responses",
        provider: "openai",
        id: "gpt-5.4-mini",
        input: ["text", "image"],
        baseUrl: "https://api.openai.com/v1",
      })),
    });
    completeMock.mockImplementation(() => new Promise(() => {}));

    const result = describeImageWithModelCore({
      cfg: {},
      agentDir: "/tmp/openclaw-agent",
      provider: "openai",
      model: "gpt-5.4-mini",
      buffer: Buffer.from("png-bytes"),
      fileName: "image.png",
      mime: "image/png",
      prompt: "Describe the image.",
      timeoutMs: 25,
    });

    const assertion = expect(result).rejects.toThrow(
      "image description request timed out after 25ms",
    );
    await vi.advanceTimersByTimeAsync(25);
    await assertion;
    const firstCall = expectDefined(completeMock.mock.calls[0], "timed image completion call 0");
    const options = firstCall[2];
    if (!options?.signal) {
      throw new Error("Expected image completion abort signal");
    }
    expect(options.signal.aborted).toBe(true);
    expect(options.timeoutMs).toBe(25);
  });

  it("retains the prepared runtime until an aborted provider actually settles", async () => {
    discoverModelsMock.mockReturnValue({
      find: vi.fn(() => ({
        api: "openai-responses",
        provider: "openai",
        id: "gpt-5.4-mini",
        input: ["text", "image"],
        baseUrl: "https://api.openai.com/v1",
      })),
    });
    const completion = createDeferred();
    completeMock.mockImplementation(async () => {
      await completion.promise;
      throw new Error("late provider failure");
    });
    const controller = new AbortController();
    const result = describeImageWithModelCore({
      cfg: {},
      agentDir: "/tmp/openclaw-agent",
      provider: "openai",
      model: "gpt-5.4-mini",
      buffer: Buffer.from("png-bytes"),
      fileName: "image.png",
      mime: "image/png",
      prompt: "Describe the image.",
      timeoutMs: 60_000,
      signal: controller.signal,
    });

    try {
      await vi.waitFor(() => expect(completeMock).toHaveBeenCalledOnce());
      const assertion = expect(result).rejects.toThrow("caller cancelled provider request");
      controller.abort(new Error("caller cancelled provider request"));
      await assertion;

      expect(releasePreparedModelRuntimeMock).not.toHaveBeenCalled();
    } finally {
      completion.resolve();
    }
    await vi.waitFor(() => expect(releasePreparedModelRuntimeMock).toHaveBeenCalledOnce());
  });

  it("keeps the full configured timeout for provider requests after slow setup", async () => {
    vi.useFakeTimers();
    const slowSetupMs = 400;
    discoverModelsMock.mockReturnValue({
      find: vi.fn(() => ({
        api: "openai-responses",
        provider: "openai",
        id: "gpt-5.4-mini",
        input: ["text", "image"],
        baseUrl: "https://api.openai.com/v1",
      })),
    });
    resolveModelAsyncMock.mockImplementationOnce(
      async (provider: string, modelId: string, agentDir?: string, cfg?: unknown) => {
        await new Promise<void>((resolve) => {
          setTimeout(resolve, slowSetupMs);
        });
        const authStorage = {
          [SET_RUNTIME_API_KEY_FIELD]: setRuntimeApiKeyMock,
        };
        const modelRegistry = discoverModelsMock(authStorage, agentDir);
        const model = resolveModelWithRegistryMock({
          provider,
          modelId,
          modelRegistry,
          cfg,
          agentDir,
        });
        return { authStorage, model, modelRegistry };
      },
    );
    completeMock.mockImplementation(() => new Promise(() => {}));

    const result = describeImageWithModelCore({
      cfg: {},
      agentDir: "/tmp/openclaw-agent",
      provider: "openai",
      model: "gpt-5.4-mini",
      buffer: Buffer.from("png-bytes"),
      fileName: "image.png",
      mime: "image/png",
      prompt: "Describe the image.",
      timeoutMs: 1000,
    });

    await vi.advanceTimersByTimeAsync(slowSetupMs);
    await Promise.resolve();
    expect(completeMock).toHaveBeenCalledTimes(1);
    const firstCall = expectDefined(
      completeMock.mock.calls[0],
      "slow setup image completion call 0",
    );
    const options = firstCall[2];
    if (!options?.signal) {
      throw new Error("Expected image completion abort signal");
    }
    expect(options.timeoutMs).toBe(1000);

    const assertion = expect(result).rejects.toThrow(
      `image description request timed out after 1000ms (setup took ${slowSetupMs}ms before provider request started)`,
    );
    await vi.advanceTimersByTimeAsync(1000);
    await assertion;
    expect(options.signal.aborted).toBe(true);
  });

  it("rejects when image runtime setup exceeds the request timeout", async () => {
    vi.useFakeTimers();
    resolveModelAsyncMock.mockImplementationOnce(() => new Promise(() => {}));

    const result = describeImageWithModelCore({
      cfg: {},
      agentDir: "/tmp/openclaw-agent",
      provider: "openai",
      model: "gpt-5.4-mini",
      buffer: Buffer.from("png-bytes"),
      fileName: "image.png",
      mime: "image/png",
      prompt: "Describe the image.",
      timeoutMs: 25,
    });

    const assertion = expect(result).rejects.toThrow(
      "image description setup timed out after 25ms before provider request started",
    );
    await vi.advanceTimersByTimeAsync(25);
    await assertion;
    expect(completeMock).not.toHaveBeenCalled();
  });

  it.each(
    (["timeout", "cancellation"] as const).flatMap((mode) =>
      (["admission", "model", "credential", "credential-model", "runtime-auth"] as const).flatMap(
        (stage) => [false, true].map((cleanupFails) => ({ mode, stage, cleanupFails })),
      ),
    ),
  )(
    "stops image setup after $mode during $stage (cleanup failure: $cleanupFails)",
    async ({ mode, stage, cleanupFails }) => {
      vi.useFakeTimers();
      const resolution = vi.spyOn(imageModelRuntime, "resolveImageRuntime");
      const cleanupError = new Error("late image runtime disposal failed");
      const cleanupStarted = createDeferred();
      const finishCleanup = createDeferred();
      releasePreparedModelRuntimeMock.mockImplementationOnce(async () => {
        cleanupStarted.resolve();
        await finishCleanup.promise;
        if (cleanupFails) {
          throw cleanupError;
        }
      });
      const started = createDeferred();
      const finish = createDeferred();
      const delay = async <T>(value: T): Promise<T> => {
        started.resolve();
        await finish.promise;
        return value;
      };
      const resolved = {
        authStorage: preparedAuthStorage,
        model: {
          provider: "openai",
          id: "gpt-5.4-mini",
          api: "openai-responses",
          input: ["text", "image"],
        },
        modelRegistry: {},
      };
      resolveModelAsyncMock.mockResolvedValue(resolved);
      shouldPreferProviderRuntimeResolvedModelMock.mockReturnValue(stage === "credential-model");
      if (stage === "admission") {
        acquireAgentRunPreparedModelRuntimeMock.mockImplementationOnce(() =>
          delay({
            snapshot: {
              agentDir: "/tmp/openclaw-agent",
              config: {},
              metadataSnapshot: createEmptyPluginMetadataSnapshot(),
              createStores: () => ({ authStorage: preparedAuthStorage, modelRegistry: {} }),
            },
            [Symbol.asyncDispose]: releasePreparedModelRuntimeMock,
          }),
        );
      } else if (stage === "model") {
        resolveModelAsyncMock.mockImplementationOnce(() => delay(resolved));
      } else if (stage === "credential") {
        getApiKeyForModelMock.mockImplementationOnce(() =>
          delay({ apiKey: "test-token", source: "test", mode: "oauth" }),
        );
      } else if (stage === "credential-model") {
        resolveModelAsyncMock
          .mockResolvedValueOnce(resolved)
          .mockImplementationOnce(() => delay(resolved));
      } else {
        prepareProviderRuntimeAuthMock.mockImplementationOnce(() =>
          delay({ apiKey: "prepared-test-token" }),
        );
      }
      const controller = new AbortController();
      const work = new AsyncWorkScope();
      const pending = work.track(() =>
        describeImageWithModelCore({
          cfg: {},
          agentDir: "/tmp/openclaw-agent",
          provider: "openai",
          model: "gpt-5.4-mini",
          buffer: Buffer.from("png-bytes"),
          fileName: "image.png",
          mime: "image/png",
          prompt: "Describe the image.",
          timeoutMs: 25,
          signal: controller.signal,
        }),
      );
      const rejected = expect(pending).rejects.toThrow(
        mode === "timeout"
          ? "image description setup timed out after 25ms before provider request started"
          : "caller cancelled during setup",
      );
      await started.promise;
      if (mode === "timeout") {
        await vi.advanceTimersByTimeAsync(25);
      } else {
        controller.abort(new Error("caller cancelled during setup"));
      }
      await rejected;
      expect(releasePreparedModelRuntimeMock).not.toHaveBeenCalled();
      const setup = resolution.mock.results[0]?.value;
      const producerFailure = expect(setup).rejects.toMatchObject({
        name: mode === "timeout" ? "AbortError" : "Error",
      });
      let drained = false;
      let drain: Promise<void> | undefined;
      try {
        finish.resolve();
        await producerFailure;
        await cleanupStarted.promise;
        drain = work.drain().then(() => {
          drained = true;
        });
        await Promise.resolve();
        expect(drained).toBe(false);
        expect(releasePreparedModelRuntimeMock).toHaveBeenCalledOnce();
        finishCleanup.resolve();
        await drain;
        expect(drained).toBe(true);
        const disposal = releasePreparedModelRuntimeMock.mock.results[0]!.value;
        if (cleanupFails) {
          await expect(disposal).rejects.toBe(cleanupError);
        } else {
          await expect(disposal).resolves.toBeUndefined();
        }
      } finally {
        finish.resolve();
        finishCleanup.resolve();
        await drain;
        await work.drain();
      }
      expect(resolveModelAsyncMock).toHaveBeenCalledTimes(
        stage === "admission" ? 0 : stage === "credential-model" ? 2 : 1,
      );
      expect(getApiKeyForModelMock).toHaveBeenCalledTimes(
        stage === "admission" || stage === "model" ? 0 : 1,
      );
      expect(prepareProviderRuntimeAuthMock).toHaveBeenCalledTimes(
        stage === "runtime-auth" ? 1 : 0,
      );
      expect(setRuntimeApiKeyMock).not.toHaveBeenCalled();
      expect(completeMock).not.toHaveBeenCalled();
    },
  );
});