diff --git a/packages/ai/src/providers/anthropic.ts b/packages/ai/src/providers/anthropic.ts new file mode 100644 index 0000000000000000000000000000000000000000..675735396dacf42d712f776059458b39bec6a3ba --- /dev/null +++ b/packages/ai/src/providers/anthropic.ts @@ -0,0 +1,59 @@ +import { anthropicMessagesApi } from "../api/anthropic-messages.lazy.ts"; +import { lazyOAuth } from "../auth/helpers.ts"; +import { loadAnthropicOAuth } from "../auth/oauth/load.ts"; +import type { ApiKeyAuth } from "../auth/types.ts"; +import { ANTHROPIC_API_KEY_ENV, ANTHROPIC_AUTH_TOKEN_ENV, ANTHROPIC_OAUTH_TOKEN_ENV } from "../env-api-keys.ts"; +import { createProvider, type Provider } from "../models.ts"; +import { ANTHROPIC_MODELS } from "./anthropic.models.ts"; + +function anthropicApiKeyAuth(): ApiKeyAuth { + return { + name: "Anthropic API key", + login: async (interaction) => { + interaction.signal.throwIfAborted(); + const key = await interaction.prompt({ type: "secret", message: "Enter Anthropic API key" }); + interaction.signal.throwIfAborted(); + return { type: "api_key", key }; + }, + resolve: async ({ ctx, credential, signal }) => { + signal.throwIfAborted(); + if (credential?.key) { + return { auth: { apiKey: credential.key }, env: credential.env, source: "stored credential" }; + } + + const authToken = await ctx.env(ANTHROPIC_AUTH_TOKEN_ENV); + signal.throwIfAborted(); + if (authToken) { + return { + auth: { headers: { Authorization: `Bearer ${authToken}` } }, + source: ANTHROPIC_AUTH_TOKEN_ENV, + }; + } + + for (const envVar of [ANTHROPIC_OAUTH_TOKEN_ENV, ANTHROPIC_API_KEY_ENV]) { + const apiKey = await ctx.env(envVar); + signal.throwIfAborted(); + if (apiKey) return { auth: { apiKey }, source: envVar }; + } + return undefined; + }, + }; +} + +export function anthropicProvider(): Provider<"anthropic-messages"> { + return createProvider({ + id: "anthropic", + name: "Anthropic", + baseUrl: "https://api.anthropic.com", + auth: { + apiKey: anthropicApiKeyAuth(), + oauth: lazyOAuth({ + name: "Anthropic (Claude Pro/Max)", + isSubscription: true, + load: loadAnthropicOAuth, + }), + }, + models: Object.values(ANTHROPIC_MODELS), + api: anthropicMessagesApi(), + }); +} diff --git a/packages/ai/src/providers/baseten.ts b/packages/ai/src/providers/baseten.ts new file mode 100644 index 0000000000000000000000000000000000000000..79371dac0d4f9e00d21a3593dd6e78c6c201b475 --- /dev/null +++ b/packages/ai/src/providers/baseten.ts @@ -0,0 +1,15 @@ +import { openAICompletionsApi } from "../api/openai-completions.lazy.ts"; +import { envApiKeyAuth } from "../auth/helpers.ts"; +import { createProvider, type Provider } from "../models.ts"; +import { BASETEN_MODELS } from "./baseten.models.ts"; + +export function basetenProvider(): Provider<"openai-completions"> { + return createProvider({ + id: "baseten", + name: "Baseten", + baseUrl: "https://inference.baseten.co/v1", + auth: { apiKey: envApiKeyAuth("Baseten API key", ["BASETEN_API_KEY"]) }, + models: Object.values(BASETEN_MODELS), + api: openAICompletionsApi(), + }); +} diff --git a/packages/ai/src/providers/cerebras.models.ts b/packages/ai/src/providers/cerebras.models.ts new file mode 100644 index 0000000000000000000000000000000000000000..0ae1100797e395ffea19a7e53390b700d00b9036 --- /dev/null +++ b/packages/ai/src/providers/cerebras.models.ts @@ -0,0 +1,8 @@ +// This file is auto-generated by scripts/generate-models.ts +// Do not edit manually - run 'npm run generate-models' to update + +import values from "./data/cerebras.json" with { type: "json" }; +import { flattenModelCatalog, type ModelCatalog } from "../model-catalog.ts"; + +export const CEREBRAS_MODELS: ModelCatalog = + flattenModelCatalog("cerebras", values); diff --git a/packages/ai/src/providers/cloudflare-ai-gateway.models.ts b/packages/ai/src/providers/cloudflare-ai-gateway.models.ts new file mode 100644 index 0000000000000000000000000000000000000000..819bf7fb6219eeb1c18da7e04214da6dc72090bf --- /dev/null +++ b/packages/ai/src/providers/cloudflare-ai-gateway.models.ts @@ -0,0 +1,8 @@ +// This file is auto-generated by scripts/generate-models.ts +// Do not edit manually - run 'npm run generate-models' to update + +import values from "./data/cloudflare-ai-gateway.json" with { type: "json" }; +import { flattenModelCatalog, type ModelCatalog } from "../model-catalog.ts"; + +export const CLOUDFLARE_AI_GATEWAY_MODELS: ModelCatalog = + flattenModelCatalog("cloudflare-ai-gateway", values); diff --git a/packages/ai/src/providers/data-json.d.ts b/packages/ai/src/providers/data-json.d.ts new file mode 100644 index 0000000000000000000000000000000000000000..73212a8f9674a910d626722e94486a9000a48519 --- /dev/null +++ b/packages/ai/src/providers/data-json.d.ts @@ -0,0 +1,4 @@ +declare module "*.json" { + const value: unknown; + export default value; +} diff --git a/packages/ai/src/providers/deepseek.ts b/packages/ai/src/providers/deepseek.ts new file mode 100644 index 0000000000000000000000000000000000000000..580e25c236feb1a283d96bbef538915a142d8416 --- /dev/null +++ b/packages/ai/src/providers/deepseek.ts @@ -0,0 +1,15 @@ +import { openAICompletionsApi } from "../api/openai-completions.lazy.ts"; +import { envApiKeyAuth } from "../auth/helpers.ts"; +import { createProvider, type Provider } from "../models.ts"; +import { DEEPSEEK_MODELS } from "./deepseek.models.ts"; + +export function deepseekProvider(): Provider<"openai-completions"> { + return createProvider({ + id: "deepseek", + name: "DeepSeek", + baseUrl: "https://api.deepseek.com", + auth: { apiKey: envApiKeyAuth("DeepSeek API key", ["DEEPSEEK_API_KEY"]) }, + models: Object.values(DEEPSEEK_MODELS), + api: openAICompletionsApi(), + }); +} diff --git a/packages/ai/src/providers/fireworks.ts b/packages/ai/src/providers/fireworks.ts new file mode 100644 index 0000000000000000000000000000000000000000..518fb259d350a5bf5277d3cbe78d33c1558a1e74 --- /dev/null +++ b/packages/ai/src/providers/fireworks.ts @@ -0,0 +1,19 @@ +import { anthropicMessagesApi } from "../api/anthropic-messages.lazy.ts"; +import { openAICompletionsApi } from "../api/openai-completions.lazy.ts"; +import { envApiKeyAuth } from "../auth/helpers.ts"; +import { createProvider, type Provider } from "../models.ts"; +import { FIREWORKS_MODELS } from "./fireworks.models.ts"; + +export function fireworksProvider(): Provider<"anthropic-messages" | "openai-completions"> { + return createProvider({ + id: "fireworks", + name: "Fireworks", + baseUrl: "https://api.fireworks.ai/inference", + auth: { apiKey: envApiKeyAuth("Fireworks API key", ["FIREWORKS_API_KEY"]) }, + models: Object.values(FIREWORKS_MODELS), + api: { + "anthropic-messages": anthropicMessagesApi(), + "openai-completions": openAICompletionsApi(), + }, + }); +} diff --git a/packages/ai/src/providers/google-vertex.ts b/packages/ai/src/providers/google-vertex.ts new file mode 100644 index 0000000000000000000000000000000000000000..83c7d0fe89cbb7e425a3e79d2bf954e2c2b8324e --- /dev/null +++ b/packages/ai/src/providers/google-vertex.ts @@ -0,0 +1,100 @@ +import { googleVertexApi } from "../api/google-vertex.lazy.ts"; +import type { ApiKeyAuth } from "../auth/types.ts"; +import { createProvider, type Provider } from "../models.ts"; +import { GOOGLE_VERTEX_MODELS } from "./google-vertex.models.ts"; + +const VERTEX_ADC_PATH = "~/.config/gcloud/application_default_credentials.json"; + +/** + * Vertex accepts an explicit API key or Application Default Credentials + * (`gcloud auth application-default login`). ADC additionally requires + * project and location env vars, which the implementation reads itself. + */ +const vertexAuth: ApiKeyAuth = { + name: "Google Cloud credentials", + login: async (interaction) => { + interaction.signal.throwIfAborted(); + const method = await interaction.prompt({ + type: "select", + message: "Select Google Vertex AI authentication method:", + options: [ + { id: "api-key", label: "Google Cloud API key" }, + { id: "adc", label: "Application Default Credentials" }, + { id: "service-account", label: "Service account credentials file" }, + ], + }); + interaction.signal.throwIfAborted(); + if (method === "api-key") { + return { + type: "api_key", + key: await interaction.prompt({ type: "secret", message: "Enter Google Cloud API key" }), + }; + } + if (method !== "adc" && method !== "service-account") { + throw new Error(`Unknown Google Vertex AI auth method: ${method}`); + } + interaction.notify({ + type: "info", + message: + method === "adc" + ? "Run `gcloud auth application-default login`, then provide the project and location." + : "Provide a service account credentials file, project, and location.", + links: [ + { + label: "Application Default Credentials", + url: "https://cloud.google.com/docs/authentication/provide-credentials-adc", + }, + ], + }); + const project = await interaction.prompt({ type: "text", message: "Enter Google Cloud project ID" }); + const location = await interaction.prompt({ type: "text", message: "Enter Google Cloud location" }); + const credentialsPath = + method === "service-account" + ? await interaction.prompt({ type: "text", message: "Enter service account credentials file path" }) + : undefined; + return { + type: "api_key", + env: { + GOOGLE_CLOUD_PROJECT: project, + GOOGLE_CLOUD_LOCATION: location, + ...(credentialsPath ? { GOOGLE_APPLICATION_CREDENTIALS: credentialsPath } : {}), + }, + }; + }, + resolve: async ({ ctx, credential, signal }) => { + const env = async (name: string) => { + signal.throwIfAborted(); + const value = await ctx.env(name); + signal.throwIfAborted(); + return value; + }; + const key = credential?.key ?? (await env("GOOGLE_CLOUD_API_KEY")); + if (key) return { auth: { apiKey: key }, source: credential?.key ? "stored credential" : "GOOGLE_CLOUD_API_KEY" }; + + const adcPath = credential?.env?.GOOGLE_APPLICATION_CREDENTIALS ?? (await env("GOOGLE_APPLICATION_CREDENTIALS")); + signal.throwIfAborted(); + const hasCredentials = await ctx.fileExists(adcPath ?? VERTEX_ADC_PATH); + signal.throwIfAborted(); + const project = + credential?.env?.GOOGLE_CLOUD_PROJECT ?? (await env("GOOGLE_CLOUD_PROJECT")) ?? (await env("GCLOUD_PROJECT")); + const location = credential?.env?.GOOGLE_CLOUD_LOCATION ?? (await env("GOOGLE_CLOUD_LOCATION")); + if (hasCredentials && project && location) { + return { + auth: {}, + env: credential?.env, + source: credential ? "stored credential" : "gcloud application default credentials", + }; + } + return undefined; + }, +}; + +export function googleVertexProvider(): Provider<"google-vertex"> { + return createProvider({ + id: "google-vertex", + name: "Google Vertex AI", + auth: { apiKey: vertexAuth }, + models: Object.values(GOOGLE_VERTEX_MODELS), + api: googleVertexApi(), + }); +} diff --git a/packages/ai/src/providers/huggingface.ts b/packages/ai/src/providers/huggingface.ts new file mode 100644 index 0000000000000000000000000000000000000000..e8fb628e199203fc63423053bb30f79c4375c992 --- /dev/null +++ b/packages/ai/src/providers/huggingface.ts @@ -0,0 +1,15 @@ +import { openAICompletionsApi } from "../api/openai-completions.lazy.ts"; +import { envApiKeyAuth } from "../auth/helpers.ts"; +import { createProvider, type Provider } from "../models.ts"; +import { HUGGINGFACE_MODELS } from "./huggingface.models.ts"; + +export function huggingfaceProvider(): Provider<"openai-completions"> { + return createProvider({ + id: "huggingface", + name: "Hugging Face", + baseUrl: "https://router.huggingface.co/v1", + auth: { apiKey: envApiKeyAuth("Hugging Face token", ["HF_TOKEN"]) }, + models: Object.values(HUGGINGFACE_MODELS), + api: openAICompletionsApi(), + }); +} diff --git a/packages/ai/src/providers/minimax.models.ts b/packages/ai/src/providers/minimax.models.ts new file mode 100644 index 0000000000000000000000000000000000000000..fa228bdc510e7e5b76d3ff8bba60257af9285dab --- /dev/null +++ b/packages/ai/src/providers/minimax.models.ts @@ -0,0 +1,8 @@ +// This file is auto-generated by scripts/generate-models.ts +// Do not edit manually - run 'npm run generate-models' to update + +import values from "./data/minimax.json" with { type: "json" }; +import { flattenModelCatalog, type ModelCatalog } from "../model-catalog.ts"; + +export const MINIMAX_MODELS: ModelCatalog = + flattenModelCatalog("minimax", values); diff --git a/packages/ai/src/providers/mistral.models.ts b/packages/ai/src/providers/mistral.models.ts new file mode 100644 index 0000000000000000000000000000000000000000..acb8d385435b21a6196f077cee7a85fa2dd08055 --- /dev/null +++ b/packages/ai/src/providers/mistral.models.ts @@ -0,0 +1,8 @@ +// This file is auto-generated by scripts/generate-models.ts +// Do not edit manually - run 'npm run generate-models' to update + +import values from "./data/mistral.json" with { type: "json" }; +import { flattenModelCatalog, type ModelCatalog } from "../model-catalog.ts"; + +export const MISTRAL_MODELS: ModelCatalog = + flattenModelCatalog("mistral", values); diff --git a/packages/ai/src/providers/moonshotai-cn.models.ts b/packages/ai/src/providers/moonshotai-cn.models.ts new file mode 100644 index 0000000000000000000000000000000000000000..652dcd640fc570040e3e53cf4c43b2060c0198e1 --- /dev/null +++ b/packages/ai/src/providers/moonshotai-cn.models.ts @@ -0,0 +1,8 @@ +// This file is auto-generated by scripts/generate-models.ts +// Do not edit manually - run 'npm run generate-models' to update + +import values from "./data/moonshotai-cn.json" with { type: "json" }; +import { flattenModelCatalog, type ModelCatalog } from "../model-catalog.ts"; + +export const MOONSHOTAI_CN_MODELS: ModelCatalog = + flattenModelCatalog("moonshotai-cn", values); diff --git a/packages/ai/src/providers/openai-codex.ts b/packages/ai/src/providers/openai-codex.ts new file mode 100644 index 0000000000000000000000000000000000000000..75ec9e6dda568a6c9127f2de15e413013942322f --- /dev/null +++ b/packages/ai/src/providers/openai-codex.ts @@ -0,0 +1,22 @@ +import { openAICodexResponsesApi } from "../api/openai-codex-responses.lazy.ts"; +import { lazyOAuth } from "../auth/helpers.ts"; +import { loadOpenAICodexOAuth } from "../auth/oauth/load.ts"; +import { createProvider, type Provider } from "../models.ts"; +import { OPENAI_CODEX_MODELS } from "./openai-codex.models.ts"; + +export function openaiCodexProvider(): Provider<"openai-codex-responses"> { + return createProvider({ + id: "openai-codex", + name: "OpenAI Codex", + baseUrl: "https://chatgpt.com/backend-api", + auth: { + oauth: lazyOAuth({ + name: "OpenAI (ChatGPT Plus/Pro)", + isSubscription: true, + load: loadOpenAICodexOAuth, + }), + }, + models: Object.values(OPENAI_CODEX_MODELS), + api: openAICodexResponsesApi(), + }); +} diff --git a/packages/ai/src/providers/openai.ts b/packages/ai/src/providers/openai.ts new file mode 100644 index 0000000000000000000000000000000000000000..43f6671f22a9d6e88bc96556afc4814b32e08032 --- /dev/null +++ b/packages/ai/src/providers/openai.ts @@ -0,0 +1,15 @@ +import { openAIResponsesApi } from "../api/openai-responses.lazy.ts"; +import { envApiKeyAuth } from "../auth/helpers.ts"; +import { createProvider, type Provider } from "../models.ts"; +import { OPENAI_MODELS } from "./openai.models.ts"; + +export function openaiProvider(): Provider<"openai-responses"> { + return createProvider({ + id: "openai", + name: "OpenAI", + baseUrl: "https://api.openai.com/v1", + auth: { apiKey: envApiKeyAuth("OpenAI API key", ["OPENAI_API_KEY"]) }, + models: Object.values(OPENAI_MODELS), + api: openAIResponsesApi(), + }); +} diff --git a/packages/ai/src/providers/opencode-go.models.ts b/packages/ai/src/providers/opencode-go.models.ts new file mode 100644 index 0000000000000000000000000000000000000000..8e6a371709a68e8f23b9f080b36c93ae27a15138 --- /dev/null +++ b/packages/ai/src/providers/opencode-go.models.ts @@ -0,0 +1,8 @@ +// This file is auto-generated by scripts/generate-models.ts +// Do not edit manually - run 'npm run generate-models' to update + +import values from "./data/opencode-go.json" with { type: "json" }; +import { flattenModelCatalog, type ModelCatalog } from "../model-catalog.ts"; + +export const OPENCODE_GO_MODELS: ModelCatalog = + flattenModelCatalog("opencode-go", values); diff --git a/packages/ai/src/providers/openrouter.models.ts b/packages/ai/src/providers/openrouter.models.ts new file mode 100644 index 0000000000000000000000000000000000000000..bc196236f1b56ded6512b645a05a0a9f6a5a388c --- /dev/null +++ b/packages/ai/src/providers/openrouter.models.ts @@ -0,0 +1,8 @@ +// This file is auto-generated by scripts/generate-models.ts +// Do not edit manually - run 'npm run generate-models' to update + +import values from "./data/openrouter.json" with { type: "json" }; +import { flattenModelCatalog, type ModelCatalog } from "../model-catalog.ts"; + +export const OPENROUTER_MODELS: ModelCatalog = + flattenModelCatalog("openrouter", values); diff --git a/packages/ai/src/providers/qwen-token-plan-individual.models.ts b/packages/ai/src/providers/qwen-token-plan-individual.models.ts new file mode 100644 index 0000000000000000000000000000000000000000..30e111a29434979aad8fcd7c3c2bdb6484196cf2 --- /dev/null +++ b/packages/ai/src/providers/qwen-token-plan-individual.models.ts @@ -0,0 +1,8 @@ +// This file is auto-generated by scripts/generate-models.ts +// Do not edit manually - run 'npm run generate-models' to update + +import values from "./data/qwen-token-plan-individual.json" with { type: "json" }; +import { flattenModelCatalog, type ModelCatalog } from "../model-catalog.ts"; + +export const QWEN_TOKEN_PLAN_INDIVIDUAL_MODELS: ModelCatalog = + flattenModelCatalog("qwen-token-plan-individual", values); diff --git a/packages/ai/src/providers/qwen-token-plan.models.ts b/packages/ai/src/providers/qwen-token-plan.models.ts new file mode 100644 index 0000000000000000000000000000000000000000..8e1e746b38d6cf32a3677ad5ac3aac22f583d4b6 --- /dev/null +++ b/packages/ai/src/providers/qwen-token-plan.models.ts @@ -0,0 +1,8 @@ +// This file is auto-generated by scripts/generate-models.ts +// Do not edit manually - run 'npm run generate-models' to update + +import values from "./data/qwen-token-plan.json" with { type: "json" }; +import { flattenModelCatalog, type ModelCatalog } from "../model-catalog.ts"; + +export const QWEN_TOKEN_PLAN_MODELS: ModelCatalog = + flattenModelCatalog("qwen-token-plan", values); diff --git a/packages/ai/src/providers/radius.ts b/packages/ai/src/providers/radius.ts new file mode 100644 index 0000000000000000000000000000000000000000..21db7852703bfae259a973788e4dc1bf2c925178 --- /dev/null +++ b/packages/ai/src/providers/radius.ts @@ -0,0 +1,82 @@ +import { piMessagesApi } from "../api/pi-messages.lazy.ts"; +import { envApiKeyAuth, lazyOAuth } from "../auth/helpers.ts"; +import { loadRadiusOAuth } from "../auth/oauth/load.ts"; +import type { Provider } from "../models.ts"; +import { + DEFAULT_RADIUS_GATEWAY, + getRadiusModels, + getRadiusModelsFromConfig, + loadRadiusGatewayConfig, + normalizeRadiusGatewayUrl, +} from "./radius-config.ts"; + +export interface RadiusProviderOptions { + id?: string; + name?: string; + gateway?: string; +} + +/** Radius gateway provider with a persisted, dynamically refreshed catalog. */ +export function radiusProvider(options: RadiusProviderOptions = {}): Provider<"pi-messages"> { + const id = options.id ?? "radius"; + const name = options.name ?? "Radius"; + const gateway = normalizeRadiusGatewayUrl(options.gateway ?? DEFAULT_RADIUS_GATEWAY); + let models = getRadiusModels(id, undefined); + const streams = piMessagesApi(); + + return { + id, + name, + auth: { + apiKey: envApiKeyAuth("Radius API key", ["RADIUS_API_KEY"]), + oauth: lazyOAuth({ name, load: () => loadRadiusOAuth({ name, gateway }) }), + }, + getModels: () => models, + refreshModels: async (context) => { + const stored = context.stored; + if (stored) { + const restored = stored.models.filter((model) => model.provider === id) as typeof models; + if ( + !(await context.publish({ + update: () => { + models = restored; + }, + })) + ) { + return; + } + } + + // Import catalogs cached by the pre-ModelsStore Radius implementation. + if (!stored && context.credential?.type === "oauth") { + const legacy = getRadiusModels(id, context.credential); + if (legacy.length > 0) { + if ( + !(await context.publish({ + persist: { models: legacy, checkedAt: Date.now() }, + update: () => { + models = legacy; + }, + })) + ) { + return; + } + } + } + + if (!context.allowNetwork || context.signal.aborted) return; + const apiKey = context.credential?.type === "oauth" ? context.credential.access : context.credential?.key; + const config = await loadRadiusGatewayConfig(gateway, apiKey, context.signal); + if (context.signal.aborted) return; + const refreshed = getRadiusModelsFromConfig(id, config); + await context.publish({ + persist: { models: refreshed, checkedAt: Date.now() }, + update: () => { + models = refreshed; + }, + }); + }, + stream: (model, context, streamOptions) => streams.stream(model, context, streamOptions), + streamSimple: (model, context, streamOptions) => streams.streamSimple(model, context, streamOptions), + }; +} diff --git a/packages/ai/src/providers/xai.ts b/packages/ai/src/providers/xai.ts new file mode 100644 index 0000000000000000000000000000000000000000..8e2b179c28a8a7c930015fe629dba9b898f7a763 --- /dev/null +++ b/packages/ai/src/providers/xai.ts @@ -0,0 +1,24 @@ +import { openAIResponsesApi } from "../api/openai-responses.lazy.ts"; +import { envApiKeyAuth, lazyOAuth } from "../auth/helpers.ts"; +import { loadXaiOAuth } from "../auth/oauth/load.ts"; +import { createProvider, type Provider } from "../models.ts"; +import { XAI_MODELS } from "./xai.models.ts"; + +export function xaiProvider(): Provider<"openai-responses"> { + return createProvider({ + id: "xai", + name: "xAI", + baseUrl: "https://api.x.ai/v1", + auth: { + apiKey: envApiKeyAuth("xAI API key", ["XAI_API_KEY"]), + oauth: lazyOAuth({ + name: "xAI (Grok/X subscription)", + isSubscription: true, + loginLabel: "Sign in with SuperGrok or X Premium", + load: loadXaiOAuth, + }), + }, + models: Object.values(XAI_MODELS), + api: openAIResponsesApi(), + }); +} diff --git a/packages/ai/src/providers/xiaomi-token-plan-cn.models.ts b/packages/ai/src/providers/xiaomi-token-plan-cn.models.ts new file mode 100644 index 0000000000000000000000000000000000000000..f085ad606f016edc64e92daeb14f70cf6ad58d31 --- /dev/null +++ b/packages/ai/src/providers/xiaomi-token-plan-cn.models.ts @@ -0,0 +1,8 @@ +// This file is auto-generated by scripts/generate-models.ts +// Do not edit manually - run 'npm run generate-models' to update + +import values from "./data/xiaomi-token-plan-cn.json" with { type: "json" }; +import { flattenModelCatalog, type ModelCatalog } from "../model-catalog.ts"; + +export const XIAOMI_TOKEN_PLAN_CN_MODELS: ModelCatalog = + flattenModelCatalog("xiaomi-token-plan-cn", values); diff --git a/packages/ai/src/providers/xiaomi-token-plan-sgp.models.ts b/packages/ai/src/providers/xiaomi-token-plan-sgp.models.ts new file mode 100644 index 0000000000000000000000000000000000000000..a8a4e627734437b620e0bd3ccf9e8e86296b3375 --- /dev/null +++ b/packages/ai/src/providers/xiaomi-token-plan-sgp.models.ts @@ -0,0 +1,8 @@ +// This file is auto-generated by scripts/generate-models.ts +// Do not edit manually - run 'npm run generate-models' to update + +import values from "./data/xiaomi-token-plan-sgp.json" with { type: "json" }; +import { flattenModelCatalog, type ModelCatalog } from "../model-catalog.ts"; + +export const XIAOMI_TOKEN_PLAN_SGP_MODELS: ModelCatalog = + flattenModelCatalog("xiaomi-token-plan-sgp", values); diff --git a/packages/ai/src/providers/xiaomi.ts b/packages/ai/src/providers/xiaomi.ts new file mode 100644 index 0000000000000000000000000000000000000000..5abf51691fe457443bb952abc2dc5b76291577a8 --- /dev/null +++ b/packages/ai/src/providers/xiaomi.ts @@ -0,0 +1,15 @@ +import { openAICompletionsApi } from "../api/openai-completions.lazy.ts"; +import { envApiKeyAuth } from "../auth/helpers.ts"; +import { createProvider, type Provider } from "../models.ts"; +import { XIAOMI_MODELS } from "./xiaomi.models.ts"; + +export function xiaomiProvider(): Provider<"openai-completions"> { + return createProvider({ + id: "xiaomi", + name: "Xiaomi", + baseUrl: "https://api.xiaomimimo.com/v1", + auth: { apiKey: envApiKeyAuth("Xiaomi API key", ["XIAOMI_API_KEY"]) }, + models: Object.values(XIAOMI_MODELS), + api: openAICompletionsApi(), + }); +} diff --git a/packages/ai/src/providers/zai-coding-cn.models.ts b/packages/ai/src/providers/zai-coding-cn.models.ts new file mode 100644 index 0000000000000000000000000000000000000000..a91f6edd12ccdb81718c6b7e5afc8eda81b71d09 --- /dev/null +++ b/packages/ai/src/providers/zai-coding-cn.models.ts @@ -0,0 +1,8 @@ +// This file is auto-generated by scripts/generate-models.ts +// Do not edit manually - run 'npm run generate-models' to update + +import values from "./data/zai-coding-cn.json" with { type: "json" }; +import { flattenModelCatalog, type ModelCatalog } from "../model-catalog.ts"; + +export const ZAI_CODING_CN_MODELS: ModelCatalog = + flattenModelCatalog("zai-coding-cn", values); diff --git a/packages/ai/test/abort.test.ts b/packages/ai/test/abort.test.ts new file mode 100644 index 0000000000000000000000000000000000000000..767be7d04d5c32bcb7963b7a783130e664a104bf --- /dev/null +++ b/packages/ai/test/abort.test.ts @@ -0,0 +1,351 @@ +import { describe, expect, it } from "vitest"; +import { complete, getModel, stream } from "../src/compat.ts"; +import type { Api, Context, Model, StreamOptions } from "../src/types.ts"; + +type StreamOptionsWithExtras = StreamOptions & Record; + +import { hasAzureOpenAICredentials, resolveAzureDeploymentName } from "./azure-utils.ts"; +import { hasBedrockCredentials } from "./bedrock-utils.ts"; +import { resolveApiKey } from "./oauth.ts"; + +// Resolve OAuth tokens at module level (async, runs before tests) +const [openaiCodexToken] = await Promise.all([resolveApiKey("openai-codex")]); + +async function testAbortSignal(llm: Model, options: StreamOptionsWithExtras = {}) { + const context: Context = { + messages: [ + { + role: "user", + content: "What is 15 + 27? Think step by step. Then list 50 first names.", + timestamp: Date.now(), + }, + ], + systemPrompt: "You are a helpful assistant.", + }; + + let abortFired = false; + let text = ""; + const controller = new AbortController(); + const response = await stream(llm, context, { ...options, signal: controller.signal }); + for await (const event of response) { + if (abortFired) return; + if (event.type === "text_delta" || event.type === "thinking_delta") { + text += event.delta; + } + if (text.length >= 50) { + controller.abort(); + abortFired = true; + } + } + const msg = await response.result(); + + // If we get here without throwing, the abort didn't work + expect(msg.stopReason).toBe("aborted"); + expect(msg.content.length).toBeGreaterThan(0); + + context.messages.push(msg); + context.messages.push({ + role: "user", + content: "Please continue, but only generate 5 names.", + timestamp: Date.now(), + }); + + const followUp = await complete(llm, context, options); + expect(followUp.stopReason).toBe("stop"); + expect(followUp.content.length).toBeGreaterThan(0); +} + +async function testImmediateAbort(llm: Model, options: StreamOptionsWithExtras = {}) { + const controller = new AbortController(); + + controller.abort(); + + const context: Context = { + messages: [{ role: "user", content: "Hello", timestamp: Date.now() }], + }; + + const response = await complete(llm, context, { ...options, signal: controller.signal }); + expect(response.stopReason).toBe("aborted"); +} + +async function testAbortThenNewMessage(llm: Model, options: StreamOptionsWithExtras = {}) { + // First request: abort immediately before any response content arrives + const controller = new AbortController(); + controller.abort(); + + const context: Context = { + messages: [{ role: "user", content: "Hello, how are you?", timestamp: Date.now() }], + }; + + const abortedResponse = await complete(llm, context, { ...options, signal: controller.signal }); + expect(abortedResponse.stopReason).toBe("aborted"); + // The aborted message has empty content since we aborted before anything arrived + expect(abortedResponse.content.length).toBe(0); + + // Add the aborted assistant message to context (this is what happens in the real coding agent) + context.messages.push(abortedResponse); + + // Second request: send a new message - this should work even with the aborted message in context + context.messages.push({ + role: "user", + content: "What is 2 + 2?", + timestamp: Date.now(), + }); + + const followUp = await complete(llm, context, options); + expect(followUp.stopReason).toBe("stop"); + expect(followUp.content.length).toBeGreaterThan(0); +} + +describe("AI Providers Abort Tests", () => { + describe.skipIf(!process.env.GEMINI_API_KEY)("Google Provider Abort", () => { + const llm = getModel("google", "gemini-2.5-flash"); + + it("should abort mid-stream", { retry: 3 }, async () => { + await testAbortSignal(llm, { thinking: { enabled: true } }); + }); + + it("should handle immediate abort", { retry: 3 }, async () => { + await testImmediateAbort(llm, { thinking: { enabled: true } }); + }); + }); + + describe.skipIf(!process.env.OPENAI_API_KEY)("OpenAI Completions Provider Abort", () => { + const { compat: _compat, ...baseModel } = getModel("openai", "gpt-4o-mini")!; + void _compat; + const llm: Model<"openai-completions"> = { + ...baseModel, + api: "openai-completions", + }; + + it("should abort mid-stream", { retry: 3 }, async () => { + await testAbortSignal(llm); + }); + + it("should handle immediate abort", { retry: 3 }, async () => { + await testImmediateAbort(llm); + }); + }); + + describe.skipIf(!process.env.OPENAI_API_KEY)("OpenAI Responses Provider Abort", () => { + const llm = getModel("openai", "gpt-5-mini"); + + it("should abort mid-stream", { retry: 3 }, async () => { + await testAbortSignal(llm); + }); + + it("should handle immediate abort", { retry: 3 }, async () => { + await testImmediateAbort(llm); + }); + }); + + describe.skipIf(!hasAzureOpenAICredentials())("Azure OpenAI Responses Provider Abort", () => { + const llm = getModel("azure-openai-responses", "gpt-4o-mini"); + const azureDeploymentName = resolveAzureDeploymentName(llm.id); + const azureOptions = azureDeploymentName ? { azureDeploymentName } : {}; + + it("should abort mid-stream", { retry: 3 }, async () => { + await testAbortSignal(llm, azureOptions); + }); + + it("should handle immediate abort", { retry: 3 }, async () => { + await testImmediateAbort(llm, azureOptions); + }); + }); + + describe.skipIf(!process.env.ANTHROPIC_OAUTH_TOKEN)("Anthropic Provider Abort", () => { + const llm = getModel("anthropic", "claude-sonnet-4-6"); + + it("should abort mid-stream", { retry: 3 }, async () => { + await testAbortSignal(llm, { thinkingEnabled: true, thinkingBudgetTokens: 2048 }); + }); + + it("should handle immediate abort", { retry: 3 }, async () => { + await testImmediateAbort(llm, { thinkingEnabled: true, thinkingBudgetTokens: 2048 }); + }); + }); + + describe.skipIf(!process.env.MISTRAL_API_KEY)("Mistral Provider Abort", () => { + const llm = getModel("mistral", "devstral-medium-latest"); + + it("should abort mid-stream", { retry: 3 }, async () => { + await testAbortSignal(llm); + }); + + it("should handle immediate abort", { retry: 3 }, async () => { + await testImmediateAbort(llm); + }); + }); + + describe.skipIf(!process.env.TOGETHER_API_KEY)("Together AI Provider Abort", () => { + const llm = getModel("together", "moonshotai/Kimi-K2.6"); + + it("should abort mid-stream", { retry: 3 }, async () => { + await testAbortSignal(llm, { reasoningEffort: "high" }); + }); + + it("should handle immediate abort", { retry: 3 }, async () => { + await testImmediateAbort(llm, { reasoningEffort: "high" }); + }); + }); + + describe.skipIf(!process.env.BASETEN_API_KEY)("Baseten Provider Abort", () => { + const llm = getModel("baseten", "zai-org/GLM-5.2"); + + it("should abort mid-stream", { retry: 3 }, async () => { + await testAbortSignal(llm, { reasoningEffort: "high" }); + }); + + it("should handle immediate abort", { retry: 3 }, async () => { + await testImmediateAbort(llm, { reasoningEffort: "high" }); + }); + }); + + describe.skipIf(!process.env.MINIMAX_API_KEY)("MiniMax Provider Abort", () => { + const llm = getModel("minimax", "MiniMax-M2.7"); + + it("should abort mid-stream", { retry: 3 }, async () => { + await testAbortSignal(llm); + }); + + it("should handle immediate abort", { retry: 3 }, async () => { + await testImmediateAbort(llm); + }); + }); + + describe.skipIf(!process.env.XIAOMI_API_KEY)("Xiaomi MiMo (API billing) Provider Abort", () => { + const llm = getModel("xiaomi", "mimo-v2.5-pro"); + + it("should abort mid-stream", { retry: 3 }, async () => { + await testAbortSignal(llm); + }); + + it("should handle immediate abort", { retry: 3 }, async () => { + await testImmediateAbort(llm); + }); + }); + + describe.skipIf(!process.env.XIAOMI_TOKEN_PLAN_CN_API_KEY)("Xiaomi MiMo Token Plan (CN) Provider Abort", () => { + const llm = getModel("xiaomi-token-plan-cn", "mimo-v2.5-pro"); + + it("should abort mid-stream", { retry: 3 }, async () => { + await testAbortSignal(llm); + }); + + it("should handle immediate abort", { retry: 3 }, async () => { + await testImmediateAbort(llm); + }); + }); + + describe.skipIf(!process.env.XIAOMI_TOKEN_PLAN_AMS_API_KEY)("Xiaomi MiMo Token Plan (AMS) Provider Abort", () => { + const llm = getModel("xiaomi-token-plan-ams", "mimo-v2.5-pro"); + + it("should abort mid-stream", { retry: 3 }, async () => { + await testAbortSignal(llm); + }); + + it("should handle immediate abort", { retry: 3 }, async () => { + await testImmediateAbort(llm); + }); + }); + + describe.skipIf(!process.env.XIAOMI_TOKEN_PLAN_SGP_API_KEY)("Xiaomi MiMo Token Plan (SGP) Provider Abort", () => { + const llm = getModel("xiaomi-token-plan-sgp", "mimo-v2.5-pro"); + + it("should abort mid-stream", { retry: 3 }, async () => { + await testAbortSignal(llm); + }); + + it("should handle immediate abort", { retry: 3 }, async () => { + await testImmediateAbort(llm); + }); + }); + + describe.skipIf(!process.env.QWEN_TOKEN_PLAN_API_KEY)("Qwen Token Plan Provider Abort", () => { + const llm = getModel("qwen-token-plan", "qwen3.7-max"); + + it("should abort mid-stream", { retry: 3 }, async () => { + await testAbortSignal(llm); + }); + + it("should handle immediate abort", { retry: 3 }, async () => { + await testImmediateAbort(llm); + }); + }); + + describe.skipIf(!process.env.QWEN_TOKEN_PLAN_API_KEY)("Qwen Token Plan Individual Provider Abort", () => { + const llm = getModel("qwen-token-plan-individual", "qwen3.8-max"); + + it("should abort mid-stream", { retry: 3 }, async () => { + await testAbortSignal(llm); + }); + + it("should handle immediate abort", { retry: 3 }, async () => { + await testImmediateAbort(llm); + }); + }); + + describe.skipIf(!process.env.QWEN_TOKEN_PLAN_CN_API_KEY)("Qwen Token Plan (CN) Provider Abort", () => { + const llm = getModel("qwen-token-plan-cn", "qwen3.7-max"); + + it("should abort mid-stream", { retry: 3 }, async () => { + await testAbortSignal(llm); + }); + + it("should handle immediate abort", { retry: 3 }, async () => { + await testImmediateAbort(llm); + }); + }); + + describe.skipIf(!process.env.KIMI_API_KEY)("Kimi For Coding Provider Abort", () => { + const llm = getModel("kimi-coding", "kimi-for-coding"); + + it("should abort mid-stream", { retry: 3 }, async () => { + await testAbortSignal(llm); + }); + + it("should handle immediate abort", { retry: 3 }, async () => { + await testImmediateAbort(llm); + }); + }); + + describe.skipIf(!process.env.AI_GATEWAY_API_KEY)("Vercel AI Gateway Provider Abort", () => { + const llm = getModel("vercel-ai-gateway", "google/gemini-2.5-flash"); + + it("should abort mid-stream", { retry: 3 }, async () => { + await testAbortSignal(llm); + }); + + it("should handle immediate abort", { retry: 3 }, async () => { + await testImmediateAbort(llm); + }); + }); + + describe("OpenAI Codex Provider Abort", () => { + it.skipIf(!openaiCodexToken)("should abort mid-stream", { retry: 3 }, async () => { + const llm = getModel("openai-codex", "gpt-5.5"); + await testAbortSignal(llm, { apiKey: openaiCodexToken }); + }); + + it.skipIf(!openaiCodexToken)("should handle immediate abort", { retry: 3 }, async () => { + const llm = getModel("openai-codex", "gpt-5.5"); + await testImmediateAbort(llm, { apiKey: openaiCodexToken }); + }); + }); + + describe.skipIf(!hasBedrockCredentials())("Amazon Bedrock Provider Abort", () => { + const llm = getModel("amazon-bedrock", "global.anthropic.claude-sonnet-4-5-20250929-v1:0"); + + it("should abort mid-stream", { retry: 3 }, async () => { + await testAbortSignal(llm, { reasoning: "medium" }); + }); + + it("should handle immediate abort", { retry: 3 }, async () => { + await testImmediateAbort(llm); + }); + + it("should handle abort then new message", { retry: 3 }, async () => { + await testAbortThenNewMessage(llm); + }); + }); +}); diff --git a/packages/ai/test/anthropic-cache-write-1h-cost.test.ts b/packages/ai/test/anthropic-cache-write-1h-cost.test.ts new file mode 100644 index 0000000000000000000000000000000000000000..967a58b189e1d803b2402a7399b1d451deeb81c8 --- /dev/null +++ b/packages/ai/test/anthropic-cache-write-1h-cost.test.ts @@ -0,0 +1,86 @@ +import type Anthropic from "@anthropic-ai/sdk"; +import { describe, expect, it } from "vitest"; +import { stream as streamAnthropic } from "../src/api/anthropic-messages.ts"; +import { getModel } from "../src/compat.ts"; +import type { Context } from "../src/types.ts"; + +function createSseResponse(events: Array<{ event: string; data: string }>): Response { + const body = events.map(({ event, data }) => `event: ${event}\ndata: ${data}\n`).join("\n"); + return new Response(body, { status: 200, headers: { "content-type": "text/event-stream" } }); +} + +function createFakeAnthropicClient(response: Response): Anthropic { + return { + beta: { messages: { create: () => ({ asResponse: async () => response }) } }, + } as unknown as Anthropic; +} + +function eventsWithCacheCreation( + cacheCreation: Record | undefined, +): Array<{ event: string; data: string }> { + const startUsage: Record = { + input_tokens: 100, + output_tokens: 0, + cache_read_input_tokens: 0, + cache_creation_input_tokens: 1_000_000, + }; + if (cacheCreation) startUsage.cache_creation = cacheCreation; + return [ + { + event: "message_start", + data: JSON.stringify({ type: "message_start", message: { id: "msg_test", usage: startUsage } }), + }, + { + event: "content_block_start", + data: JSON.stringify({ type: "content_block_start", index: 0, content_block: { type: "text", text: "" } }), + }, + { + event: "content_block_delta", + data: JSON.stringify({ type: "content_block_delta", index: 0, delta: { type: "text_delta", text: "Hi" } }), + }, + { event: "content_block_stop", data: JSON.stringify({ type: "content_block_stop", index: 0 }) }, + { + event: "message_delta", + data: JSON.stringify({ + type: "message_delta", + delta: { stop_reason: "end_turn" }, + usage: { + input_tokens: 100, + output_tokens: 5, + cache_read_input_tokens: 0, + cache_creation_input_tokens: 1_000_000, + }, + }), + }, + { event: "message_stop", data: JSON.stringify({ type: "message_stop" }) }, + ]; +} + +// claude-opus-4-8: input 5, cacheWrite (5m) 6.25 per Mtok. 1h write = 2x input = 10. +const context: Context = { messages: [{ role: "user", content: "hi", timestamp: Date.now() }] }; + +describe("Anthropic 1h cache write cost", () => { + it("prices the 1h portion at 2x input and the rest at the 5m rate", async () => { + const model = getModel("anthropic", "claude-opus-4-8"); + const response = createSseResponse( + eventsWithCacheCreation({ ephemeral_5m_input_tokens: 600_000, ephemeral_1h_input_tokens: 400_000 }), + ); + const result = await streamAnthropic(model, context, { client: createFakeAnthropicClient(response) }).result(); + + expect(result.usage.cacheWrite).toBe(1_000_000); + expect(result.usage.cacheWrite1h).toBe(400_000); + // 600k * 6.25/Mtok + 400k * 10/Mtok = 3.75 + 4.0 = 7.75 + expect(result.usage.cost.cacheWrite).toBeCloseTo(7.75, 10); + }); + + it("falls back to the 5m rate when no breakdown is reported", async () => { + const model = getModel("anthropic", "claude-opus-4-8"); + const response = createSseResponse(eventsWithCacheCreation(undefined)); + const result = await streamAnthropic(model, context, { client: createFakeAnthropicClient(response) }).result(); + + expect(result.usage.cacheWrite).toBe(1_000_000); + expect(result.usage.cacheWrite1h ?? 0).toBe(0); + // 1M * 6.25/Mtok = 6.25 + expect(result.usage.cost.cacheWrite).toBeCloseTo(6.25, 10); + }); +}); diff --git a/packages/ai/test/anthropic-long-cache-retention-e2e.test.ts b/packages/ai/test/anthropic-long-cache-retention-e2e.test.ts new file mode 100644 index 0000000000000000000000000000000000000000..7855c872df9491ffc60f91ed41bf28015e65d68d --- /dev/null +++ b/packages/ai/test/anthropic-long-cache-retention-e2e.test.ts @@ -0,0 +1,127 @@ +import { describe, expect, it } from "vitest"; +import { type BuiltinProvider, complete, getModels, getProviders } from "../src/compat.ts"; +import { getEnvApiKey } from "../src/env-api-keys.ts"; +import type { Api, KnownProvider, Model, ProviderStreamOptions } from "../src/types.ts"; +import { resolveApiKey } from "./oauth.ts"; + +const githubCopilotToken = await resolveApiKey("github-copilot"); + +interface AnthropicLongCacheRetentionE2ECase { + name: string; + provider: BuiltinProvider; + model: Model<"anthropic-messages">; + apiKey: string | undefined; +} + +function getE2EApiKey(provider: KnownProvider): string | undefined { + if (provider === "github-copilot") { + return githubCopilotToken; + } + return getEnvApiKey(provider); +} + +function getAnthropicMessagesModels(provider: BuiltinProvider): Model<"anthropic-messages">[] { + const models = getModels(provider) as Model[]; + return models.filter((model) => model.api === "anthropic-messages") as Model<"anthropic-messages">[]; +} + +const anthropicMessagesCases: AnthropicLongCacheRetentionE2ECase[] = getProviders().flatMap((provider) => + getAnthropicMessagesModels(provider).map((model) => ({ + name: `${provider}/${model.id}`, + provider, + model, + apiKey: getE2EApiKey(provider), + })), +); + +function getProbePriority(model: Model<"anthropic-messages">): number { + const modelId = model.id.toLowerCase(); + const cost = model.cost.input + model.cost.output; + let priority = cost; + + if (modelId.includes("haiku") && (modelId.includes("4-5") || modelId.includes("4.5"))) { + priority -= 1000; + } else if (modelId.includes("sonnet") && (modelId.includes("4-") || modelId.includes("4."))) { + priority -= 750; + } else if (modelId.includes("claude") && (modelId.includes("4-") || modelId.includes("4."))) { + priority -= 500; + } + + return priority; +} + +function selectOneCasePerProvider(cases: AnthropicLongCacheRetentionE2ECase[]): AnthropicLongCacheRetentionE2ECase[] { + const byProvider = new Map(); + for (const testCase of cases) { + const providerCases = byProvider.get(testCase.provider) ?? []; + providerCases.push(testCase); + byProvider.set(testCase.provider, providerCases); + } + + return Array.from(byProvider.values()).map( + (providerCases) => + providerCases.sort( + (a, b) => getProbePriority(a.model) - getProbePriority(b.model) || a.model.id.localeCompare(b.model.id), + )[0], + ); +} + +const probeCases = selectOneCasePerProvider(anthropicMessagesCases); + +function withLongCacheRetention(model: Model<"anthropic-messages">): Model<"anthropic-messages"> { + return { + ...model, + compat: { + ...model.compat, + supportsLongCacheRetention: true, + }, + }; +} + +async function expectLongCacheRetentionAccepted( + model: Model<"anthropic-messages">, + apiKey: string | undefined, +): Promise { + const options: ProviderStreamOptions = { + apiKey, + cacheRetention: "long", + maxTokens: 128, + thinkingEnabled: false, + }; + const response = await complete( + model, + { + systemPrompt: "You are a concise assistant.", + messages: [ + { + role: "user", + content: "Reply with exactly: long cache retention accepted", + timestamp: Date.now(), + }, + ], + }, + options, + ); + + expect(response.errorMessage, response.errorMessage).toBeFalsy(); + expect(response.stopReason, response.errorMessage).not.toBe("error"); +} + +describe("Anthropic Messages long cache retention E2E", () => { + it("covers every generated anthropic-messages model", () => { + const expectedModels = getProviders().flatMap((provider) => + getAnthropicMessagesModels(provider).map((model) => `${provider}/${model.id}`), + ); + expect(anthropicMessagesCases.map((testCase) => testCase.name).sort()).toEqual(expectedModels.sort()); + }); + + describe("forced long cache retention probe", () => { + for (const testCase of probeCases) { + const model = withLongCacheRetention(testCase.model); + + it.skipIf(!testCase.apiKey)(`${testCase.name} accepts long cache retention`, { retry: 2 }, async () => { + await expectLongCacheRetentionAccepted(model, testCase.apiKey); + }); + } + }); +}); diff --git a/packages/ai/test/anthropic-mid-conversation-effort.test.ts b/packages/ai/test/anthropic-mid-conversation-effort.test.ts new file mode 100644 index 0000000000000000000000000000000000000000..10461f859807004a12a495a4bfa3ff5b74d4311b --- /dev/null +++ b/packages/ai/test/anthropic-mid-conversation-effort.test.ts @@ -0,0 +1,212 @@ +import { describe, expect, it } from "vitest"; +import { stream } from "../src/api/anthropic-messages.ts"; +import { getModel } from "../src/compat.ts"; +import type { AssistantMessage, Context, Model } from "../src/types.ts"; + +interface WireMessage { + role: string; + content: unknown; + output_config?: { effort?: string }; +} + +interface CapturedPayload { + messages: WireMessage[]; + thinking?: { + type: string; + display?: string; + block_binding?: { prefix_mismatch_behavior?: string }; + }; + output_config?: { effort?: string }; + fallbacks?: Array<{ model: string }>; +} + +function managedModel(provider = "anthropic"): Model<"anthropic-messages"> { + return { + id: "claude-fable-5-1", + name: "Claude Fable 5.1", + api: "anthropic-messages", + provider, + baseUrl: "http://127.0.0.1:9", + reasoning: true, + thinkingLevelMap: { off: null, minimal: "low", low: "low", medium: "medium", high: "high", max: "max" }, + input: ["text"], + cost: { input: 0, output: 0, cacheRead: 0, cacheWrite: 0 }, + contextWindow: 200000, + maxTokens: 32000, + compat: { forceAdaptiveThinking: true, supportsMidConvoEffort: true }, + }; +} + +function assistant(model: Model<"anthropic-messages">, level?: string): AssistantMessage { + return { + role: "assistant", + content: [ + { type: "thinking", thinking: "reasoning", thinkingSignature: "signature" }, + { type: "text", text: "answer" }, + ], + api: "anthropic-messages", + provider: model.provider, + model: model.id, + ...(level === undefined ? {} : { providerThinkingLevel: level }), + usage: { + input: 0, + output: 0, + cacheRead: 0, + cacheWrite: 0, + totalTokens: 0, + cost: { input: 0, output: 0, cacheRead: 0, cacheWrite: 0, total: 0 }, + }, + stopReason: "stop", + timestamp: 1, + }; +} + +async function capture( + model: Model<"anthropic-messages">, + context: Context, + effort?: "low" | "medium" | "high" | "xhigh" | "max", +): Promise<{ payload: CapturedPayload; message: AssistantMessage }> { + let payload: CapturedPayload | undefined; + const result = stream(model, context, { + apiKey: "test-key", + cacheRetention: "none", + thinkingEnabled: true, + effort, + onPayload: (value) => { + payload = value as CapturedPayload; + throw new Error("payload captured"); + }, + }); + const message = await result.result(); + if (!payload) throw new Error("Expected payload capture"); + return { payload, message }; +} + +const user = (text: string, timestamp: number) => ({ role: "user" as const, content: text, timestamp }); + +function effortMessages(payload: CapturedPayload): WireMessage[] { + return payload.messages.filter((message) => message.role === "system"); +} + +describe("Anthropic mid-conversation effort", () => { + it("reconstructs an exact historical marker prefix and appends the current marker", async () => { + const model = managedModel(); + const first = await capture(model, { messages: [user("one", 1)] }, "low"); + const second = await capture( + model, + { messages: [user("one", 1), assistant(model, "low"), user("two", 2)] }, + "high", + ); + + expect(first.payload.messages).toEqual([ + { role: "user", content: "one" }, + { role: "system", content: [], output_config: { effort: "low" } }, + ]); + expect(second.payload.messages.slice(0, first.payload.messages.length)).toEqual(first.payload.messages); + expect(second.payload.messages.at(-1)).toEqual({ + role: "system", + content: [], + output_config: { effort: "high" }, + }); + expect(first.payload.output_config).toEqual({ effort: "high" }); + expect(second.payload.output_config).toEqual({ effort: "high" }); + expect(second.payload.thinking).toEqual({ + type: "adaptive", + display: "summarized", + block_binding: { prefix_mismatch_behavior: "drop_block" }, + }); + expect(first.message.providerThinkingLevel).toBe("low"); + }); + + it.each(["low", "medium", "high", "xhigh", "max"] as const)("preserves native effort %s", async (effort) => { + const model = managedModel(); + const { payload, message } = await capture(model, { messages: [user("one", 1)] }, effort); + expect(effortMessages(payload)).toEqual([{ role: "system", content: [], output_config: { effort } }]); + expect(message.providerThinkingLevel).toBe(effort); + }); + + it("defaults omitted effort to high and still enables drop_block", async () => { + const { payload, message } = await capture(managedModel(), { messages: [user("one", 1)] }); + expect(payload.messages.at(-1)).toEqual({ + role: "system", + content: [], + output_config: { effort: "high" }, + }); + expect(payload.thinking?.block_binding?.prefix_mismatch_behavior).toBe("drop_block"); + expect(message.providerThinkingLevel).toBe("high"); + }); + + it("does not invent markers for legacy or other-provider assistants", async () => { + const model = managedModel(); + const legacy = assistant(model); + const otherProvider = { ...assistant(model, "low"), provider: "other-provider" }; + const { payload } = await capture( + model, + { messages: [user("one", 1), legacy, user("two", 2), otherProvider, user("three", 3)] }, + "medium", + ); + expect(effortMessages(payload)).toEqual([{ role: "system", content: [], output_config: { effort: "medium" } }]); + }); + + it("leaves unsupported models on top-level effort", async () => { + const model = managedModel(); + model.compat = { forceAdaptiveThinking: true }; + const { payload, message } = await capture(model, { messages: [user("one", 1)] }, "low"); + expect(payload.messages).toEqual([{ role: "user", content: "one" }]); + expect(payload.output_config).toEqual({ effort: "low" }); + expect(payload.thinking).toEqual({ type: "adaptive", display: "summarized" }); + expect(message.providerThinkingLevel).toBeUndefined(); + }); + + it("sends the effort and binding beta headers", async () => { + let betaHeader: string | null = null; + const events = [ + { + type: "message_start", + message: { + id: "msg_test", + model: "claude-fable-5-1", + usage: { input_tokens: 1, output_tokens: 0 }, + }, + }, + { + type: "message_delta", + delta: { stop_reason: "end_turn" }, + usage: { input_tokens: 1, output_tokens: 1 }, + }, + { type: "message_stop" }, + ]; + const body = events.map((event) => `event: ${event.type}\ndata: ${JSON.stringify(event)}\n\n`).join(""); + const fetchImpl: typeof fetch = async (input, init) => { + const request = input instanceof Request ? input : new Request(input, init); + betaHeader = request.headers.get("anthropic-beta"); + return new Response(body, { status: 200, headers: { "content-type": "text/event-stream" } }); + }; + const result = await stream( + managedModel(), + { messages: [user("one", 1)] }, + { + apiKey: "test-key", + cacheRetention: "none", + fetch: fetchImpl, + }, + ).result(); + + expect(result.stopReason).toBe("stop"); + expect(betaHeader).toContain("mid-conversation-output-config-2026-07-01"); + expect(betaHeader).toContain("thinking-binding-controls-2026-08-01"); + }); + + it("generates exact model and transport gates", () => { + const direct = getModel("anthropic", "claude-fable-5-1"); + const openRouter = getModel("openrouter", "anthropic/claude-fable-5.1"); + const unsupported = getModel("anthropic", "claude-opus-4-8"); + expect(direct.compat?.supportsMidConvoEffort).toBe(true); + expect(direct.thinkingLevelMap?.off).toBeNull(); + expect(openRouter.api).toBe("anthropic-messages"); + expect(openRouter.baseUrl).toBe("https://openrouter.ai/api"); + expect(openRouter.compat?.supportsMidConvoEffort).toBe(true); + expect(unsupported.compat?.supportsMidConvoEffort).toBeUndefined(); + expect(getModel("anthropic", "claude-opus-5").compat?.allowedFallbackModels).toBeUndefined(); + }); +}); diff --git a/packages/ai/test/anthropic-oauth.test.ts b/packages/ai/test/anthropic-oauth.test.ts new file mode 100644 index 0000000000000000000000000000000000000000..9dc227b93904455f1e65dc4e1d4674374eb8cb4e --- /dev/null +++ b/packages/ai/test/anthropic-oauth.test.ts @@ -0,0 +1,144 @@ +import { afterEach, describe, expect, it, vi } from "vitest"; +import { anthropicOAuth } from "../src/auth/oauth/anthropic.ts"; +import type { AuthEvent, AuthPrompt } from "../src/auth/types.ts"; + +const neverAbortedSignal = new AbortController().signal; + +function jsonResponse(body: unknown, status: number = 200): Response { + return new Response(JSON.stringify(body), { + status, + headers: { + "Content-Type": "application/json", + }, + }); +} + +function getUrl(input: unknown): string { + if (typeof input === "string") { + return input; + } + if (input instanceof URL) { + return input.toString(); + } + if (input instanceof Request) { + return input.url; + } + throw new Error(`Unsupported fetch input: ${String(input)}`); +} + +function getJsonBody(init?: RequestInit): Record { + if (typeof init?.body !== "string") { + throw new Error(`Expected string request body, got ${typeof init?.body}`); + } + return JSON.parse(init.body) as Record; +} + +describe.sequential("Anthropic OAuth", () => { + afterEach(() => { + vi.unstubAllGlobals(); + }); + + it("keeps the localhost redirect_uri for manual callback login", async () => { + let authUrl = ""; + const fetchMock = vi.fn(async (input: unknown, init?: RequestInit): Promise => { + expect(getUrl(input)).toBe("https://platform.claude.com/v1/oauth/token"); + expect(init?.method).toBe("POST"); + const body = getJsonBody(init); + expect(body.grant_type).toBe("authorization_code"); + expect(body.code).toBe("manual-code"); + expect(body.redirect_uri).toBe("http://localhost:53692/callback"); + return jsonResponse({ + access_token: "access-token", + refresh_token: "refresh-token", + expires_in: 3600, + }); + }); + vi.stubGlobal("fetch", fetchMock); + + const credentials = await anthropicOAuth.login({ + signal: neverAbortedSignal, + notify: (event) => { + if (event.type === "auth_url") authUrl = event.url; + }, + prompt: async (prompt) => { + if (prompt.type !== "manual_code") throw new Error(`Unexpected prompt: ${prompt.type}`); + const url = new URL(authUrl); + const state = url.searchParams.get("state"); + const redirectUri = url.searchParams.get("redirect_uri"); + if (!state || !redirectUri) throw new Error("Missing OAuth state or redirect_uri in auth URL"); + return `${redirectUri}?code=manual-code&state=${state}`; + }, + }); + + expect(credentials.access).toBe("access-token"); + expect(credentials.refresh).toBe("refresh-token"); + expect(fetchMock).toHaveBeenCalledOnce(); + }); + + it("omits scope from refresh token requests", async () => { + const fetchMock = vi.fn(async (input: unknown, init?: RequestInit): Promise => { + expect(getUrl(input)).toBe("https://platform.claude.com/v1/oauth/token"); + expect(init?.method).toBe("POST"); + const body = getJsonBody(init); + expect(body.grant_type).toBe("refresh_token"); + expect(body.client_id).toBeTruthy(); + expect(body.refresh_token).toBe("refresh-token"); + expect(body).not.toHaveProperty("scope"); + return jsonResponse({ + access_token: "new-access-token", + refresh_token: "new-refresh-token", + expires_in: 3600, + }); + }); + vi.stubGlobal("fetch", fetchMock); + + const credentials = await anthropicOAuth.refresh( + { + type: "oauth", + access: "old-access-token", + refresh: "refresh-token", + expires: 0, + }, + neverAbortedSignal, + ); + + expect(credentials.access).toBe("new-access-token"); + expect(credentials.refresh).toBe("new-refresh-token"); + expect(fetchMock).toHaveBeenCalledOnce(); + }); + + it("anthropicOAuth.login resolves through the manual_code prompt and aborts it after settling", async () => { + const fetchMock = vi.fn(async (input: unknown): Promise => { + const url = typeof input === "string" ? input : String(input); + if (url.includes("/oauth/token")) { + return jsonResponse({ access_token: "access", refresh_token: "refresh", expires_in: 3600 }); + } + throw new Error(`Unexpected fetch: ${url}`); + }); + vi.stubGlobal("fetch", fetchMock); + + const events: AuthEvent[] = []; + const prompts: AuthPrompt[] = []; + let manualSignal: AbortSignal | undefined; + + const credential = await anthropicOAuth.login({ + signal: neverAbortedSignal, + notify: (event) => events.push(event), + prompt: async (prompt) => { + prompts.push(prompt); + if (prompt.type === "manual_code") { + manualSignal = prompt.signal; + return "the-code"; + } + throw new Error(`Unexpected prompt: ${prompt.type}`); + }, + }); + + expect(credential.type).toBe("oauth"); + expect(credential.access).toBe("access"); + expect(events.some((e) => e.type === "auth_url")).toBe(true); + expect(prompts.some((p) => p.type === "manual_code")).toBe(true); + // the prompt's signal is aborted once login settles, so UIs can dismiss it + expect(manualSignal?.aborted).toBe(true); + }); +}); diff --git a/packages/ai/test/anthropic-opus-4-8-smoke.test.ts b/packages/ai/test/anthropic-opus-4-8-smoke.test.ts new file mode 100644 index 0000000000000000000000000000000000000000..44fa5aebb2cf133213d8a01f2d0e12bc0bf88b57 --- /dev/null +++ b/packages/ai/test/anthropic-opus-4-8-smoke.test.ts @@ -0,0 +1,71 @@ +import { describe, expect, it } from "vitest"; +import { getModel, streamSimple } from "../src/compat.ts"; +import type { Context } from "../src/types.ts"; + +interface AnthropicThinkingPayload { + thinking?: { type: string }; + output_config?: { effort?: string }; +} + +function makeContext(): Context { + return { + systemPrompt: "You are a precise assistant. Follow the user's instructions exactly.", + messages: [ + { + role: "user", + content: + "Compute 48291 * 7317 and 90844 - 17729, add the results, and determine whether the sum is divisible by 11. Reply with exactly this format and nothing else: sum=; divisibleBy11=", + timestamp: Date.now(), + }, + ], + }; +} + +describe.skipIf(!process.env.ANTHROPIC_API_KEY)("Anthropic Opus 4.8 smoke", () => { + it("streams Claude Opus 4.8 with reasoning enabled", { retry: 2, timeout: 30000 }, async () => { + const model = getModel("anthropic", "claude-opus-4-8"); + let capturedPayload: AnthropicThinkingPayload | undefined; + const s = streamSimple(model, makeContext(), { + reasoning: "high", + maxTokens: 1024, + onPayload: (payload) => { + capturedPayload = payload as AnthropicThinkingPayload; + return payload; + }, + }); + + let sawThinking = false; + + for await (const event of s) { + if (event.type === "thinking_start" || event.type === "thinking_delta" || event.type === "thinking_end") { + sawThinking = true; + } + } + + const response = await s.result(); + expect(response.stopReason, response.errorMessage).toBe("stop"); + expect(response.errorMessage).toBeFalsy(); + expect(capturedPayload?.thinking).toEqual({ type: "adaptive" }); + expect(capturedPayload?.output_config).toEqual({ effort: "high" }); + expect(sawThinking).toBe(true); + + const thinkingBlock = response.content.find((block) => block.type === "thinking"); + expect(thinkingBlock?.type).toBe("thinking"); + if (!thinkingBlock || thinkingBlock.type !== "thinking") { + throw new Error("Expected thinking block from Claude Opus 4.8"); + } + expect(typeof thinkingBlock.thinkingSignature).toBe("string"); + const thinkingSignature = thinkingBlock.thinkingSignature; + if (!thinkingSignature) { + throw new Error("Expected thinking signature from Claude Opus 4.8"); + } + expect(thinkingSignature.length).toBeGreaterThan(0); + + const text = response.content + .filter((block) => block.type === "text") + .map((block) => block.text) + .join("") + .trim(); + expect(text).toBe("sum=353418362; divisibleBy11=yes"); + }); +}); diff --git a/packages/ai/test/anthropic-sse-parsing.test.ts b/packages/ai/test/anthropic-sse-parsing.test.ts new file mode 100644 index 0000000000000000000000000000000000000000..9982ae82eae49d68911037d290477a3ef4b91269 --- /dev/null +++ b/packages/ai/test/anthropic-sse-parsing.test.ts @@ -0,0 +1,585 @@ +import type Anthropic from "@anthropic-ai/sdk"; +import { Type } from "typebox"; +import { describe, expect, it } from "vitest"; +import { stream as streamAnthropic } from "../src/api/anthropic-messages.ts"; +import { getModel } from "../src/compat.ts"; +import type { Context, ToolCall } from "../src/types.ts"; + +function createSseResponse(events: Array<{ event: string; data: string }>): Response { + const body = events.map(({ event, data }) => `event: ${event}\ndata: ${data}\n`).join("\n"); + return new Response(body, { + status: 200, + headers: { "content-type": "text/event-stream" }, + }); +} + +const minimalAnthropicEvents = [ + { + event: "message_start", + data: JSON.stringify({ + type: "message_start", + message: { + id: "msg_test", + usage: { + input_tokens: 12, + output_tokens: 0, + cache_read_input_tokens: 0, + cache_creation_input_tokens: 0, + }, + }, + }), + }, + { + event: "content_block_start", + data: JSON.stringify({ + type: "content_block_start", + index: 0, + content_block: { type: "text", text: "" }, + }), + }, + { + event: "content_block_delta", + data: JSON.stringify({ + type: "content_block_delta", + index: 0, + delta: { type: "text_delta", text: "Hello" }, + }), + }, + { + event: "content_block_stop", + data: JSON.stringify({ type: "content_block_stop", index: 0 }), + }, + { + event: "message_delta", + data: JSON.stringify({ + type: "message_delta", + delta: { stop_reason: "end_turn" }, + usage: { + input_tokens: 12, + output_tokens: 5, + cache_read_input_tokens: 0, + cache_creation_input_tokens: 0, + }, + }), + }, + { + event: "message_stop", + data: JSON.stringify({ type: "message_stop" }), + }, +]; + +function createFakeAnthropicClient(response: Response): Anthropic { + return { + beta: { + messages: { + create: () => ({ + asResponse: async () => response, + }), + }, + }, + } as unknown as Anthropic; +} + +describe("Anthropic raw SSE parsing", () => { + it("fails safely when Anthropic falls back after output begins", async () => { + const model = getModel("anthropic", "claude-opus-5"); + const response = createSseResponse([ + { + event: "message_start", + data: JSON.stringify({ + type: "message_start", + message: { + id: "msg_fallback", + model: "claude-opus-5", + usage: { input_tokens: 1, output_tokens: 0 }, + }, + }), + }, + { + event: "content_block_start", + data: JSON.stringify({ + type: "content_block_start", + index: 0, + content_block: { type: "text", text: "partial" }, + }), + }, + { event: "content_block_stop", data: JSON.stringify({ type: "content_block_stop", index: 0 }) }, + { + event: "content_block_start", + data: JSON.stringify({ + type: "content_block_start", + index: 1, + content_block: { + type: "fallback", + from: { model: "claude-opus-5" }, + to: { model: "claude-opus-4-8" }, + }, + }), + }, + ]); + + const result = await streamAnthropic( + model, + { messages: [{ role: "user", content: "Hello", timestamp: 1 }] }, + { client: createFakeAnthropicClient(response) }, + ).result(); + + expect(result.stopReason).toBe("error"); + expect(result.errorMessage).toContain("unsupported mid-output model fallback"); + }); + + it("forces streaming after an onPayload replacement", async () => { + let streaming: boolean | undefined; + const client = { + beta: { + messages: { + create: (params: { stream?: boolean }) => { + streaming = params.stream; + return { asResponse: async () => createSseResponse(minimalAnthropicEvents) }; + }, + }, + }, + } as unknown as Anthropic; + + await streamAnthropic( + getModel("anthropic", "claude-fable-5-1"), + { messages: [{ role: "user", content: "Hello", timestamp: 1 }] }, + { + client, + onPayload: (payload) => ({ ...(payload as Record), stream: false }), + }, + ).result(); + + expect(streaming).toBe(true); + }); + + it("omits the interleaved-thinking beta when thinking is disabled", async () => { + let betaFeatures: string[] | undefined; + const client = { + beta: { + messages: { + create: (params: { betas?: string[] }) => { + betaFeatures = params.betas; + return { asResponse: async () => createSseResponse(minimalAnthropicEvents) }; + }, + }, + }, + } as unknown as Anthropic; + + await streamAnthropic( + getModel("openrouter", "anthropic/claude-3-haiku"), + { messages: [{ role: "user", content: "Hello", timestamp: 1 }] }, + { client, thinkingEnabled: false }, + ).result(); + + expect(betaFeatures ?? []).not.toContain("interleaved-thinking-2025-05-14"); + }); + + it("passes managed beta features to injected clients", async () => { + let betaFeatures: string[] | undefined; + const client = { + beta: { + messages: { + create: (params: { betas?: string[] }) => { + betaFeatures = params.betas; + return { asResponse: async () => createSseResponse(minimalAnthropicEvents) }; + }, + }, + }, + } as unknown as Anthropic; + + const result = await streamAnthropic( + getModel("anthropic", "claude-fable-5-1"), + { messages: [{ role: "user", content: "Hello", timestamp: 1 }] }, + { client }, + ).result(); + + expect(result.stopReason).toBe("stop"); + expect(betaFeatures).toContain("mid-conversation-output-config-2026-07-01"); + expect(betaFeatures).toContain("thinking-binding-controls-2026-08-01"); + }); + + it("uses the serving model input transformations from the final stream event", async () => { + const events = minimalAnthropicEvents.map((event) => ({ ...event })); + events[0].data = JSON.stringify({ + type: "message_start", + message: { + id: "msg_transformations", + model: "claude-fable-5-1", + usage: { input_tokens: 12, output_tokens: 0 }, + input_transformations: [ + { type: "thinking_dropped", path: "messages.1.content.0", reason: "prefix_binding_mismatch" }, + ], + }, + }); + const delta = JSON.parse(events[4].data) as Record; + delta.input_transformations = [ + { type: "thinking_dropped", path: "messages.3.content.0", reason: "model_binding_mismatch" }, + ]; + events[4].data = JSON.stringify(delta); + + const result = await streamAnthropic( + getModel("anthropic", "claude-fable-5-1"), + { messages: [{ role: "user", content: "Hello", timestamp: 1 }] }, + { client: createFakeAnthropicClient(createSseResponse(events)) }, + ).result(); + + expect(result.diagnostics).toEqual([ + { + type: "anthropic_input_transformations", + timestamp: expect.any(Number), + details: { + transformations: [ + { + type: "thinking_dropped", + path: "messages.3.content.0", + reason: "model_binding_mismatch", + }, + ], + }, + }, + ]); + }); + it("repairs malformed SSE JSON and malformed streamed tool JSON", async () => { + const model = getModel("anthropic", "claude-haiku-4-5"); + const context: Context = { + messages: [{ role: "user", content: "Use the edit tool.", timestamp: Date.now() }], + tools: [ + { + name: "edit", + description: "Edit a file.", + parameters: Type.Object({ + path: Type.String(), + text: Type.String(), + }), + }, + ], + }; + + const malformedToolJsonDelta = String.raw`{"type":"content_block_delta","index":0,"delta":{"type":"input_json_delta","partial_json":"{\"path\":\"A\H\",\"text\":\"col1 col2\"}"}}`; + + const response = createSseResponse([ + { + event: "message_start", + data: JSON.stringify({ + type: "message_start", + message: { + id: "msg_test", + usage: { + input_tokens: 12, + output_tokens: 0, + cache_read_input_tokens: 0, + cache_creation_input_tokens: 0, + }, + }, + }), + }, + { + event: "content_block_start", + data: JSON.stringify({ + type: "content_block_start", + index: 0, + content_block: { + type: "tool_use", + id: "toolu_test", + name: "edit", + input: {}, + }, + }), + }, + { event: "content_block_delta", data: malformedToolJsonDelta }, + { + event: "content_block_stop", + data: JSON.stringify({ type: "content_block_stop", index: 0 }), + }, + { + event: "message_delta", + data: JSON.stringify({ + type: "message_delta", + delta: { stop_reason: "tool_use" }, + usage: { + input_tokens: 12, + output_tokens: 5, + cache_read_input_tokens: 0, + cache_creation_input_tokens: 0, + }, + }), + }, + { + event: "message_stop", + data: JSON.stringify({ type: "message_stop" }), + }, + ]); + + const stream = streamAnthropic(model, context, { + client: createFakeAnthropicClient(response), + }); + const result = await stream.result(); + + expect(result.stopReason).toBe("toolUse"); + expect(result.errorMessage).toBeUndefined(); + + const toolCall = result.content.find((block): block is ToolCall => block.type === "toolCall"); + expect(toolCall).toBeDefined(); + expect(toolCall?.arguments).toEqual({ + path: "A\\H", + text: "col1\tcol2", + }); + }); + + it("preserves content from content_block_start events", async () => { + const model = getModel("anthropic", "claude-haiku-4-5"); + const context: Context = { + messages: [{ role: "user", content: "Say hello.", timestamp: Date.now() }], + }; + const response = createSseResponse([ + { + event: "message_start", + data: JSON.stringify({ + type: "message_start", + message: { + id: "msg_initial_content", + usage: { + input_tokens: 12, + output_tokens: 0, + cache_read_input_tokens: 0, + cache_creation_input_tokens: 0, + }, + }, + }), + }, + { + event: "content_block_start", + data: JSON.stringify({ + type: "content_block_start", + index: 0, + content_block: { type: "text", text: "Initial text" }, + }), + }, + { + event: "content_block_delta", + data: JSON.stringify({ + type: "content_block_delta", + index: 0, + delta: { type: "text_delta", text: " plus delta" }, + }), + }, + { event: "content_block_stop", data: JSON.stringify({ type: "content_block_stop", index: 0 }) }, + { + event: "content_block_start", + data: JSON.stringify({ + type: "content_block_start", + index: 1, + content_block: { + type: "thinking", + thinking: "Initial thinking", + signature: "initial signature", + }, + }), + }, + { + event: "content_block_delta", + data: JSON.stringify({ + type: "content_block_delta", + index: 1, + delta: { type: "thinking_delta", thinking: " plus delta" }, + }), + }, + { + event: "content_block_delta", + data: JSON.stringify({ + type: "content_block_delta", + index: 1, + delta: { type: "signature_delta", signature: " plus delta" }, + }), + }, + { event: "content_block_stop", data: JSON.stringify({ type: "content_block_stop", index: 1 }) }, + { + event: "message_delta", + data: JSON.stringify({ + type: "message_delta", + delta: { stop_reason: "end_turn" }, + usage: { + input_tokens: 12, + output_tokens: 5, + cache_read_input_tokens: 0, + cache_creation_input_tokens: 0, + }, + }), + }, + { event: "message_stop", data: JSON.stringify({ type: "message_stop" }) }, + ]); + + const stream = streamAnthropic(model, context, { + client: createFakeAnthropicClient(response), + }); + const result = await stream.result(); + + expect(result.content).toEqual([ + { type: "text", text: "Initial text plus delta" }, + { + type: "thinking", + thinking: "Initial thinking plus delta", + thinkingSignature: "initial signature plus delta", + }, + ]); + }); + + it("preserves refusal stop details from message_delta", async () => { + const model = getModel("anthropic", "claude-fable-5"); + const context: Context = { + messages: [{ role: "user", content: "blocked request", timestamp: Date.now() }], + }; + const explanation = + "This request triggered restrictions on violative cyber content and was blocked under Anthropic's Usage Policy. To learn more, provide feedback, or request an exemption based on how you use Claude, visit our help center: https://support.claude.com/en/articles/14604842-real-time-cyber-safeguards-on-claude."; + const response = createSseResponse([ + { + event: "message_start", + data: JSON.stringify({ + type: "message_start", + message: { + id: "msg_01XFUDYJgAACzvnptvVoYEL", + usage: { + input_tokens: 412, + output_tokens: 0, + cache_read_input_tokens: 0, + cache_creation_input_tokens: 0, + }, + }, + }), + }, + { + event: "message_delta", + data: JSON.stringify({ + type: "message_delta", + delta: { + stop_reason: "refusal", + stop_details: { + type: "refusal", + category: "cyber", + explanation, + }, + }, + usage: { + input_tokens: 412, + output_tokens: 0, + cache_read_input_tokens: 0, + cache_creation_input_tokens: 0, + }, + }), + }, + { + event: "message_stop", + data: JSON.stringify({ type: "message_stop" }), + }, + ]); + + const stream = streamAnthropic(model, context, { + client: createFakeAnthropicClient(response), + }); + const result = await stream.result(); + + expect(result.stopReason).toBe("error"); + expect(result.rawStopReason).toBe("refusal"); + expect(result.errorMessage).toBe(explanation); + }); + + it("preserves sensitive stop reasons with a descriptive error message", async () => { + const model = getModel("anthropic", "claude-haiku-4-5"); + const context: Context = { + messages: [{ role: "user", content: "blocked request", timestamp: Date.now() }], + }; + const response = createSseResponse([ + { + event: "message_start", + data: JSON.stringify({ + type: "message_start", + message: { + id: "msg_sensitive", + usage: { + input_tokens: 12, + output_tokens: 0, + cache_read_input_tokens: 0, + cache_creation_input_tokens: 0, + }, + }, + }), + }, + { + event: "message_delta", + data: JSON.stringify({ + type: "message_delta", + delta: { stop_reason: "sensitive" }, + usage: { + input_tokens: 12, + output_tokens: 0, + cache_read_input_tokens: 0, + cache_creation_input_tokens: 0, + }, + }), + }, + { + event: "message_stop", + data: JSON.stringify({ type: "message_stop" }), + }, + ]); + + const stream = streamAnthropic(model, context, { + client: createFakeAnthropicClient(response), + }); + const result = await stream.result(); + + expect(result.stopReason).toBe("error"); + expect(result.rawStopReason).toBe("sensitive"); + expect(result.errorMessage).toBe("Provider stopped with: sensitive"); + }); + + it("treats message_delta without usage as a no-op for usage accumulation", async () => { + const model = getModel("anthropic", "claude-haiku-4-5"); + const context: Context = { + messages: [{ role: "user", content: "Say hello.", timestamp: Date.now() }], + }; + const response = createSseResponse( + minimalAnthropicEvents.map((event) => + event.event === "message_delta" + ? { + event: "message_delta", + data: JSON.stringify({ type: "message_delta", delta: { stop_reason: "end_turn" } }), + } + : event, + ), + ); + + const stream = streamAnthropic(model, context, { + client: createFakeAnthropicClient(response), + }); + const result = await stream.result(); + + expect(result.stopReason).toBe("stop"); + expect(result.errorMessage).toBeUndefined(); + expect(result.content).toEqual([{ type: "text", text: "Hello" }]); + expect(result.usage.input).toBe(12); + expect(result.usage.totalTokens).toBe(12); + }); + + it("ignores unknown SSE events after message_stop", async () => { + const model = getModel("anthropic", "claude-haiku-4-5"); + const context: Context = { + messages: [{ role: "user", content: "Say hello.", timestamp: Date.now() }], + }; + const response = createSseResponse([ + ...minimalAnthropicEvents, + { event: "done", data: "[DONE]" }, + { event: "proxy.stats", data: "not json" }, + ]); + + const stream = streamAnthropic(model, context, { + client: createFakeAnthropicClient(response), + }); + const result = await stream.result(); + + expect(result.stopReason).toBe("stop"); + expect(result.errorMessage).toBeUndefined(); + expect(result.content).toEqual([{ type: "text", text: "Hello" }]); + }); +}); diff --git a/packages/ai/test/anthropic-thinking-binding-e2e.test.ts b/packages/ai/test/anthropic-thinking-binding-e2e.test.ts new file mode 100644 index 0000000000000000000000000000000000000000..07c9885774a3392193b99e4fd33d3913401b5bd7 --- /dev/null +++ b/packages/ai/test/anthropic-thinking-binding-e2e.test.ts @@ -0,0 +1,59 @@ +import { describe, expect, it } from "vitest"; +import { stream } from "../src/api/anthropic-messages.ts"; +import { getModel } from "../src/compat.ts"; +import type { AssistantMessage, Context } from "../src/types.ts"; + +const enabled = Boolean(process.env.ANTHROPIC_API_KEY); +const model = getModel("anthropic", "claude-fable-5-1"); +const user = (content: string, timestamp: number) => ({ role: "user" as const, content, timestamp }); + +function strictBinding(payload: unknown): unknown { + const params = payload as { + thinking?: { + block_binding?: { prefix_mismatch_behavior?: "drop_block" | "error" }; + }; + }; + if (params.thinking?.block_binding) { + params.thinking.block_binding.prefix_mismatch_behavior = "error"; + } + return params; +} + +async function request(context: Context, effort: "low" | "high"): Promise { + return stream(model, context, { + apiKey: process.env.ANTHROPIC_API_KEY, + cacheRetention: "none", + maxTokens: 1536, + thinkingEnabled: true, + thinkingDisplay: "summarized", + effort, + onPayload: strictBinding, + }).result(); +} + +describe.skipIf(!enabled)("Anthropic thinking binding conformance", () => { + it("replays managed effort markers required by signed Fable thinking", { timeout: 120000 }, async () => { + const firstUser = user("Compute 982451653 multiplied by 961748941. Return only the integer.", 1); + const first = await request({ messages: [firstUser] }, "low"); + expect(first.stopReason, first.errorMessage).toBe("stop"); + expect( + first.content.some( + (block) => + block.type === "thinking" && + typeof block.thinkingSignature === "string" && + block.thinkingSignature.length > 0, + ), + ).toBe(true); + expect(first.providerThinkingLevel).toBe("low"); + + const secondUser = user("Reply with exactly: ok", 2); + const exact = await request({ messages: [firstUser, first, secondUser] }, "high"); + expect(exact.stopReason, exact.errorMessage).toBe("stop"); + + const unmanagedHistory: AssistantMessage = { ...first }; + delete unmanagedHistory.providerThinkingLevel; + const missingMarker = await request({ messages: [firstUser, unmanagedHistory, secondUser] }, "high"); + expect(missingMarker.stopReason).toBe("error"); + expect(missingMarker.errorMessage).toContain("Invalid `signature`"); + }); +}); diff --git a/packages/ai/test/anthropic-thinking-disable.test.ts b/packages/ai/test/anthropic-thinking-disable.test.ts new file mode 100644 index 0000000000000000000000000000000000000000..3f55107c862897f712ac720c479b271690e801c2 --- /dev/null +++ b/packages/ai/test/anthropic-thinking-disable.test.ts @@ -0,0 +1,172 @@ +import { describe, expect, it } from "vitest"; +import { getModel, streamSimple } from "../src/compat.ts"; +import type { Context, Model, SimpleStreamOptions } from "../src/types.ts"; + +interface AnthropicThinkingPayload { + thinking?: { type: string; budget_tokens?: number; display?: string }; + output_config?: { effort?: string }; +} + +class PayloadCaptured extends Error { + constructor() { + super("payload captured"); + this.name = "PayloadCaptured"; + } +} + +function makePayloadCaptureContext(): Context { + return { + messages: [{ role: "user", content: "Hello", timestamp: Date.now() }], + }; +} + +async function capturePayload( + model: Model<"anthropic-messages">, + options?: SimpleStreamOptions, +): Promise { + let capturedPayload: AnthropicThinkingPayload | undefined; + const payloadCaptureModel: Model<"anthropic-messages"> = { + ...model, + baseUrl: "http://127.0.0.1:9", + }; + + const s = streamSimple(payloadCaptureModel, makePayloadCaptureContext(), { + ...options, + apiKey: "fake-key", + onPayload: (payload) => { + capturedPayload = payload as AnthropicThinkingPayload; + throw new PayloadCaptured(); + }, + }); + + await s.result(); + + if (!capturedPayload) { + throw new Error("Expected payload to be captured before request failure"); + } + + return capturedPayload; +} + +interface RunResult { + thinkingEventCount: number; + thinkingCharCount: number; + text: string; + contentTypes: string[]; +} + +function makeE2EContext(): Context { + return { + systemPrompt: "You are a precise assistant. Follow the requested output format exactly.", + messages: [ + { + role: "user", + content: + "Before replying, carefully solve 36863 * 5279 internally. Then reply with the word pong repeated exactly 40 times, separated by single spaces. Do not add any other text.", + timestamp: Date.now(), + }, + ], + }; +} + +function countPongs(text: string): number { + return text.match(/\bpong\b/gi)?.length ?? 0; +} + +async function runWithoutReasoning(model: Model<"anthropic-messages">): Promise { + const s = streamSimple(model, makeE2EContext(), { + temperature: 0, + maxTokens: 160, + }); + + let thinkingEventCount = 0; + let thinkingCharCount = 0; + + for await (const event of s) { + if (event.type === "thinking_start" || event.type === "thinking_end") { + thinkingEventCount += 1; + } + if (event.type === "thinking_delta") { + thinkingEventCount += 1; + thinkingCharCount += event.delta.length; + } + } + + const response = await s.result(); + expect(response.stopReason, response.errorMessage).toBe("stop"); + + const text = response.content + .filter((block) => block.type === "text") + .map((block) => block.text) + .join("") + .trim(); + + return { + thinkingEventCount, + thinkingCharCount, + text, + contentTypes: response.content.map((block) => block.type), + }; +} + +describe("Anthropic thinking disable payload", () => { + it("sends thinking.type=disabled for budget-based reasoning models when thinking is off", async () => { + const payload = await capturePayload(getModel("anthropic", "claude-sonnet-4-5")); + + expect(payload.thinking).toEqual({ type: "disabled" }); + expect(payload.output_config).toBeUndefined(); + }); + + it("sends thinking.type=disabled for adaptive reasoning models when thinking is off", async () => { + const payload = await capturePayload(getModel("anthropic", "claude-opus-4-6")); + + expect(payload.thinking).toEqual({ type: "disabled" }); + expect(payload.output_config).toBeUndefined(); + }); + + it("sends thinking.type=disabled for Claude Opus 4.8 when thinking is off", async () => { + const payload = await capturePayload(getModel("anthropic", "claude-opus-4-8")); + + expect(payload.thinking).toEqual({ type: "disabled" }); + expect(payload.output_config).toBeUndefined(); + }); + + it("omits thinking.type=disabled for Claude Fable 5 when thinking is off", async () => { + const payload = await capturePayload(getModel("anthropic", "claude-fable-5")); + + expect(payload.thinking).toBeUndefined(); + expect(payload.output_config).toBeUndefined(); + }); + + it("uses adaptive thinking for Claude Opus 4.8 when reasoning is enabled", async () => { + const payload = await capturePayload(getModel("anthropic", "claude-opus-4-8"), { reasoning: "high" }); + + expect(payload.thinking).toEqual({ type: "adaptive", display: "summarized" }); + expect(payload.output_config).toEqual({ effort: "high" }); + }); + + it("uses adaptive thinking for Claude Sonnet 5 when reasoning is enabled", async () => { + const payload = await capturePayload(getModel("anthropic", "claude-sonnet-5"), { reasoning: "high" }); + + expect(payload.thinking).toEqual({ type: "adaptive", display: "summarized" }); + expect(payload.output_config).toEqual({ effort: "high" }); + }); + + it("maps xhigh reasoning to effort=xhigh for Claude Opus 4.8", async () => { + const payload = await capturePayload(getModel("anthropic", "claude-opus-4-8"), { reasoning: "xhigh" }); + + expect(payload.thinking).toEqual({ type: "adaptive", display: "summarized" }); + expect(payload.output_config).toEqual({ effort: "xhigh" }); + }); +}); + +describe.skipIf(!process.env.ANTHROPIC_API_KEY)("Anthropic thinking disable E2E", () => { + it("disables thinking for Claude reasoning models", { retry: 2, timeout: 30000 }, async () => { + const result = await runWithoutReasoning(getModel("anthropic", "claude-sonnet-4-5")); + + expect(result.thinkingEventCount).toBe(0); + expect(result.thinkingCharCount).toBe(0); + expect(result.contentTypes).not.toContain("thinking"); + expect(countPongs(result.text)).toBeGreaterThanOrEqual(35); + }); +}); diff --git a/packages/ai/test/assistant-message-frame.test.ts b/packages/ai/test/assistant-message-frame.test.ts new file mode 100644 index 0000000000000000000000000000000000000000..ab7f5f9acb259d20e5268abc5e5822a398ec3053 --- /dev/null +++ b/packages/ai/test/assistant-message-frame.test.ts @@ -0,0 +1,606 @@ +import type { ResponseStreamEvent } from "openai/resources/responses/responses.js"; +import { describe, expect, it } from "vitest"; +import { processResponsesStream } from "../src/api/openai-responses-shared.ts"; +import { + type AssistantMessage, + type AssistantMessageEvent, + type AssistantMessageFrame, + AssistantMessageFrameEncoder, + type Model, + reduceAssistantMessageFrames, +} from "../src/index.ts"; +import { AssistantMessageEventStream } from "../src/utils/event-stream.ts"; + +function seed(): AssistantMessage { + return { + role: "assistant", + content: [], + api: "test-api", + provider: "test-provider", + model: "test-model", + usage: { + input: 0, + output: 0, + cacheRead: 0, + cacheWrite: 0, + totalTokens: 0, + cost: { input: 0, output: 0, cacheRead: 0, cacheWrite: 0, total: 0 }, + }, + stopReason: "pending", + timestamp: 1, + }; +} + +function frame(encoder: AssistantMessageFrameEncoder, event: AssistantMessageEvent): AssistantMessageFrame { + const converted = encoder.encode(event); + if (!converted) throw new Error(`Expected ${event.type} event to produce a frame`); + return converted; +} + +describe("assistant message frames", () => { + it("uses authoritative text end content and signature", () => { + const partial = seed(); + const encoder = new AssistantMessageFrameEncoder(); + const frames: AssistantMessageFrame[] = [frame(encoder, { type: "start", partial })]; + partial.content.push({ type: "text", text: "Hello " }); + frames.push(frame(encoder, { type: "text_start", contentIndex: 0, partial })); + partial.content[0] = { type: "text", text: "Hello world", textSignature: "sig-text" }; + frames.push( + frame(encoder, { type: "text_delta", contentIndex: 0, delta: "incorrect", partial }), + frame(encoder, { type: "text_end", contentIndex: 0, content: "Hello world", partial }), + ); + + expect(frames.at(-1)).toEqual({ + type: "text_end", + contentIndex: 0, + content: "Hello world", + textSignature: "sig-text", + }); + expect(reduceAssistantMessageFrames(frames)?.content).toEqual([ + { type: "text", text: "Hello world", textSignature: "sig-text" }, + ]); + }); + + it("preserves provider thinking level from the stream start", () => { + const partial = seed(); + partial.providerThinkingLevel = "high"; + const encoder = new AssistantMessageFrameEncoder(); + const start = frame(encoder, { type: "start", partial }); + + expect(start).toMatchObject({ type: "start", partial: { providerThinkingLevel: "high" } }); + expect(reduceAssistantMessageFrames([start])?.providerThinkingLevel).toBe("high"); + }); + + it("preserves initial and final thinking metadata, including redaction", () => { + const partial = seed(); + const encoder = new AssistantMessageFrameEncoder(); + const frames: AssistantMessageFrame[] = [frame(encoder, { type: "start", partial })]; + partial.content.push({ + type: "thinking", + thinking: "[redacted]", + thinkingSignature: "encrypted-start", + redacted: true, + }); + frames.push(frame(encoder, { type: "thinking_start", contentIndex: 0, partial })); + partial.content[0] = { + type: "thinking", + thinking: "[redacted]", + thinkingSignature: "encrypted-final", + redacted: true, + }; + frames.push(frame(encoder, { type: "thinking_end", contentIndex: 0, content: "[redacted]", partial })); + + expect(frames.at(-1)).toEqual({ + type: "thinking_end", + contentIndex: 0, + content: "[redacted]", + thinkingSignature: "encrypted-final", + redacted: true, + }); + expect(reduceAssistantMessageFrames(frames)?.content[0]).toEqual({ + type: "thinking", + thinking: "[redacted]", + thinkingSignature: "encrypted-final", + redacted: true, + }); + }); + + it("parses unfinished tool JSON once and uses authoritative completed arguments", () => { + const initialFrames: AssistantMessageFrame[] = [ + { type: "start", partial: seed() }, + { + type: "toolcall_start", + contentIndex: 0, + toolCall: { type: "toolCall", id: "initial-id", name: "write", arguments: {} }, + }, + { type: "toolcall_delta", contentIndex: 0, delta: '{"path":"READ' }, + ]; + + expect(reduceAssistantMessageFrames(initialFrames)?.content[0]).toMatchObject({ + type: "toolCall", + arguments: { path: "READ" }, + }); + + const completeFrames: AssistantMessageFrame[] = [ + ...initialFrames, + { type: "toolcall_delta", contentIndex: 0, delta: 'ME.md","lines":[1,2]}' }, + { + type: "toolcall_end", + contentIndex: 0, + id: "final-id", + name: "write_file", + arguments: { path: "final.md", lines: [3] }, + thoughtSignature: "thought", + namespace: "files", + }, + ]; + expect(reduceAssistantMessageFrames(completeFrames)?.content[0]).toEqual({ + type: "toolCall", + id: "final-id", + name: "write_file", + arguments: { path: "final.md", lines: [3] }, + thoughtSignature: "thought", + namespace: "files", + }); + }); + + it("round-trips OpenAI Responses content supplied only by authoritative end events", async () => { + const output = seed(); + output.api = "openai-responses"; + output.provider = "openai"; + const model: Model<"openai-responses"> = { + id: output.model, + name: "Test", + api: "openai-responses", + provider: "openai", + baseUrl: "https://api.openai.com/v1", + reasoning: false, + input: ["text"], + cost: { input: 0, output: 0, cacheRead: 0, cacheWrite: 0 }, + contextWindow: 1000, + maxTokens: 100, + }; + const events: ResponseStreamEvent[] = [ + { + type: "response.output_item.added", + sequence_number: 0, + output_index: 0, + item: { type: "message", id: "msg", role: "assistant", status: "in_progress", content: [] }, + } as ResponseStreamEvent, + { + type: "response.output_item.done", + sequence_number: 1, + output_index: 0, + item: { + type: "message", + id: "msg", + role: "assistant", + status: "completed", + content: [{ type: "output_text", text: "final text", annotations: [] }], + }, + } as ResponseStreamEvent, + { + type: "response.output_item.added", + sequence_number: 2, + output_index: 1, + item: { + type: "function_call", + id: "fc", + call_id: "call", + name: "lookup", + arguments: "", + }, + } as ResponseStreamEvent, + { + type: "response.output_item.done", + sequence_number: 3, + output_index: 1, + item: { + type: "function_call", + id: "fc", + call_id: "call", + name: "lookup", + arguments: '{"query":"pi"}', + }, + } as ResponseStreamEvent, + { + type: "response.completed", + sequence_number: 4, + response: { id: "response", status: "completed", output: [] }, + } as unknown as ResponseStreamEvent, + ]; + async function* source(): AsyncGenerator { + for (const event of events) yield event; + } + + const encoder = new AssistantMessageFrameEncoder(); + const frames: AssistantMessageFrame[] = [frame(encoder, { type: "start", partial: output })]; + const stream = new AssistantMessageEventStream(); + const push = stream.push.bind(stream); + stream.push = (event) => { + const converted = encoder.encode(event); + if (converted) frames.push(converted); + push(event); + }; + await processResponsesStream(source(), output, stream, model); + + expect(reduceAssistantMessageFrames(frames)?.content).toEqual(output.content); + }); + + it("reconciles queued text events against one advanced live partial without duplicate content", () => { + const partial = seed(); + const events: AssistantMessageEvent[] = [{ type: "start", partial }]; + const text = { type: "text" as const, text: "" }; + partial.content.push(text); + events.push({ type: "text_start", contentIndex: 0, partial }); + for (const delta of ["Hel", "lo", " ", "world"]) { + text.text += delta; + events.push({ type: "text_delta", contentIndex: 0, delta, partial }); + } + + const encoder = new AssistantMessageFrameEncoder(); + const frames = events.flatMap((event) => { + const encoded = encoder.encode(event); + return encoded === undefined ? [] : [encoded]; + }); + + expect(frames.map((item) => item.type)).toEqual(["start", "text_start"]); + expect(frames[0]).toMatchObject({ type: "start", partial: { content: [], stopReason: "pending" } }); + expect(reduceAssistantMessageFrames(frames)?.content).toEqual([{ type: "text", text: "Hello world" }]); + }); + + it("trims only the covered prefix when a start snapshot lands inside a delta", () => { + const partial = seed(); + const encoder = new AssistantMessageFrameEncoder(); + const frames: AssistantMessageFrame[] = [frame(encoder, { type: "start", partial })]; + const text = { type: "text" as const, text: "Hel" }; + partial.content.push(text); + frames.push(frame(encoder, { type: "text_start", contentIndex: 0, partial })); + expect(encoder.encode({ type: "text_delta", contentIndex: 0, delta: "He", partial })).toBeUndefined(); + const remainder = encoder.encode({ type: "text_delta", contentIndex: 0, delta: "llo", partial }); + if (remainder === undefined) throw new Error("Expected uncovered text delta"); + frames.push(remainder); + + expect(remainder).toEqual({ type: "text_delta", contentIndex: 0, delta: "lo" }); + expect(reduceAssistantMessageFrames(frames)?.content).toEqual([{ type: "text", text: "Hello" }]); + }); + + it("checkpoints queued tool JSON without replaying covered deltas", () => { + const partial = seed(); + const toolCall = { type: "toolCall" as const, id: "call", name: "write", arguments: {} }; + const events: AssistantMessageEvent[] = [{ type: "start", partial }]; + partial.content.push(toolCall); + events.push({ type: "toolcall_start", contentIndex: 0, partial }); + toolCall.arguments = { path: "README.md" }; + events.push( + { type: "toolcall_delta", contentIndex: 0, delta: '{"path":"READ', partial }, + { type: "toolcall_delta", contentIndex: 0, delta: 'ME.md"}', partial }, + ); + + const encoder = new AssistantMessageFrameEncoder(); + const frames = events.flatMap((event) => { + const encoded = encoder.encode(event); + return encoded === undefined ? [] : [encoded]; + }); + expect(frames.map((item) => item.type)).toEqual(["start", "toolcall_start", "toolcall_checkpoint"]); + expect(frames.at(-1)).toEqual({ + type: "toolcall_checkpoint", + contentIndex: 0, + json: '{"path":"README.md"}', + }); + expect(reduceAssistantMessageFrames(frames)?.content).toEqual([ + { type: "toolCall", id: "call", name: "write", arguments: { path: "README.md" } }, + ]); + }); + + it("resumes legacy grammar tool JSON from initial arguments", () => { + const partial = seed(); + const encoder = new AssistantMessageFrameEncoder(); + const frames: AssistantMessageFrame[] = [frame(encoder, { type: "start", partial })]; + const toolCall = { type: "toolCall" as const, id: "call", name: "bash", arguments: { input: "a" } }; + partial.content.push(toolCall); + frames.push(frame(encoder, { type: "toolcall_start", contentIndex: 0, partial })); + toolCall.arguments = { input: "ab" }; + frames.push( + frame(encoder, { + type: "toolcall_delta", + contentIndex: 0, + delta: '{"input":"ab', + partial, + }), + ); + toolCall.arguments = { input: "abc" }; + frames.push( + frame(encoder, { + type: "toolcall_delta", + contentIndex: 0, + delta: 'c"}', + partial, + }), + ); + + expect(frames.slice(2)).toEqual([ + { type: "toolcall_checkpoint", contentIndex: 0, json: '{"input":"ab' }, + { type: "toolcall_delta", contentIndex: 0, delta: 'c"}' }, + ]); + expect(reduceAssistantMessageFrames(frames)?.content).toEqual([ + { type: "toolCall", id: "call", name: "bash", arguments: { input: "abc" } }, + ]); + }); + + it("streams tool JSON compactly from an empty argument start", () => { + const partial = seed(); + const encoder = new AssistantMessageFrameEncoder(); + const frames: AssistantMessageFrame[] = [frame(encoder, { type: "start", partial })]; + const toolCall = { type: "toolCall" as const, id: "call", name: "bash", arguments: {} }; + partial.content.push(toolCall); + frames.push(frame(encoder, { type: "toolcall_start", contentIndex: 0, partial })); + toolCall.arguments = { command: "ls -la /tmp" }; + frames.push( + frame(encoder, { + type: "toolcall_delta", + contentIndex: 0, + delta: '{"command":"ls -la /tmp"}', + partial, + }), + ); + + expect(frames.at(-1)).toEqual({ + type: "toolcall_delta", + contentIndex: 0, + delta: '{"command":"ls -la /tmp"}', + }); + expect(reduceAssistantMessageFrames(frames)?.content[0]).toMatchObject({ + type: "toolCall", + arguments: { command: "ls -la /tmp" }, + }); + }); + + it("accepts a pre-generation error but rejects success or updates before start", () => { + const failed = seed(); + failed.stopReason = "error"; + failed.errorMessage = "setup failed"; + expect( + new AssistantMessageFrameEncoder().encode({ type: "error", reason: "error", error: failed }), + ).toBeUndefined(); + + const completed = seed(); + completed.stopReason = "stop"; + expect(() => + new AssistantMessageFrameEncoder().encode({ type: "done", reason: "stop", message: completed }), + ).toThrow("done event appears before start"); + expect(() => + new AssistantMessageFrameEncoder().encode({ + type: "text_delta", + contentIndex: 0, + delta: "x", + partial: seed(), + }), + ).toThrow("text_delta event appears before start"); + }); + + it("treats end signature metadata, including absence, as authoritative", () => { + const frames: AssistantMessageFrame[] = [ + { type: "start", partial: seed() }, + { + type: "text_start", + contentIndex: 0, + content: { type: "text", text: "", textSignature: "stale-text" }, + }, + { type: "text_end", contentIndex: 0, content: "" }, + { + type: "thinking_start", + contentIndex: 1, + content: { + type: "thinking", + thinking: "", + thinkingSignature: "stale-thinking", + redacted: true, + }, + }, + { type: "thinking_end", contentIndex: 1, content: "", thinkingSignature: "", redacted: false }, + { + type: "toolcall_start", + contentIndex: 2, + toolCall: { + type: "toolCall", + id: "call", + name: "read", + arguments: {}, + thoughtSignature: "stale-tool", + namespace: "stale-namespace", + }, + }, + { type: "toolcall_end", contentIndex: 2, id: "call", name: "read", arguments: {} }, + ]; + + expect(reduceAssistantMessageFrames(frames)?.content).toEqual([ + { type: "text", text: "" }, + { type: "thinking", thinking: "", thinkingSignature: "", redacted: false }, + { type: "toolCall", id: "call", name: "read", arguments: {} }, + ]); + }); + + it("stores authoritative final arguments in toolcall_end frames", () => { + const partial = seed(); + const toolCall = { + type: "toolCall" as const, + id: "call-1", + name: "read", + arguments: { path: "README.md" }, + thoughtSignature: "thought", + namespace: "files", + }; + partial.content.push(toolCall); + + const encoder = new AssistantMessageFrameEncoder(); + frame(encoder, { type: "start", partial }); + frame(encoder, { type: "toolcall_start", contentIndex: 0, partial }); + const end = frame(encoder, { type: "toolcall_end", contentIndex: 0, toolCall, partial }); + expect(end).toEqual({ + type: "toolcall_end", + contentIndex: 0, + id: "call-1", + name: "read", + arguments: { path: "README.md" }, + thoughtSignature: "thought", + namespace: "files", + }); + }); + + it("whitelists public block fields from provider-shaped partials", () => { + const partial = seed(); + const text = { type: "text" as const, text: "visible", textSignature: "text-sig", index: 4 }; + const thinking = { + type: "thinking" as const, + thinking: "reasoning", + thinkingSignature: "thinking-sig", + redacted: false, + index: 5, + }; + const toolCall = { + type: "toolCall" as const, + id: "call", + name: "run", + arguments: { value: 1 }, + thoughtSignature: "tool-sig", + namespace: "tools", + partialJson: '{"value":', + streamIndex: 6, + }; + partial.content.push(text, thinking, toolCall); + const partialWithScratch = partial as AssistantMessage & { outputIndex?: number }; + partialWithScratch.outputIndex = 3; + + const encoder = new AssistantMessageFrameEncoder(); + const start = frame(encoder, { type: "start", partial }); + const textStart = frame(encoder, { type: "text_start", contentIndex: 0, partial }); + const thinkingStart = frame(encoder, { type: "thinking_start", contentIndex: 1, partial }); + const toolStart = frame(encoder, { type: "toolcall_start", contentIndex: 2, partial }); + + expect(start.type === "start" && start.partial.content).toEqual([]); + expect(start).not.toHaveProperty("partial.outputIndex"); + expect(textStart).not.toHaveProperty("content.index"); + expect(thinkingStart).not.toHaveProperty("content.index"); + expect(toolStart).not.toHaveProperty("toolCall.partialJson"); + expect(toolStart).not.toHaveProperty("toolCall.streamIndex"); + }); + + it("supports interleaved streams by contentIndex", () => { + const frames: AssistantMessageFrame[] = [ + { type: "start", partial: seed() }, + { type: "text_start", contentIndex: 0, content: { type: "text", text: "" } }, + { + type: "toolcall_start", + contentIndex: 1, + toolCall: { type: "toolCall", id: "call", name: "lookup", arguments: {} }, + }, + { type: "thinking_start", contentIndex: 2, content: { type: "thinking", thinking: "" } }, + { type: "text_delta", contentIndex: 0, delta: "answer" }, + { type: "toolcall_delta", contentIndex: 1, delta: '{"query":"pi"}' }, + { type: "thinking_delta", contentIndex: 2, delta: "check" }, + { type: "toolcall_end", contentIndex: 1, id: "call", name: "lookup", arguments: { query: "pi" } }, + { type: "text_end", contentIndex: 0, content: "answer" }, + { type: "thinking_end", contentIndex: 2, content: "check" }, + ]; + + expect(reduceAssistantMessageFrames(frames)?.content).toEqual([ + { type: "text", text: "answer" }, + { type: "toolCall", id: "call", name: "lookup", arguments: { query: "pi" } }, + { type: "thinking", thinking: "check" }, + ]); + }); + + it("snapshots mutable event data and keeps reduction pure", () => { + const partial = seed(); + partial.diagnostics = [{ type: "test", timestamp: 2, details: { value: "original" } }]; + const encoder = new AssistantMessageFrameEncoder(); + const start = frame(encoder, { type: "start", partial }); + partial.diagnostics[0]!.details!.value = "mutated"; + partial.usage.cost.total = 99; + + partial.content.push({ + type: "toolCall", + id: "call", + name: "run", + arguments: { nested: { value: "original" } }, + }); + const toolStart = frame(encoder, { type: "toolcall_start", contentIndex: 0, partial }); + const sourceTool = partial.content[0]; + if (sourceTool?.type !== "toolCall") throw new Error("Expected source tool call"); + (sourceTool.arguments.nested as Record).value = "mutated"; + + const reduced = reduceAssistantMessageFrames([start, toolStart]); + expect(reduced?.diagnostics?.[0]?.details?.value).toBe("original"); + expect(reduced?.usage.cost.total).toBe(0); + expect(reduced?.content[0]).toMatchObject({ arguments: { nested: { value: "original" } } }); + + if (reduced?.content[0]?.type !== "toolCall") throw new Error("Expected reduced tool call"); + reduced.content[0].arguments.nested = "changed-output"; + expect(toolStart.type === "toolcall_start" && toolStart.toolCall.arguments.nested).toEqual({ + value: "original", + }); + }); + + it("omits terminal events because settlement is separate", () => { + const message = seed(); + const completed = new AssistantMessageFrameEncoder(); + completed.encode({ type: "start", partial: message }); + message.stopReason = "stop"; + expect(completed.encode({ type: "done", reason: "stop", message })).toBeUndefined(); + message.stopReason = "error"; + message.errorMessage = "failed"; + expect( + new AssistantMessageFrameEncoder().encode({ type: "error", reason: "error", error: message }), + ).toBeUndefined(); + }); + + it("returns undefined when there is no start frame", () => { + expect(reduceAssistantMessageFrames([])).toBeUndefined(); + expect(reduceAssistantMessageFrames([{ type: "text_delta", contentIndex: 0, delta: "x" }])).toBeUndefined(); + }); + + it("rejects frames before start, wrong block kinds, duplicate ends, and index gaps", () => { + expect(() => + reduceAssistantMessageFrames([ + { type: "text_delta", contentIndex: 0, delta: "x" }, + { type: "start", partial: seed() }, + ]), + ).toThrow("before the start frame"); + expect(() => + reduceAssistantMessageFrames([ + { type: "start", partial: seed() }, + { + type: "toolcall_start", + contentIndex: 0, + toolCall: { type: "toolCall", id: "call", name: "run", arguments: {} }, + }, + { type: "text_delta", contentIndex: 0, delta: "wrong" }, + ]), + ).toThrow("expected text block"); + expect(() => + reduceAssistantMessageFrames([ + { type: "start", partial: seed() }, + { type: "text_start", contentIndex: 0, content: { type: "text", text: "" } }, + { type: "text_end", contentIndex: 0, content: "" }, + { type: "text_end", contentIndex: 0, content: "" }, + ]), + ).toThrow("follows the end"); + expect(() => + reduceAssistantMessageFrames([ + { type: "start", partial: seed() }, + { type: "text_start", contentIndex: 1, content: { type: "text", text: "" } }, + ]), + ).toThrow("would leave a gap"); + }); + + it("rejects conversion events whose contentIndex points to the wrong block kind", () => { + const partial = seed(); + const encoder = new AssistantMessageFrameEncoder(); + encoder.encode({ type: "start", partial }); + partial.content.push({ type: "thinking", thinking: "" }); + expect(() => encoder.encode({ type: "text_start", contentIndex: 0, partial })).toThrow( + "text_start event points to thinking block", + ); + }); +}); diff --git a/packages/ai/test/azure-openai-base-url.test.ts b/packages/ai/test/azure-openai-base-url.test.ts new file mode 100644 index 0000000000000000000000000000000000000000..b317fd79f002319c4cbc5f8b2e23a7ba3e6a26c3 --- /dev/null +++ b/packages/ai/test/azure-openai-base-url.test.ts @@ -0,0 +1,227 @@ +import { arch, platform, release } from "node:os"; +import { Type } from "typebox"; +import { afterEach, beforeEach, describe, expect, it, vi } from "vitest"; +import { stream as streamAzureOpenAIResponses } from "../src/api/azure-openai-responses.ts"; +import { getModel } from "../src/compat.ts"; +import type { Context, Model } from "../src/types.ts"; + +interface CapturedAzureClientOptions { + apiKey: string; + apiVersion: string; + dangerouslyAllowBrowser: boolean; + defaultHeaders?: Record; + baseURL: string; +} + +interface CapturedAzureResponsesPayload { + prompt_cache_key?: string; + store?: boolean; + tools?: Array<{ strict?: boolean }>; +} + +const azureMock = vi.hoisted(() => ({ + constructorCalls: [] as CapturedAzureClientOptions[], + lastParams: undefined as CapturedAzureResponsesPayload | undefined, +})); + +vi.mock("openai", () => { + class AzureOpenAI { + responses = { + create: (params: CapturedAzureResponsesPayload) => { + azureMock.lastParams = params; + throw new Error("mock create"); + }, + }; + + constructor(config: CapturedAzureClientOptions) { + azureMock.constructorCalls.push(config); + } + } + + return { AzureOpenAI }; +}); + +const PI_USER_AGENT = `pi (${platform()} ${release()}; ${arch()})`; + +const context: Context = { + messages: [{ role: "user", content: "hello", timestamp: Date.now() }], +}; + +const originalAzureOpenAIBaseUrl = process.env.AZURE_OPENAI_BASE_URL; +const originalAzureOpenAIResourceName = process.env.AZURE_OPENAI_RESOURCE_NAME; +const originalAzureOpenAIApiVersion = process.env.AZURE_OPENAI_API_VERSION; +const originalAzureOpenAIApiKey = process.env.AZURE_OPENAI_API_KEY; + +beforeEach(() => { + azureMock.constructorCalls.length = 0; + azureMock.lastParams = undefined; + delete process.env.AZURE_OPENAI_BASE_URL; + delete process.env.AZURE_OPENAI_RESOURCE_NAME; + delete process.env.AZURE_OPENAI_API_VERSION; + delete process.env.AZURE_OPENAI_API_KEY; +}); + +afterEach(() => { + if (originalAzureOpenAIBaseUrl === undefined) { + delete process.env.AZURE_OPENAI_BASE_URL; + } else { + process.env.AZURE_OPENAI_BASE_URL = originalAzureOpenAIBaseUrl; + } + + if (originalAzureOpenAIResourceName === undefined) { + delete process.env.AZURE_OPENAI_RESOURCE_NAME; + } else { + process.env.AZURE_OPENAI_RESOURCE_NAME = originalAzureOpenAIResourceName; + } + + if (originalAzureOpenAIApiVersion === undefined) { + delete process.env.AZURE_OPENAI_API_VERSION; + } else { + process.env.AZURE_OPENAI_API_VERSION = originalAzureOpenAIApiVersion; + } + + if (originalAzureOpenAIApiKey === undefined) { + delete process.env.AZURE_OPENAI_API_KEY; + } else { + process.env.AZURE_OPENAI_API_KEY = originalAzureOpenAIApiKey; + } +}); + +async function captureClientBaseUrl(baseUrl: string): Promise { + process.env.AZURE_OPENAI_BASE_URL = baseUrl; + const model = getModel("azure-openai-responses", "gpt-4o-mini"); + await streamAzureOpenAIResponses(model, context, { apiKey: "test-api-key" }).result(); + expect(azureMock.constructorCalls).toHaveLength(1); + return azureMock.constructorCalls[0].baseURL; +} + +async function captureClientHeaders(headers?: Record): Promise> { + const model = getModel("azure-openai-responses", "gpt-4o-mini"); + await streamAzureOpenAIResponses(model, context, { + apiKey: "test-api-key", + azureBaseUrl: "https://my-resource.openai.azure.com", + headers, + }).result(); + expect(azureMock.constructorCalls).toHaveLength(1); + return azureMock.constructorCalls[0].defaultHeaders ?? {}; +} + +describe("azure-openai-responses base URL normalization", () => { + it("normalizes Cognitive Services root endpoints to /openai/v1", async () => { + const baseURL = await captureClientBaseUrl("https://marc-quicktests-resource.cognitiveservices.azure.com"); + expect(baseURL).toBe("https://marc-quicktests-resource.cognitiveservices.azure.com/openai/v1"); + }); + + it("normalizes Microsoft Foundry root endpoints to /openai/v1", async () => { + const baseURL = await captureClientBaseUrl("https://marc-quicktests-resource.ai.azure.com"); + expect(baseURL).toBe("https://marc-quicktests-resource.ai.azure.com/openai/v1"); + }); + + it("normalizes Azure OpenAI root endpoints to /openai/v1", async () => { + const baseURL = await captureClientBaseUrl("https://my-resource.openai.azure.com"); + expect(baseURL).toBe("https://my-resource.openai.azure.com/openai/v1"); + }); + + it("normalizes /openai to /openai/v1", async () => { + const baseURL = await captureClientBaseUrl("https://my-resource.cognitiveservices.azure.com/openai"); + expect(baseURL).toBe("https://my-resource.cognitiveservices.azure.com/openai/v1"); + }); + + it("preserves /openai/v1 endpoints", async () => { + const baseURL = await captureClientBaseUrl("https://my-resource.cognitiveservices.azure.com/openai/v1"); + expect(baseURL).toBe("https://my-resource.cognitiveservices.azure.com/openai/v1"); + }); + + it("normalizes /openai/v1/responses to /openai/v1", async () => { + const baseURL = await captureClientBaseUrl("https://my-resource.services.ai.azure.com/openai/v1/responses"); + expect(baseURL).toBe("https://my-resource.services.ai.azure.com/openai/v1"); + }); + + it("preserves explicit non-Azure proxy paths", async () => { + const baseURL = await captureClientBaseUrl("https://my-proxy.example.com/v1"); + expect(baseURL).toBe("https://my-proxy.example.com/v1"); + }); + + it("strips query params when normalizing Azure host URLs", async () => { + const baseURL = await captureClientBaseUrl("https://my-resource.openai.azure.com/openai?api-version=2024-12-01"); + expect(baseURL).toBe("https://my-resource.openai.azure.com/openai/v1"); + }); + + it("preserves query params on non-Azure proxy URLs", async () => { + const baseURL = await captureClientBaseUrl("https://my-proxy.example.com/v1?custom=true"); + expect(baseURL).toBe("https://my-proxy.example.com/v1?custom=true"); + }); + + it("throws on invalid URLs", async () => { + process.env.AZURE_OPENAI_BASE_URL = "not-a-url"; + const model = getModel("azure-openai-responses", "gpt-4o-mini"); + const result = await streamAzureOpenAIResponses(model, context, { apiKey: "test-api-key" }).result(); + expect(result.stopReason).toBe("error"); + expect(result.errorMessage).toContain("Invalid Azure OpenAI base URL"); + }); + + it("clamps prompt_cache_key to OpenAI's 64-character limit", async () => { + const model = getModel("azure-openai-responses", "gpt-4o-mini"); + await streamAzureOpenAIResponses(model, context, { + apiKey: "test-api-key", + azureBaseUrl: "https://my-resource.openai.azure.com", + sessionId: "x".repeat(67), + }).result(); + + expect(azureMock.lastParams?.prompt_cache_key).toBe("x".repeat(64)); + }); + + it("disables server-side response storage", async () => { + const model = getModel("azure-openai-responses", "gpt-4o-mini"); + await streamAzureOpenAIResponses(model, context, { + apiKey: "test-api-key", + azureBaseUrl: "https://my-resource.openai.azure.com", + }).result(); + + expect(azureMock.lastParams?.store).toBe(false); + }); + + it("honors supportsStrictMode: false", async () => { + const baseModel = getModel("azure-openai-responses", "gpt-4o-mini"); + const model: Model<"azure-openai-responses"> = { + ...baseModel, + compat: { ...baseModel.compat, supportsStrictMode: false }, + }; + + await streamAzureOpenAIResponses( + model, + { + ...context, + tools: [ + { + name: "preferred", + description: "Preferred constrained tool", + parameters: Type.Object({ value: Type.String() }), + constrainedSampling: { type: "json_schema", strict: "prefer" }, + }, + ], + }, + { apiKey: "test-api-key", azureBaseUrl: "https://my-resource.openai.azure.com" }, + ).result(); + + expect(azureMock.lastParams?.tools?.[0]).not.toHaveProperty("strict"); + }); + + it("builds correct default URL from AZURE_OPENAI_RESOURCE_NAME", async () => { + process.env.AZURE_OPENAI_RESOURCE_NAME = "my-resource"; + const model = getModel("azure-openai-responses", "gpt-4o-mini"); + await streamAzureOpenAIResponses(model, context, { apiKey: "test-api-key" }).result(); + expect(azureMock.constructorCalls).toHaveLength(1); + expect(azureMock.constructorCalls[0].baseURL).toBe("https://my-resource.openai.azure.com/openai/v1"); + }); +}); + +describe("azure-openai-responses user agent", () => { + it("uses pi's User-Agent by default", async () => { + expect((await captureClientHeaders())["User-Agent"]).toBe(PI_USER_AGENT); + }); + + it("lets explicit headers override the default User-Agent", async () => { + expect((await captureClientHeaders({ "User-Agent": "custom-agent" }))["User-Agent"]).toBe("custom-agent"); + }); +}); diff --git a/packages/ai/test/azure-openai-tool-choice.test.ts b/packages/ai/test/azure-openai-tool-choice.test.ts new file mode 100644 index 0000000000000000000000000000000000000000..3ce768e1c63db995752b496ac82585716111bac7 --- /dev/null +++ b/packages/ai/test/azure-openai-tool-choice.test.ts @@ -0,0 +1,79 @@ +import { Type } from "typebox"; +import { describe, expect, it } from "vitest"; +import { stream, streamSimple } from "../src/api/azure-openai-responses.ts"; +import type { Model } from "../src/types.ts"; + +const model: Model<"azure-openai-responses"> = { + id: "test-deployment", + name: "Test Deployment", + api: "azure-openai-responses", + provider: "azure-openai-responses", + baseUrl: "http://127.0.0.1:9/openai/v1", + reasoning: false, + input: ["text"], + cost: { input: 0, output: 0, cacheRead: 0, cacheWrite: 0 }, + contextWindow: 10_000, + maxTokens: 1_000, +}; + +describe("Azure OpenAI tool choice", () => { + it("forwards provider-specific tool choice while preserving tool definitions", async () => { + let payload: unknown; + const result = stream( + model, + { + messages: [{ role: "user", content: "Summarize this", timestamp: 1 }], + tools: [ + { + name: "read", + description: "Read a file", + parameters: Type.Object({ path: Type.String() }), + }, + ], + }, + { + apiKey: "test-key", + toolChoice: "required", + onPayload: (requestPayload) => { + payload = requestPayload; + throw new Error("payload captured"); + }, + }, + ); + + await result.result(); + + expect(payload).toMatchObject({ tool_choice: "required" }); + expect((payload as { tools?: unknown[] }).tools).toHaveLength(1); + }); + + it("forwards provider-neutral tool choice from simple options", async () => { + let payload: unknown; + const result = streamSimple( + model, + { + messages: [{ role: "user", content: "Summarize this", timestamp: 1 }], + tools: [ + { + name: "read", + description: "Read a file", + parameters: Type.Object({ path: Type.String() }), + }, + ], + }, + { + apiKey: "test-key", + toolChoice: "none", + onPayload: (requestPayload) => { + payload = requestPayload; + throw new Error("payload captured"); + }, + }, + ); + + await result.result(); + + expect(payload).toMatchObject({ tool_choice: "none" }); + expect((payload as { tools?: unknown[] }).tools).toHaveLength(1); + }); +}); diff --git a/packages/ai/test/azure-utils.ts b/packages/ai/test/azure-utils.ts new file mode 100644 index 0000000000000000000000000000000000000000..d83b198a638bcbdc6bcd0d8de113e5fe98d06696 --- /dev/null +++ b/packages/ai/test/azure-utils.ts @@ -0,0 +1,28 @@ +/** + * Utility functions for Azure OpenAI tests + */ + +function parseDeploymentNameMap(value: string | undefined): Map { + const map = new Map(); + if (!value) return map; + for (const entry of value.split(",")) { + const trimmed = entry.trim(); + if (!trimmed) continue; + const [modelId, deploymentName] = trimmed.split("=", 2); + if (!modelId || !deploymentName) continue; + map.set(modelId.trim(), deploymentName.trim()); + } + return map; +} + +export function hasAzureOpenAICredentials(): boolean { + const hasKey = !!process.env.AZURE_OPENAI_API_KEY; + const hasBaseUrl = !!(process.env.AZURE_OPENAI_BASE_URL || process.env.AZURE_OPENAI_RESOURCE_NAME); + return hasKey && hasBaseUrl; +} + +export function resolveAzureDeploymentName(modelId: string): string | undefined { + const mapValue = process.env.AZURE_OPENAI_DEPLOYMENT_NAME_MAP; + if (!mapValue) return undefined; + return parseDeploymentNameMap(mapValue).get(modelId); +} diff --git a/packages/ai/test/baseten-models.test.ts b/packages/ai/test/baseten-models.test.ts new file mode 100644 index 0000000000000000000000000000000000000000..97fcc2e24addc5b66efde0db59d33e9e3b7cae37 --- /dev/null +++ b/packages/ai/test/baseten-models.test.ts @@ -0,0 +1,106 @@ +import { afterEach, describe, expect, it } from "vitest"; +import { getModel, streamSimple } from "../src/compat.ts"; +import { findEnvKeys, getEnvApiKey } from "../src/env-api-keys.ts"; +import { getSupportedThinkingLevels } from "../src/models.ts"; + +const originalBasetenApiKey = process.env.BASETEN_API_KEY; + +afterEach(() => { + if (originalBasetenApiKey === undefined) { + delete process.env.BASETEN_API_KEY; + } else { + process.env.BASETEN_API_KEY = originalBasetenApiKey; + } +}); + +describe("Baseten models", () => { + it("keeps both GLM 5.2 endpoints text-only", () => { + expect(getModel("baseten", "zai-org/GLM-5.2").input).toEqual(["text"]); + expect(getModel("baseten", "zai-org/GLM-5.2-Fast").input).toEqual(["text"]); + }); + + it("models Kimi K2.6 reasoning as an explicit off/on toggle", async () => { + const model = getModel("baseten", "moonshotai/Kimi-K2.6"); + let payload: Record | undefined; + + expect(model.thinkingLevelMap).toEqual({ + off: "off", + minimal: null, + low: null, + medium: null, + high: "high", + xhigh: null, + max: null, + }); + expect(model.compat).toMatchObject({ + supportsReasoningEffort: false, + thinkingFormat: "baseten", + chatTemplateArgs: { enable_thinking: { $var: "thinking.enabled" } }, + }); + expect(getSupportedThinkingLevels(model)).toEqual(["off", "high"]); + + await streamSimple( + model, + { messages: [{ role: "user", content: "test", timestamp: 0 }] }, + { + apiKey: "test-baseten-key", + reasoning: "high", + onPayload: (value) => { + payload = value as Record; + throw new Error("payload captured"); + }, + }, + ).result(); + + expect(payload?.chat_template_args).toEqual({ enable_thinking: true }); + expect(payload?.reasoning_effort).toBeUndefined(); + }); + + it("sends Baseten chat_template_args with reasoning effort", async () => { + const model = getModel("baseten", "zai-org/GLM-5.2"); + let payload: Record | undefined; + + await streamSimple( + model, + { messages: [{ role: "user", content: "test", timestamp: 0 }] }, + { + apiKey: "test-baseten-key", + reasoning: "high", + onPayload: (value) => { + payload = value as Record; + throw new Error("payload captured"); + }, + }, + ).result(); + + expect(payload?.chat_template_args).toEqual({ enable_thinking: true }); + expect(payload?.reasoning_effort).toBe("high"); + }); + + it("disables Baseten opt-in reasoning when thinking is off", async () => { + const model = getModel("baseten", "zai-org/GLM-5.2"); + let payload: Record | undefined; + + await streamSimple( + model, + { messages: [{ role: "user", content: "test", timestamp: 0 }] }, + { + apiKey: "test-baseten-key", + onPayload: (value) => { + payload = value as Record; + throw new Error("payload captured"); + }, + }, + ).result(); + + expect(payload?.chat_template_args).toEqual({ enable_thinking: false }); + expect(payload?.reasoning_effort).toBe("none"); + }); + + it("resolves BASETEN_API_KEY from the environment", () => { + process.env.BASETEN_API_KEY = "test-baseten-key"; + + expect(findEnvKeys("baseten")).toEqual(["BASETEN_API_KEY"]); + expect(getEnvApiKey("baseten")).toBe("test-baseten-key"); + }); +}); diff --git a/packages/ai/test/bedrock-credentials.test.ts b/packages/ai/test/bedrock-credentials.test.ts new file mode 100644 index 0000000000000000000000000000000000000000..bcf7ceef8522e2b145c61e91939d48cb88bdad13 --- /dev/null +++ b/packages/ai/test/bedrock-credentials.test.ts @@ -0,0 +1,115 @@ +import { afterEach, describe, expect, it, vi } from "vitest"; + +const bedrockMock = vi.hoisted(() => ({ + constructorCalls: [] as Array>, +})); + +vi.mock("@aws-sdk/client-bedrock-runtime", () => { + class BedrockRuntimeServiceException extends Error {} + + class BedrockRuntimeClient { + constructor(config: Record) { + bedrockMock.constructorCalls.push(config); + } + + send(): Promise { + return Promise.reject(new Error("mock send")); + } + } + + class ConverseStreamCommand { + readonly input: unknown; + + constructor(input: unknown) { + this.input = input; + } + } + + return { + BedrockRuntimeClient, + BedrockRuntimeServiceException, + ConverseStreamCommand, + StopReason: { + END_TURN: "end_turn", + STOP_SEQUENCE: "stop_sequence", + MAX_TOKENS: "max_tokens", + MODEL_CONTEXT_WINDOW_EXCEEDED: "model_context_window_exceeded", + TOOL_USE: "tool_use", + }, + CachePointType: { DEFAULT: "default" }, + CacheTTL: { ONE_HOUR: "ONE_HOUR" }, + ConversationRole: { ASSISTANT: "assistant", USER: "user" }, + ImageFormat: { JPEG: "jpeg", PNG: "png", GIF: "gif", WEBP: "webp" }, + ToolResultStatus: { ERROR: "error", SUCCESS: "success" }, + }; +}); + +import type { BedrockOptions } from "../src/api/bedrock-converse-stream.ts"; +import { stream as streamBedrock } from "../src/compat.ts"; +import { getBuiltinModel } from "../src/providers/all.ts"; +import type { Context, Model } from "../src/types.ts"; + +const context: Context = { + messages: [{ role: "user", content: "hello", timestamp: Date.now() }], +}; + +async function captureClientConfig( + model: Model<"bedrock-converse-stream">, + options: BedrockOptions = {}, +): Promise> { + bedrockMock.constructorCalls.length = 0; + await streamBedrock(model, context, { cacheRetention: "none", ...options }).result(); + expect(bedrockMock.constructorCalls).toHaveLength(1); + return bedrockMock.constructorCalls[0]; +} + +describe("bedrock credential priority", () => { + afterEach(() => { + vi.unstubAllEnvs(); + }); + + it("prefers explicit and scoped profiles over ambient AWS access keys", async () => { + vi.stubEnv("AWS_ACCESS_KEY_ID", "AKIAEXAMPLE"); + vi.stubEnv("AWS_SECRET_ACCESS_KEY", "secretexample"); + const model = getBuiltinModel("amazon-bedrock", "us.anthropic.claude-opus-4-8"); + + let config = await captureClientConfig(model, { profile: "explicit-profile" }); + + expect(config.profile).toBe("explicit-profile"); + expect(config.credentials).toBeUndefined(); + + config = await captureClientConfig(model, { env: { AWS_PROFILE: "scoped-profile" } }); + + expect(config.profile).toBe("scoped-profile"); + expect(config.credentials).toBeUndefined(); + }); + + it("uses ambient AWS access keys when no profile is configured", async () => { + vi.stubEnv("AWS_ACCESS_KEY_ID", "AKIAEXAMPLE"); + vi.stubEnv("AWS_SECRET_ACCESS_KEY", "secretexample"); + const model = getBuiltinModel("amazon-bedrock", "us.anthropic.claude-opus-4-8"); + + const config = await captureClientConfig(model); + + expect(config.profile).toBeUndefined(); + expect(config.credentials).toEqual({ + accessKeyId: "AKIAEXAMPLE", + secretAccessKey: "secretexample", + }); + }); + + it("uses ambient AWS access keys when only an ambient profile is set", async () => { + vi.stubEnv("AWS_ACCESS_KEY_ID", "AKIAEXAMPLE"); + vi.stubEnv("AWS_SECRET_ACCESS_KEY", "secretexample"); + vi.stubEnv("AWS_PROFILE", "ambient-profile"); + const model = getBuiltinModel("amazon-bedrock", "us.anthropic.claude-opus-4-8"); + + const config = await captureClientConfig(model); + + expect(config.profile).toBe("ambient-profile"); + expect(config.credentials).toEqual({ + accessKeyId: "AKIAEXAMPLE", + secretAccessKey: "secretexample", + }); + }); +}); diff --git a/packages/ai/test/bedrock-custom-headers.test.ts b/packages/ai/test/bedrock-custom-headers.test.ts new file mode 100644 index 0000000000000000000000000000000000000000..43ee692cea573e7a493427e2c128d7e4dd10dc53 --- /dev/null +++ b/packages/ai/test/bedrock-custom-headers.test.ts @@ -0,0 +1,202 @@ +import { beforeEach, describe, expect, it, vi } from "vitest"; + +type MiddlewareHandler = (next: (args: unknown) => Promise) => (args: unknown) => Promise; + +const bedrockMock = vi.hoisted(() => ({ + middlewareRegistrations: [] as Array<{ + handler: MiddlewareHandler; + opts: { step?: string; name?: string; priority?: string }; + }>, +})); + +vi.mock("@aws-sdk/client-bedrock-runtime", () => { + class BedrockRuntimeServiceException extends Error {} + + class BedrockRuntimeClient { + middlewareStack = { + add: (handler: MiddlewareHandler, opts: { step?: string; name?: string; priority?: string }) => { + bedrockMock.middlewareRegistrations.push({ handler, opts }); + }, + }; + + send(): Promise { + return Promise.reject(new Error("mock send")); + } + } + + class ConverseStreamCommand { + readonly input: unknown; + + constructor(input: unknown) { + this.input = input; + } + } + + return { + BedrockRuntimeClient, + BedrockRuntimeServiceException, + ConverseStreamCommand, + StopReason: { + END_TURN: "end_turn", + STOP_SEQUENCE: "stop_sequence", + MAX_TOKENS: "max_tokens", + MODEL_CONTEXT_WINDOW_EXCEEDED: "model_context_window_exceeded", + TOOL_USE: "tool_use", + }, + CachePointType: { DEFAULT: "default" }, + CacheTTL: { ONE_HOUR: "ONE_HOUR" }, + ConversationRole: { ASSISTANT: "assistant", USER: "user" }, + ImageFormat: { JPEG: "jpeg", PNG: "png", GIF: "gif", WEBP: "webp" }, + ToolResultStatus: { ERROR: "error", SUCCESS: "success" }, + }; +}); + +import type { BedrockOptions } from "../src/api/bedrock-converse-stream.ts"; +import { stream as streamBedrock, streamSimple as streamSimpleBedrock } from "../src/api/bedrock-converse-stream.ts"; +import { getModel } from "../src/compat.ts"; +import type { Context, Model } from "../src/types.ts"; + +const context: Context = { + messages: [{ role: "user", content: "hello", timestamp: Date.now() }], +}; + +const MIDDLEWARE_NAME = "pi-ai-custom-headers"; + +function getModelFixture(): Model<"bedrock-converse-stream"> { + return getModel("amazon-bedrock", "us.anthropic.claude-opus-4-8"); +} + +/** + * Drive a stream to completion so the middleware (registered before `client.send`) + * is captured even though the mocked `send()` rejects. Errors are swallowed because + * the rejecting mock is expected — we only care about the recorded registrations. + */ +async function driveBedrock(options: BedrockOptions): Promise { + await streamBedrock(getModelFixture(), context, options) + .result() + .catch(() => undefined); +} + +function findCustomHeadersRegistration() { + const matches = bedrockMock.middlewareRegistrations.filter((r) => r.opts.name === MIDDLEWARE_NAME); + return matches; +} + +beforeEach(() => { + bedrockMock.middlewareRegistrations.length = 0; +}); + +describe("bedrock custom headers middleware", () => { + it("VC1: registers a build-step middleware that injects the caller header (happy path)", async () => { + await driveBedrock({ cacheRetention: "none", headers: { "x-custom": "v" } }); + + const registrations = findCustomHeadersRegistration(); + expect(registrations).toHaveLength(1); + + const [reg] = registrations; + expect(reg.opts.step).toBe("build"); + expect(reg.opts.priority).toBe("low"); + expect(reg.opts.name).toBe(MIDDLEWARE_NAME); + + const nextSpy = vi.fn(async (a: unknown) => a); + const fakeArgs = { request: { headers: {} as Record } }; + await reg.handler(nextSpy)(fakeArgs); + + expect(fakeArgs.request.headers["x-custom"]).toBe("v"); + expect(nextSpy).toHaveBeenCalledTimes(1); + expect(nextSpy).toHaveBeenCalledWith(fakeArgs); + }); + + it("VC2: skips reserved headers case-insensitively while applying allowed ones", async () => { + await driveBedrock({ + cacheRetention: "none", + headers: { + authorization: "evil", + "x-amz-date": "evil", + "x-allowed": "ok", + Authorization: "evil2", + "X-Amz-Date": "evil2", + HOST: "evil3", + }, + }); + + const [reg] = findCustomHeadersRegistration(); + expect(reg).toBeDefined(); + + const nextSpy = vi.fn(async (a: unknown) => a); + const fakeArgs = { + request: { + headers: { + authorization: "real-auth", + "x-amz-date": "real-date", + host: "real-host", + } as Record, + }, + }; + await reg.handler(nextSpy)(fakeArgs); + + expect(fakeArgs.request.headers.authorization).toBe("real-auth"); + expect(fakeArgs.request.headers["x-amz-date"]).toBe("real-date"); + expect(fakeArgs.request.headers.host).toBe("real-host"); + expect(fakeArgs.request.headers["x-allowed"]).toBe("ok"); + // Mixed-case reserved keys must be skipped too: a case-sensitive guard would + // add them back as distinct capitalised keys. Assert no such leak occurred and + // that the only new key beyond the three pre-existing ones is `x-allowed`. + expect(fakeArgs.request.headers.Authorization).toBeUndefined(); + expect(fakeArgs.request.headers["X-Amz-Date"]).toBeUndefined(); + expect(fakeArgs.request.headers.HOST).toBeUndefined(); + expect(Object.keys(fakeArgs.request.headers).sort()).toEqual( + ["authorization", "host", "x-allowed", "x-amz-date"].sort(), + ); + expect(nextSpy).toHaveBeenCalledTimes(1); + }); + + it("VC3: registers no middleware when headers is undefined", async () => { + await driveBedrock({ cacheRetention: "none" }); + + expect(findCustomHeadersRegistration()).toHaveLength(0); + }); + + it("VC3: registers no middleware when headers is empty", async () => { + await driveBedrock({ cacheRetention: "none", headers: {} }); + + expect(findCustomHeadersRegistration()).toHaveLength(0); + }); + + it("VC3 (structural guard): passes through unchanged when the request has no headers", async () => { + await driveBedrock({ cacheRetention: "none", headers: { "x-custom": "v" } }); + + const [reg] = findCustomHeadersRegistration(); + expect(reg).toBeDefined(); + + const nextSpy = vi.fn(async (a: unknown) => a); + + const argsNoHeaders = { request: {} }; + await expect(reg.handler(nextSpy)(argsNoHeaders)).resolves.toBeDefined(); + expect(nextSpy).toHaveBeenCalledWith(argsNoHeaders); + + const argsUndefinedRequest = { request: undefined }; + await expect(reg.handler(nextSpy)(argsUndefinedRequest)).resolves.toBeDefined(); + expect(nextSpy).toHaveBeenCalledWith(argsUndefinedRequest); + + expect(nextSpy).toHaveBeenCalledTimes(2); + }); + + it("VC4: streamSimpleBedrock forwards headers end-to-end (regression guard)", async () => { + await streamSimpleBedrock(getModelFixture(), context, { headers: { "x-custom": "v" } }) + .result() + .catch(() => undefined); + + const registrations = findCustomHeadersRegistration(); + expect(registrations).toHaveLength(1); + + const [reg] = registrations; + expect(reg.opts.step).toBe("build"); + + const nextSpy = vi.fn(async (a: unknown) => a); + const fakeArgs = { request: { headers: {} as Record } }; + await reg.handler(nextSpy)(fakeArgs); + + expect(fakeArgs.request.headers["x-custom"]).toBe("v"); + }); +}); diff --git a/packages/ai/test/bedrock-endpoint-resolution.test.ts b/packages/ai/test/bedrock-endpoint-resolution.test.ts new file mode 100644 index 0000000000000000000000000000000000000000..62797d15f953b8d4d2ab6e470d6ec299615b7753 --- /dev/null +++ b/packages/ai/test/bedrock-endpoint-resolution.test.ts @@ -0,0 +1,208 @@ +import { afterEach, beforeEach, describe, expect, it, vi } from "vitest"; + +const bedrockMock = vi.hoisted(() => ({ + constructorCalls: [] as Array>, +})); + +vi.mock("@aws-sdk/client-bedrock-runtime", () => { + class BedrockRuntimeServiceException extends Error {} + + class BedrockRuntimeClient { + constructor(config: Record) { + bedrockMock.constructorCalls.push(config); + } + + send(): Promise { + return Promise.reject(new Error("mock send")); + } + } + + class ConverseStreamCommand { + readonly input: unknown; + + constructor(input: unknown) { + this.input = input; + } + } + + return { + BedrockRuntimeClient, + BedrockRuntimeServiceException, + ConverseStreamCommand, + StopReason: { + END_TURN: "end_turn", + STOP_SEQUENCE: "stop_sequence", + MAX_TOKENS: "max_tokens", + MODEL_CONTEXT_WINDOW_EXCEEDED: "model_context_window_exceeded", + TOOL_USE: "tool_use", + }, + CachePointType: { DEFAULT: "default" }, + CacheTTL: { ONE_HOUR: "ONE_HOUR" }, + ConversationRole: { ASSISTANT: "assistant", USER: "user" }, + ImageFormat: { JPEG: "jpeg", PNG: "png", GIF: "gif", WEBP: "webp" }, + ToolResultStatus: { ERROR: "error", SUCCESS: "success" }, + }; +}); + +import type { BedrockOptions } from "../src/api/bedrock-converse-stream.ts"; +import { getModel, stream as streamBedrock } from "../src/compat.ts"; +import type { Context, Model } from "../src/types.ts"; + +const context: Context = { + messages: [{ role: "user", content: "hello", timestamp: Date.now() }], +}; + +const originalAwsRegion = process.env.AWS_REGION; +const originalAwsDefaultRegion = process.env.AWS_DEFAULT_REGION; +const originalAwsProfile = process.env.AWS_PROFILE; + +beforeEach(() => { + bedrockMock.constructorCalls.length = 0; + delete process.env.AWS_REGION; + delete process.env.AWS_DEFAULT_REGION; + delete process.env.AWS_PROFILE; +}); + +afterEach(() => { + if (originalAwsRegion === undefined) { + delete process.env.AWS_REGION; + } else { + process.env.AWS_REGION = originalAwsRegion; + } + + if (originalAwsDefaultRegion === undefined) { + delete process.env.AWS_DEFAULT_REGION; + } else { + process.env.AWS_DEFAULT_REGION = originalAwsDefaultRegion; + } + + if (originalAwsProfile === undefined) { + delete process.env.AWS_PROFILE; + } else { + process.env.AWS_PROFILE = originalAwsProfile; + } +}); + +async function captureClientConfig( + model: Model<"bedrock-converse-stream">, + options: BedrockOptions = {}, +): Promise> { + bedrockMock.constructorCalls.length = 0; + await streamBedrock(model, context, { cacheRetention: "none", ...options }).result(); + expect(bedrockMock.constructorCalls).toHaveLength(1); + return bedrockMock.constructorCalls[0]; +} + +describe("bedrock endpoint resolution", () => { + it("assigns eu-central-1 runtime URLs to built-in EU inference profiles", () => { + const model = getModel("amazon-bedrock", "eu.anthropic.claude-sonnet-4-5-20250929-v1:0"); + + expect(model.baseUrl).toBe("https://bedrock-runtime.eu-central-1.amazonaws.com"); + }); + + it("does not pin standard AWS endpoints when AWS_REGION is configured", async () => { + process.env.AWS_REGION = "us-east-2"; + const model = getModel("amazon-bedrock", "us.anthropic.claude-opus-4-8"); + + const config = await captureClientConfig(model); + + expect(config.region).toBe("us-east-2"); + expect(config.endpoint).toBeUndefined(); + }); + + it("derives region from a built-in EU endpoint when no region or profile is configured", async () => { + const model = getModel("amazon-bedrock", "eu.anthropic.claude-sonnet-4-5-20250929-v1:0"); + + const config = await captureClientConfig(model); + + expect(config.endpoint).toBe("https://bedrock-runtime.eu-central-1.amazonaws.com"); + expect(config.region).toBe("eu-central-1"); + }); + + it("handles missing regions for explicit, scoped, and ambient profiles", async () => { + const model = getModel("amazon-bedrock", "eu.anthropic.claude-sonnet-4-5-20250929-v1:0"); + + let config = await captureClientConfig(model, { profile: "bedrock-profile" }); + + expect(config.profile).toBe("bedrock-profile"); + expect(config.endpoint).toBe("https://bedrock-runtime.eu-central-1.amazonaws.com"); + expect(config.region).toBe("eu-central-1"); + + config = await captureClientConfig(model, { env: { AWS_PROFILE: "scoped-bedrock-profile" } }); + + expect(config.profile).toBe("scoped-bedrock-profile"); + expect(config.endpoint).toBe("https://bedrock-runtime.eu-central-1.amazonaws.com"); + expect(config.region).toBe("eu-central-1"); + + process.env.AWS_PROFILE = "ambient-bedrock-profile"; + config = await captureClientConfig(model); + + expect(config.profile).toBe("ambient-bedrock-profile"); + expect(config.endpoint).toBeUndefined(); + expect(config.region).toBeUndefined(); + }); + + it("still passes custom Bedrock endpoints through to the SDK client", async () => { + process.env.AWS_REGION = "us-west-2"; + const baseModel = getModel("amazon-bedrock", "us.anthropic.claude-opus-4-8"); + const model: Model<"bedrock-converse-stream"> = { + ...baseModel, + baseUrl: "https://bedrock-vpc.example.com", + }; + + const config = await captureClientConfig(model); + + expect(config.endpoint).toBe("https://bedrock-vpc.example.com"); + expect(config.region).toBe("us-west-2"); + }); + + it("extracts region from inference profile ARN regardless of AWS_REGION", async () => { + process.env.AWS_REGION = "us-east-1"; + const baseModel = getModel("amazon-bedrock", "us.anthropic.claude-opus-4-8"); + const model: Model<"bedrock-converse-stream"> = { + ...baseModel, + id: "arn:aws:bedrock:us-west-2:123456789012:application-inference-profile/abc123", + }; + + const config = await captureClientConfig(model); + + expect(config.region).toBe("us-west-2"); + }); + + it("extracts region from GovCloud inference profile ARN", async () => { + process.env.AWS_REGION = "us-east-1"; + const baseModel = getModel("amazon-bedrock", "us.anthropic.claude-opus-4-8"); + const model: Model<"bedrock-converse-stream"> = { + ...baseModel, + id: "arn:aws-us-gov:bedrock:us-gov-west-1:123456789012:application-inference-profile/abc123", + }; + + const config = await captureClientConfig(model); + + expect(config.region).toBe("us-gov-west-1"); + }); + + it("preserves ambient AWS auth for custom model IDs through compat dispatch", async () => { + process.env.AWS_PROFILE = "bedrock-profile"; + const baseModel = getModel("amazon-bedrock", "us.anthropic.claude-opus-4-8"); + const model: Model<"bedrock-converse-stream"> = { + ...baseModel, + id: "arn:aws:bedrock:us-east-1:123456789012:application-inference-profile/example", + }; + + const config = await captureClientConfig(model); + + expect(config.profile).toBe("bedrock-profile"); + expect(config.token).toBeUndefined(); + expect(config.authSchemePreference).toBeUndefined(); + }); + + it("uses the generic API key option as a Bedrock bearer token", async () => { + const model = getModel("amazon-bedrock", "us.anthropic.claude-opus-4-8"); + + const config = await captureClientConfig(model, { apiKey: "bedrock-api-key" }); + + expect(config.token).toEqual({ token: "bedrock-api-key" }); + expect(config.authSchemePreference).toEqual(["httpBearerAuth"]); + }); +}); diff --git a/packages/ai/test/bedrock-models.test.ts b/packages/ai/test/bedrock-models.test.ts new file mode 100644 index 0000000000000000000000000000000000000000..02e8b8f16b4baabc553b91dd8ffe09b61c83bfa2 --- /dev/null +++ b/packages/ai/test/bedrock-models.test.ts @@ -0,0 +1,70 @@ +/** + * A test suite to ensure all configured Amazon Bedrock models are usable. + * + * This is here to make sure we got correct model identifiers from models.dev and other sources. + * Because Amazon Bedrock requires cross-region inference in some models, + * plain model identifiers are not always usable and it requires tweaking of model identifiers to use cross-region inference. + * See https://docs.aws.amazon.com/bedrock/latest/userguide/inference-profiles-support.html#inference-profiles-support-system for more details. + * + * This test suite is not enabled by default unless AWS credentials and `BEDROCK_EXTENSIVE_MODEL_TEST` environment variables are set. + * This test suite takes ~2 minutes to run. Because not all models are available in all regions, + * it's recommended to use `us-west-2` region for best coverage for running this test suite. + * + * You can run this test suite with: + * ```bash + * $ AWS_REGION=us-west-2 BEDROCK_EXTENSIVE_MODEL_TEST=1 AWS_PROFILE=... npm test -- ./test/bedrock-models.test.ts + * ``` + */ + +import { describe, expect, it } from "vitest"; +import { complete, getModels } from "../src/compat.ts"; +import type { Context } from "../src/types.ts"; +import { hasBedrockCredentials } from "./bedrock-utils.ts"; + +describe("Amazon Bedrock Models", () => { + const models = getModels("amazon-bedrock"); + + it("should get all available Bedrock models", () => { + expect(models.length).toBeGreaterThan(0); + console.log(`Found ${models.length} Bedrock models`); + }); + + it("exposes Claude Opus 5 through an inference profile only", () => { + expect(models.some((model) => model.id === "global.anthropic.claude-opus-5")).toBe(true); + expect(models.some((model) => model.id === "anthropic.claude-opus-5")).toBe(false); + }); + + if (hasBedrockCredentials() && process.env.BEDROCK_EXTENSIVE_MODEL_TEST) { + for (const model of models) { + it(`should make a simple request with ${model.id}`, { timeout: 10_000 }, async () => { + const context: Context = { + systemPrompt: "You are a helpful assistant. Be extremely concise.", + messages: [ + { + role: "user", + content: "Reply with exactly: 'OK'", + timestamp: Date.now(), + }, + ], + }; + + const response = await complete(model, context); + + expect(response.role).toBe("assistant"); + expect(response.content).toBeTruthy(); + expect(response.content.length).toBeGreaterThan(0); + expect(response.usage.input + response.usage.cacheRead).toBeGreaterThan(0); + expect(response.usage.output).toBeGreaterThan(0); + expect(response.errorMessage).toBeFalsy(); + + const textContent = response.content + .filter((b) => b.type === "text") + .map((b) => (b.type === "text" ? b.text : "")) + .join("") + .trim(); + expect(textContent).toBeTruthy(); + console.log(`${model.id}: ${textContent.substring(0, 100)}`); + }); + } + } +}); diff --git a/packages/ai/test/bedrock-redacted-reasoning.test.ts b/packages/ai/test/bedrock-redacted-reasoning.test.ts new file mode 100644 index 0000000000000000000000000000000000000000..86301e155ae7aff340111f4e9b024db3004ba50f --- /dev/null +++ b/packages/ai/test/bedrock-redacted-reasoning.test.ts @@ -0,0 +1,274 @@ +import { beforeEach, describe, expect, it, vi } from "vitest"; + +/** + * OpenAI models served through Bedrock Converse (e.g. `global.openai.gpt-5.6-terra`) + * return encrypted reasoning as the opaque `redactedContent` member of + * `reasoningContent`, not as `reasoningText`. The AWS SDK decodes the wire blob to + * `Uint8Array`. + * @see https://docs.aws.amazon.com/bedrock/latest/APIReference/API_runtime_ReasoningContentBlockDelta.html + */ +const bedrockMock = vi.hoisted(() => { + const redactedBase64 = "cnNuXzVaVnJpZjRKMGJYSXFtV2RsZWRqN1FJRmVOaWtSUWJF"; + return { + redactedBase64, + redactedBytes: new Uint8Array(Buffer.from(redactedBase64, "base64")), + streamEvents: undefined as unknown[] | undefined, + }; +}); + +vi.mock("@aws-sdk/client-bedrock-runtime", () => { + class BedrockRuntimeServiceException extends Error {} + + class BedrockRuntimeClient { + send(): Promise { + if (bedrockMock.streamEvents) { + const events = bedrockMock.streamEvents; + return Promise.resolve({ + $metadata: { httpStatusCode: 200 }, + stream: (async function* () { + yield* events; + })(), + }); + } + return Promise.reject(new Error("mock send")); + } + } + + class ConverseStreamCommand { + readonly input: unknown; + + constructor(input: unknown) { + this.input = input; + } + } + + return { + BedrockRuntimeClient, + BedrockRuntimeServiceException, + ConverseStreamCommand, + StopReason: { + END_TURN: "end_turn", + STOP_SEQUENCE: "stop_sequence", + MAX_TOKENS: "max_tokens", + MODEL_CONTEXT_WINDOW_EXCEEDED: "model_context_window_exceeded", + TOOL_USE: "tool_use", + }, + CachePointType: { DEFAULT: "default" }, + CacheTTL: { ONE_HOUR: "ONE_HOUR" }, + ConversationRole: { ASSISTANT: "assistant", USER: "user" }, + ImageFormat: { JPEG: "jpeg", PNG: "png", GIF: "gif", WEBP: "webp" }, + ToolResultStatus: { ERROR: "error", SUCCESS: "success" }, + }; +}); + +import { stream as streamBedrock } from "../src/api/bedrock-converse-stream.ts"; +import type { Context, Message, Model, ThinkingContent } from "../src/types.ts"; + +const gptModel: Model<"bedrock-converse-stream"> = { + id: "global.openai.gpt-5.6-terra", + name: "GPT-5.6 Terra (Global)", + api: "bedrock-converse-stream", + provider: "amazon-bedrock", + baseUrl: "https://bedrock-runtime.ap-northeast-1.amazonaws.com", + reasoning: true, + input: ["text"], + cost: { input: 1.25, output: 10, cacheRead: 0.125, cacheWrite: 0 }, + contextWindow: 400000, + maxTokens: 128000, +}; + +const emptyUsage = { + input: 0, + output: 0, + cacheRead: 0, + cacheWrite: 0, + totalTokens: 0, + cost: { input: 0, output: 0, cacheRead: 0, cacheWrite: 0, total: 0 }, +}; + +/** Mirrors the ConverseStream frames GPT-5.6 emits: encrypted reasoning, then text. */ +function redactedReasoningEvents(): unknown[] { + return [ + { messageStart: { role: "assistant" } }, + { + contentBlockDelta: { + contentBlockIndex: 0, + delta: { reasoningContent: { redactedContent: bedrockMock.redactedBytes } }, + }, + }, + { contentBlockStop: { contentBlockIndex: 0 } }, + { contentBlockDelta: { contentBlockIndex: 1, delta: { text: "done" } } }, + { contentBlockStop: { contentBlockIndex: 1 } }, + { messageStop: { stopReason: "end_turn" } }, + ]; +} + +interface BedrockRequestPayload { + messages: Array<{ role: string; content: Array> }>; +} + +async function capturePayload(context: Context): Promise { + let capturedPayload: BedrockRequestPayload | undefined; + const s = streamBedrock(gptModel, context, { + cacheRetention: "none", + signal: AbortSignal.abort(), + onPayload: (payload) => { + capturedPayload = payload as BedrockRequestPayload; + return payload; + }, + }); + for await (const event of s) { + if (event.type === "error") break; + } + if (!capturedPayload) { + throw new Error("Expected Bedrock payload to be captured before request abort"); + } + return capturedPayload; +} + +describe("Bedrock redacted reasoning", () => { + beforeEach(() => { + bedrockMock.streamEvents = undefined; + }); + + it("does not fail the stream when reasoning arrives as redactedContent", async () => { + bedrockMock.streamEvents = redactedReasoningEvents(); + + const response = await streamBedrock(gptModel, { + messages: [{ role: "user", content: "hello", timestamp: Date.now() }], + }).result(); + + expect(response.stopReason, response.errorMessage).not.toBe("error"); + // Reasoning precedes the answer, matching the order Bedrock streamed it. + expect(response.content.map((c) => c.type)).toEqual(["thinking", "text"]); + expect(response.content[1]).toEqual({ type: "text", text: "done" }); + }); + + it("preserves the encrypted reasoning payload on the assistant message", async () => { + bedrockMock.streamEvents = redactedReasoningEvents(); + + const response = await streamBedrock(gptModel, { + messages: [{ role: "user", content: "hello", timestamp: Date.now() }], + }).result(); + + const thinking = response.content.find((c): c is ThinkingContent => c.type === "thinking"); + expect(thinking).toBeDefined(); + // Same representation Anthropic redacted thinking already uses: the opaque + // payload rides in `thinkingSignature` with `redacted: true`. + expect(thinking?.redacted).toBe(true); + expect(thinking?.thinkingSignature).toBe(bedrockMock.redactedBase64); + // The byte buffer is streaming scratch state: persisting it would bloat the + // session, since a Uint8Array serializes to an index-keyed object. + expect("redactedChunks" in thinking!).toBe(false); + }); + + it("encodes the payload when the stream never sends contentBlockStop", async () => { + bedrockMock.streamEvents = [ + { messageStart: { role: "assistant" } }, + { + contentBlockDelta: { + contentBlockIndex: 0, + delta: { reasoningContent: { redactedContent: bedrockMock.redactedBytes } }, + }, + }, + { messageStop: { stopReason: "end_turn" } }, + ]; + + const response = await streamBedrock(gptModel, { + messages: [{ role: "user", content: "hello", timestamp: Date.now() }], + }).result(); + + const thinking = response.content.find((c): c is ThinkingContent => c.type === "thinking"); + expect(thinking?.thinkingSignature).toBe(bedrockMock.redactedBase64); + // Streaming scratch state must not survive into the persisted message. + expect("redactedChunks" in thinking!).toBe(false); + expect("index" in thinking!).toBe(false); + }); + + it("joins encrypted reasoning split across deltas", async () => { + const [head, tail] = [bedrockMock.redactedBytes.slice(0, 7), bedrockMock.redactedBytes.slice(7)]; + bedrockMock.streamEvents = [ + { messageStart: { role: "assistant" } }, + { contentBlockDelta: { contentBlockIndex: 0, delta: { reasoningContent: { redactedContent: head } } } }, + { contentBlockDelta: { contentBlockIndex: 0, delta: { reasoningContent: { redactedContent: tail } } } }, + { contentBlockStop: { contentBlockIndex: 0 } }, + { messageStop: { stopReason: "end_turn" } }, + ]; + + const response = await streamBedrock(gptModel, { + messages: [{ role: "user", content: "hello", timestamp: Date.now() }], + }).result(); + + const thinking = response.content.find((c): c is ThinkingContent => c.type === "thinking"); + expect(thinking?.thinkingSignature).toBe(bedrockMock.redactedBase64); + // The placeholder marks the block once, not once per delta. + expect(thinking?.thinking).toBe("[Reasoning redacted]"); + }); + + it("replays redacted reasoning as reasoningContent.redactedContent", async () => { + const messages: Message[] = [ + { role: "user", content: "hello", timestamp: Date.now() }, + { + role: "assistant", + content: [ + { type: "thinking", thinking: "", thinkingSignature: bedrockMock.redactedBase64, redacted: true }, + { type: "text", text: "done" }, + ], + api: "bedrock-converse-stream", + provider: "amazon-bedrock", + model: gptModel.id, + usage: emptyUsage, + stopReason: "stop", + timestamp: Date.now(), + }, + { role: "user", content: "continue", timestamp: Date.now() }, + ]; + + const payload = await capturePayload({ messages }); + + const assistant = payload.messages.find((m: any) => m.role === "assistant"); + expect(assistant).toBeDefined(); + expect(assistant!.content).toEqual([ + { reasoningContent: { redactedContent: bedrockMock.redactedBytes } }, + { text: "done" }, + ]); + }); + + it("replays redacted reasoning before the toolUse block it belongs to", async () => { + // Bedrock rejects a tool continuation whose reasoning block is missing or + // reordered, so the opaque payload must land ahead of the matching toolUse. + const messages: Message[] = [ + { role: "user", content: "read the file", timestamp: Date.now() }, + { + role: "assistant", + content: [ + { type: "thinking", thinking: "", thinkingSignature: bedrockMock.redactedBase64, redacted: true }, + { type: "toolCall", id: "tool-1", name: "read", arguments: { path: "/tmp/a.txt" } }, + ], + api: "bedrock-converse-stream", + provider: "amazon-bedrock", + model: gptModel.id, + usage: emptyUsage, + stopReason: "toolUse", + timestamp: Date.now(), + }, + { + role: "toolResult", + toolCallId: "tool-1", + toolName: "read", + content: [{ type: "text", text: "file body" }], + isError: false, + timestamp: Date.now(), + }, + ]; + + const payload = await capturePayload({ messages }); + + const assistant = payload.messages.find((m: any) => m.role === "assistant"); + expect(assistant).toBeDefined(); + expect(assistant!.content).toEqual([ + { reasoningContent: { redactedContent: bedrockMock.redactedBytes } }, + { toolUse: { toolUseId: "tool-1", name: "read", input: { path: "/tmp/a.txt" } } }, + ]); + }); +}); diff --git a/packages/ai/test/bedrock-response-headers.test.ts b/packages/ai/test/bedrock-response-headers.test.ts new file mode 100644 index 0000000000000000000000000000000000000000..82d264882f37ea586d776a94e949fe71023b5ead --- /dev/null +++ b/packages/ai/test/bedrock-response-headers.test.ts @@ -0,0 +1,63 @@ +import { createServer, type Server } from "node:http"; +import { afterEach, describe, expect, it } from "vitest"; +import { stream as streamBedrock } from "../src/api/bedrock-converse-stream.ts"; +import { getModel } from "../src/compat.ts"; +import type { Model, ProviderResponse } from "../src/types.ts"; + +const MODEL_ID = "us.anthropic.claude-haiku-4-5-20251001-v1:0"; + +let server: Server | undefined; + +afterEach(async () => { + if (!server) return; + await new Promise((resolve, reject) => { + server?.close((error) => (error ? reject(error) : resolve())); + }); + server = undefined; +}); + +async function startBedrockResponseServer(): Promise { + server = createServer((_req, res) => { + res.writeHead(200, { + "content-type": "application/vnd.amazon.eventstream", + "x-bifrost-provider": "bedrock", + "x-bifrost-resolved-model": MODEL_ID, + "x-amzn-requestid": "req-123", + }); + res.end(); + }); + + await new Promise((resolve) => server?.listen(0, "127.0.0.1", resolve)); + const address = server.address(); + if (!address || typeof address === "string") throw new Error("Expected TCP test server address"); + return `http://127.0.0.1:${address.port}`; +} + +describe("bedrock response headers", () => { + it("forwards raw Smithy response headers to onResponse", async () => { + const baseModel = getModel("amazon-bedrock", MODEL_ID) as Model<"bedrock-converse-stream">; + const model = { ...baseModel, baseUrl: await startBedrockResponseServer() }; + const responses: ProviderResponse[] = []; + + const result = await streamBedrock( + model, + { messages: [{ role: "user", content: "hello", timestamp: Date.now() }] }, + { + cacheRetention: "none", + env: { AWS_BEDROCK_FORCE_HTTP1: "1", AWS_BEDROCK_SKIP_AUTH: "1" }, + onResponse: (response) => { + responses.push(response); + }, + }, + ).result(); + + // The fake server intentionally returns an empty event stream; this assertion + // documents that the header callback still fires before stream consumption. + expect(result.stopReason).toBe("error"); + expect(responses).toHaveLength(1); + expect(responses[0].status).toBe(200); + expect(responses[0].headers["x-amzn-requestid"]).toBe("req-123"); + expect(responses[0].headers["x-bifrost-provider"]).toBe("bedrock"); + expect(responses[0].headers["x-bifrost-resolved-model"]).toBe(MODEL_ID); + }); +}); diff --git a/packages/ai/test/bedrock-thinking-payload.test.ts b/packages/ai/test/bedrock-thinking-payload.test.ts new file mode 100644 index 0000000000000000000000000000000000000000..acc370574292cb09b44e82d8217db20cecbdfb68 --- /dev/null +++ b/packages/ai/test/bedrock-thinking-payload.test.ts @@ -0,0 +1,266 @@ +import { describe, expect, it } from "vitest"; +import { type BedrockOptions, stream as streamBedrock } from "../src/api/bedrock-converse-stream.ts"; +import { getModel } from "../src/compat.ts"; +import type { Context, Model } from "../src/types.ts"; +import { hasBedrockCredentials } from "./bedrock-utils.ts"; + +interface BedrockThinkingPayload { + additionalModelRequestFields?: { + thinking?: { type: string; budget_tokens?: number; display?: string }; + output_config?: { effort?: string }; + anthropic_beta?: string[]; + }; +} + +class PayloadCaptured extends Error { + constructor() { + super("payload captured"); + this.name = "PayloadCaptured"; + } +} + +function makeContext(): Context { + return { + messages: [{ role: "user", content: "Hello", timestamp: Date.now() }], + }; +} + +async function capturePayload( + model: Model<"bedrock-converse-stream">, + options?: BedrockOptions, +): Promise { + let capturedPayload: BedrockThinkingPayload | undefined; + const s = streamBedrock(model, makeContext(), { + ...options, + reasoning: options?.reasoning ?? "high", + onPayload: (payload) => { + capturedPayload = payload as BedrockThinkingPayload; + throw new PayloadCaptured(); + }, + }); + + for await (const event of s) { + if (event.type === "error") { + break; + } + } + + if (!capturedPayload) { + throw new Error("Expected Bedrock payload to be captured before request abort"); + } + + return capturedPayload; +} + +describe("Bedrock thinking payload", () => { + it("uses adaptive thinking for Claude Opus 4.8 when reasoning is enabled", async () => { + const baseModel = getModel("amazon-bedrock", "global.anthropic.claude-opus-4-6-v1"); + const model: Model<"bedrock-converse-stream"> = { + ...baseModel, + id: "global.anthropic.claude-opus-4-8-v1", + name: "Claude Opus 4.8 (Global)", + }; + + const payload = await capturePayload(model); + + expect(payload.additionalModelRequestFields?.thinking).toEqual({ type: "adaptive", display: "summarized" }); + expect(payload.additionalModelRequestFields?.output_config).toEqual({ effort: "high" }); + expect(payload.additionalModelRequestFields?.anthropic_beta).toBeUndefined(); + }); + + it("maps xhigh reasoning to effort=xhigh for Claude Opus 4.8", async () => { + const baseModel = getModel("amazon-bedrock", "global.anthropic.claude-opus-4-6-v1"); + const model: Model<"bedrock-converse-stream"> = { + ...baseModel, + id: "global.anthropic.claude-opus-4-8-v1", + name: "Claude Opus 4.8 (Global)", + }; + + const payload = await capturePayload(model, { reasoning: "xhigh" }); + + expect(payload.additionalModelRequestFields?.thinking).toEqual({ type: "adaptive", display: "summarized" }); + expect(payload.additionalModelRequestFields?.output_config).toEqual({ effort: "xhigh" }); + expect(payload.additionalModelRequestFields?.anthropic_beta).toBeUndefined(); + }); + + it("uses adaptive thinking for Claude Fable 5 when reasoning is enabled", async () => { + const model = getModel("amazon-bedrock", "global.anthropic.claude-fable-5"); + + const payload = await capturePayload(model); + + expect(payload.additionalModelRequestFields?.thinking).toEqual({ type: "adaptive", display: "summarized" }); + expect(payload.additionalModelRequestFields?.output_config).toEqual({ effort: "high" }); + expect(payload.additionalModelRequestFields?.anthropic_beta).toBeUndefined(); + }); + + it("uses adaptive thinking for Claude Sonnet 5 when reasoning is enabled", async () => { + const model = getModel("amazon-bedrock", "global.anthropic.claude-sonnet-5"); + + const payload = await capturePayload(model); + + expect(payload.additionalModelRequestFields?.thinking).toEqual({ type: "adaptive", display: "summarized" }); + expect(payload.additionalModelRequestFields?.output_config).toEqual({ effort: "high" }); + expect(payload.additionalModelRequestFields?.anthropic_beta).toBeUndefined(); + }); + + it("uses adaptive thinking for Claude Opus 5 when reasoning is enabled", async () => { + const model = getModel("amazon-bedrock", "global.anthropic.claude-opus-5"); + + const payload = await capturePayload(model); + + expect(payload.additionalModelRequestFields?.thinking).toEqual({ type: "adaptive", display: "summarized" }); + expect(payload.additionalModelRequestFields?.output_config).toEqual({ effort: "high" }); + expect(payload.additionalModelRequestFields?.anthropic_beta).toBeUndefined(); + }); + + it("maps xhigh reasoning to effort=xhigh for Claude Opus 5", async () => { + const model = getModel("amazon-bedrock", "global.anthropic.claude-opus-5"); + + const payload = await capturePayload(model, { reasoning: "xhigh" }); + + expect(payload.additionalModelRequestFields?.thinking).toEqual({ type: "adaptive", display: "summarized" }); + expect(payload.additionalModelRequestFields?.output_config).toEqual({ effort: "xhigh" }); + expect(payload.additionalModelRequestFields?.anthropic_beta).toBeUndefined(); + }); + + it("maps xhigh reasoning to effort=xhigh for Claude Fable 5", async () => { + const model = getModel("amazon-bedrock", "global.anthropic.claude-fable-5"); + + const payload = await capturePayload(model, { reasoning: "xhigh" }); + + expect(payload.additionalModelRequestFields?.thinking).toEqual({ type: "adaptive", display: "summarized" }); + expect(payload.additionalModelRequestFields?.output_config).toEqual({ effort: "xhigh" }); + }); + + it("omits display for GovCloud model ids on non-adaptive Claude thinking", async () => { + const baseModel = getModel("amazon-bedrock", "us.anthropic.claude-sonnet-4-5-20250929-v1:0"); + const model: Model<"bedrock-converse-stream"> = { + ...baseModel, + id: "us-gov.anthropic.claude-sonnet-4-5-20250929-v1:0", + name: "Claude Sonnet 4.5 (GovCloud)", + }; + + const payload = await capturePayload(model); + + expect(payload.additionalModelRequestFields?.thinking).toEqual({ type: "enabled", budget_tokens: 16384 }); + expect(payload.additionalModelRequestFields?.anthropic_beta).toEqual(["interleaved-thinking-2025-05-14"]); + }); + + it("omits display for GovCloud regions on adaptive Claude thinking", async () => { + const baseModel = getModel("amazon-bedrock", "global.anthropic.claude-opus-4-6-v1"); + const model: Model<"bedrock-converse-stream"> = { + ...baseModel, + id: "global.anthropic.claude-opus-4-8-v1", + name: "Claude Opus 4.8 (Global)", + }; + + const payload = await capturePayload(model, { region: "us-gov-west-1" }); + + expect(payload.additionalModelRequestFields?.thinking).toEqual({ type: "adaptive" }); + expect(payload.additionalModelRequestFields?.output_config).toEqual({ effort: "high" }); + expect(payload.additionalModelRequestFields?.anthropic_beta).toBeUndefined(); + }); +}); + +describe.skipIf(!hasBedrockCredentials())("Bedrock Claude max tokens E2E", () => { + it( + "uses the model maxTokens cap instead of Bedrock's 4096-token default for adaptive Claude models", + { retry: 2, timeout: 180000 }, + async () => { + const baseModel = getModel("amazon-bedrock", "global.anthropic.claude-sonnet-4-6"); + const model: Model<"bedrock-converse-stream"> = { + ...baseModel, + maxTokens: 6000, + }; + + const response = await streamBedrock( + model, + { + systemPrompt: "You are a deterministic text generator. Follow the requested output format exactly.", + messages: [ + { + role: "user", + content: + "Output exactly 5200 repetitions of the token alpha, separated by single spaces. Do not number them. Do not use markdown. Do not add any other text.", + timestamp: Date.now(), + }, + ], + }, + { reasoning: "low" }, + ).result(); + + expect(response.stopReason, response.errorMessage).not.toBe("error"); + expect(response.usage.output).toBeGreaterThan(4096); + }, + ); +}); + +describe("Application inference profile support", () => { + it("uses adaptive thinking when model.name contains the model name but ARN does not", async () => { + const baseModel = getModel("amazon-bedrock", "global.anthropic.claude-opus-4-6-v1"); + const model: Model<"bedrock-converse-stream"> = { + ...baseModel, + id: "arn:aws:bedrock:us-east-1:123456789012:application-inference-profile/my-profile", + name: "Claude Opus 4.6", + }; + + const payload = await capturePayload(model); + + expect(payload.additionalModelRequestFields?.thinking).toEqual({ type: "adaptive", display: "summarized" }); + expect(payload.additionalModelRequestFields?.output_config).toEqual({ effort: "high" }); + }); + + it("injects cache points when model.name identifies a supported Claude model", async () => { + const baseModel = getModel("amazon-bedrock", "global.anthropic.claude-opus-4-6-v1"); + const model: Model<"bedrock-converse-stream"> = { + ...baseModel, + id: "arn:aws:bedrock:us-east-1:123456789012:application-inference-profile/my-profile", + name: "Claude Sonnet 4.6", + }; + + let capturedPayload: any; + const s = streamBedrock( + model, + { + systemPrompt: "You are helpful.", + messages: [{ role: "user", content: "Hello", timestamp: Date.now() }], + }, + { + onPayload: (payload) => { + capturedPayload = payload; + throw new PayloadCaptured(); + }, + }, + ); + + for await (const event of s) { + if (event.type === "error") break; + } + + // System prompt should have a cache point + expect(capturedPayload.system).toHaveLength(2); + expect(capturedPayload.system[1]).toHaveProperty("cachePoint"); + + // Last user message should have a cache point + const lastMsg = capturedPayload.messages[capturedPayload.messages.length - 1]; + const lastContent = lastMsg.content[lastMsg.content.length - 1]; + expect(lastContent).toHaveProperty("cachePoint"); + }); + + it("falls back to fixed-budget thinking for non-adaptive Claude via model.name", async () => { + const baseModel = getModel("amazon-bedrock", "us.anthropic.claude-sonnet-4-5-20250929-v1:0"); + const model: Model<"bedrock-converse-stream"> = { + ...baseModel, + id: "arn:aws:bedrock:us-east-1:123456789012:application-inference-profile/my-profile", + name: "Claude Sonnet 4.5", + }; + + const payload = await capturePayload(model); + + expect(payload.additionalModelRequestFields?.thinking).toMatchObject({ + type: "enabled", + budget_tokens: expect.any(Number), + }); + expect(payload.additionalModelRequestFields?.anthropic_beta).toEqual(["interleaved-thinking-2025-05-14"]); + }); +}); diff --git a/packages/ai/test/cloudflare-utils.ts b/packages/ai/test/cloudflare-utils.ts new file mode 100644 index 0000000000000000000000000000000000000000..40ec74c542fbed67581ae7c634fdffda71beeb53 --- /dev/null +++ b/packages/ai/test/cloudflare-utils.ts @@ -0,0 +1,9 @@ +export function hasCloudflareWorkersAICredentials(): boolean { + return !!process.env.CLOUDFLARE_API_KEY && !!process.env.CLOUDFLARE_ACCOUNT_ID; +} + +export function hasCloudflareAiGatewayCredentials(): boolean { + return ( + !!process.env.CLOUDFLARE_API_KEY && !!process.env.CLOUDFLARE_ACCOUNT_ID && !!process.env.CLOUDFLARE_GATEWAY_ID + ); +} diff --git a/packages/ai/test/compat-env.test.ts b/packages/ai/test/compat-env.test.ts new file mode 100644 index 0000000000000000000000000000000000000000..5992821180a79670fe1995e605624409164c9d27 --- /dev/null +++ b/packages/ai/test/compat-env.test.ts @@ -0,0 +1,74 @@ +import { afterEach, describe, expect, it } from "vitest"; +import { complete, registerApiProvider, resetApiProviders } from "../src/compat.ts"; +import type { AssistantMessage, Context, Model } from "../src/types.ts"; +import { AssistantMessageEventStream } from "../src/utils/event-stream.ts"; + +const context: Context = { messages: [{ role: "user", content: "hi", timestamp: Date.now() }] }; + +const model: Model<"openai-responses"> = { + id: "test-model", + name: "Test Model", + api: "openai-responses", + provider: "custom-openai", + baseUrl: "https://example.test/v1", + reasoning: false, + input: ["text"], + cost: { input: 0, output: 0, cacheRead: 0, cacheWrite: 0 }, + contextWindow: 128000, + maxTokens: 4096, +}; + +function message(): AssistantMessage { + return { + role: "assistant", + content: [{ type: "text", text: "ok" }], + api: model.api, + provider: model.provider, + model: model.id, + usage: { + input: 0, + output: 0, + cacheRead: 0, + cacheWrite: 0, + totalTokens: 0, + cost: { input: 0, output: 0, cacheRead: 0, cacheWrite: 0, total: 0 }, + }, + stopReason: "stop", + timestamp: Date.now(), + }; +} + +describe("compat legacy API fallback", () => { + afterEach(() => { + resetApiProviders(); + }); + + it("dispatches unknown providers through the legacy API registry", async () => { + let capturedApiKey: string | undefined; + registerApiProvider({ + api: "openai-responses", + stream: (_model, _context, options) => { + capturedApiKey = options?.apiKey; + const stream = new AssistantMessageEventStream(); + const output = message(); + stream.push({ type: "start", partial: output }); + stream.push({ type: "done", reason: "stop", message: output }); + stream.end(output); + return stream; + }, + streamSimple: (_model, _context, options) => { + capturedApiKey = options?.apiKey; + const stream = new AssistantMessageEventStream(); + const output = message(); + stream.push({ type: "start", partial: output }); + stream.push({ type: "done", reason: "stop", message: output }); + stream.end(output); + return stream; + }, + }); + + await complete(model, context, { apiKey: "request-key" }); + + expect(capturedApiKey).toBe("request-key"); + }); +}); diff --git a/packages/ai/test/cross-provider-handoff.test.ts b/packages/ai/test/cross-provider-handoff.test.ts new file mode 100644 index 0000000000000000000000000000000000000000..ada2b536f856f521d09ff5ac406f05123ce65657 --- /dev/null +++ b/packages/ai/test/cross-provider-handoff.test.ts @@ -0,0 +1,522 @@ +/** + * Cross-Provider Handoff Test + * + * Tests that contexts generated by one provider/model can be consumed by another. + * This catches issues like: + * - Tool call ID format incompatibilities (e.g., OpenAI Codex pipe characters) + * - Thinking block transformation issues + * - Message format incompatibilities + * + * Strategy: + * 1. beforeAll: For each provider/model, generate a "small context" (if not cached): + * - User message asking to use a tool + * - Assistant response with thinking + tool call + * - Tool result + * - Final assistant response + * + * 2. Test: For each target provider/model: + * - Concatenate ALL other contexts into one + * - Ask the model to "say hi" + * - If it fails, there's a compatibility issue + * + * Fixtures are generated fresh on each run. + */ + +import { writeFileSync } from "fs"; +import { Type } from "typebox"; +import { beforeAll, describe, expect, it } from "vitest"; +import { completeSimple, getEnvApiKey, getModel } from "../src/compat.ts"; +import type { Api, AssistantMessage, Message, Model, Tool, ToolResultMessage } from "../src/types.ts"; +import { hasAzureOpenAICredentials } from "./azure-utils.ts"; +import { hasCloudflareAiGatewayCredentials, hasCloudflareWorkersAICredentials } from "./cloudflare-utils.ts"; +import { resolveApiKey } from "./oauth.ts"; + +// Simple tool for testing +const testToolSchema = Type.Object({ + value: Type.Number({ description: "A number to double" }), +}); + +const testTool: Tool = { + name: "double_number", + description: "Doubles a number and returns the result", + parameters: testToolSchema, +}; + +// Provider/model pairs to test +interface ProviderModelPair { + provider: string; + model: string; + label: string; + apiOverride?: Api; + upstreamApiKeyEnv?: string; +} + +const PROVIDER_MODEL_PAIRS: ProviderModelPair[] = [ + // Anthropic + { provider: "anthropic", model: "claude-sonnet-4-5", label: "anthropic-claude-sonnet-4-5" }, + // Google + { provider: "google", model: "gemini-3-flash-preview", label: "google-gemini-3-flash-preview" }, + // OpenAI + { + provider: "openai", + model: "gpt-4o-mini", + label: "openai-completions-gpt-4o-mini", + apiOverride: "openai-completions", + }, + { provider: "openai", model: "gpt-5-mini", label: "openai-responses-gpt-5-mini" }, + { provider: "azure-openai-responses", model: "gpt-4o-mini", label: "azure-openai-responses-gpt-4o-mini" }, + // OpenAI Codex + { provider: "openai-codex", model: "gpt-5.5", label: "openai-codex-gpt-5.5" }, + // GitHub Copilot + { provider: "github-copilot", model: "claude-sonnet-4.5", label: "copilot-claude-sonnet-4.5" }, + { provider: "github-copilot", model: "gpt-5.1-codex", label: "copilot-gpt-5.1-codex" }, + { provider: "github-copilot", model: "gemini-3-flash-preview", label: "copilot-gemini-3-flash-preview" }, + { provider: "github-copilot", model: "grok-code-fast-1", label: "copilot-grok-code-fast-1" }, + // Amazon Bedrock + { + provider: "amazon-bedrock", + model: "global.anthropic.claude-sonnet-4-5-20250929-v1:0", + label: "bedrock-claude-sonnet-4-5", + }, + // xAI + { provider: "xai", model: "grok-4.3", label: "xai-grok-4.3" }, + // Cerebras + { provider: "cerebras", model: "zai-glm-4.7", label: "cerebras-zai-glm-4.7" }, + // Cloudflare Workers AI + { provider: "cloudflare-workers-ai", model: "@cf/moonshotai/kimi-k2.6", label: "cloudflare-kimi-k2.6" }, + // Cloudflare AI Gateway + { + provider: "cloudflare-ai-gateway", + model: "workers-ai/@cf/moonshotai/kimi-k2.6", + label: "cloudflare-gateway-kimi-k2.6", + }, + { + provider: "cloudflare-ai-gateway", + model: "claude-sonnet-4-5", + label: "cloudflare-gateway-claude-sonnet-4-5", + upstreamApiKeyEnv: "ANTHROPIC_API_KEY", + }, + { + provider: "cloudflare-ai-gateway", + model: "gpt-5.1", + label: "cloudflare-gateway-gpt-5.1", + upstreamApiKeyEnv: "OPENAI_API_KEY", + }, + // Groq + { provider: "groq", model: "openai/gpt-oss-120b", label: "groq-gpt-oss-120b" }, + // Hugging Face + { provider: "huggingface", model: "moonshotai/Kimi-K2.5", label: "huggingface-kimi-k2.5" }, + // Together AI + { provider: "together", model: "moonshotai/Kimi-K2.6", label: "together-kimi-k2.6" }, + // Baseten + { provider: "baseten", model: "zai-org/GLM-5.2", label: "baseten-glm-5.2" }, + // Kimi For Coding + { provider: "kimi-coding", model: "kimi-for-coding", label: "kimi-for-coding" }, + // Mistral + { provider: "mistral", model: "devstral-medium-latest", label: "mistral-devstral-medium" }, + // MiniMax + { provider: "minimax", model: "MiniMax-M2.7", label: "minimax-m2.7" }, + { provider: "minimax-cn", model: "MiniMax-M2.7", label: "minimax-m2.7" }, + // OpenCode Zen + { provider: "opencode", model: "big-pickle", label: "zen-big-pickle" }, + { provider: "opencode", model: "claude-sonnet-4-5", label: "zen-claude-sonnet-4-5" }, + { provider: "opencode", model: "gemini-3-flash", label: "zen-gemini-3-flash" }, + { provider: "opencode", model: "glm-4.7-free", label: "zen-glm-4.7-free" }, + { provider: "opencode", model: "gpt-5.2-codex", label: "zen-gpt-5.2-codex" }, + { provider: "opencode", model: "minimax-m2.1-free", label: "zen-minimax-m2.1-free" }, + // OpenCode Go + { provider: "opencode-go", model: "kimi-k2.5", label: "go-kimi-k2.5" }, + { provider: "opencode-go", model: "minimax-m2.5", label: "go-minimax-m2.5" }, + // Xiaomi MiMo + { provider: "xiaomi", model: "mimo-v2.5-pro", label: "xiaomi-mimo-v2.5-pro" }, + { provider: "xiaomi-token-plan-cn", model: "mimo-v2.5-pro", label: "xiaomi-token-plan-cn-mimo-v2.5-pro" }, + { provider: "xiaomi-token-plan-ams", model: "mimo-v2.5-pro", label: "xiaomi-token-plan-ams-mimo-v2.5-pro" }, + { provider: "xiaomi-token-plan-sgp", model: "mimo-v2.5-pro", label: "xiaomi-token-plan-sgp-mimo-v2.5-pro" }, + // Qwen Token Plan + { provider: "qwen-token-plan", model: "qwen3.7-max", label: "qwen-token-plan-qwen3.7-max" }, + { provider: "qwen-token-plan-cn", model: "qwen3.7-max", label: "qwen-token-plan-cn-qwen3.7-max" }, + { + provider: "qwen-token-plan-individual", + model: "qwen3.8-max", + label: "qwen-token-plan-individual-qwen3.8-max", + }, + { + provider: "qwen-token-plan-individual", + model: "deepseek-v4-flash-0731", + label: "qwen-token-plan-individual-deepseek-v4-flash-0731", + }, + { + provider: "qwen-token-plan-individual", + model: "glm-5.2", + label: "qwen-token-plan-individual-glm-5.2", + }, +]; + +// Cached context structure +interface CachedContext { + label: string; + provider: string; + model: string; + api: Api; + messages: Message[]; + generatedAt: string; +} + +/** + * Get API key for provider - checks OAuth storage first, then env vars + */ +async function getApiKey(provider: string): Promise { + const oauthKey = await resolveApiKey(provider); + if (oauthKey) return oauthKey; + return getEnvApiKey(provider); +} + +/** + * Synchronous check for API key availability (env vars only, for skipIf) + */ +function hasApiKey(pair: ProviderModelPair): boolean { + if (pair.provider === "azure-openai-responses") { + return hasAzureOpenAICredentials(); + } + if (pair.provider === "cloudflare-workers-ai") { + return hasCloudflareWorkersAICredentials(); + } + if (pair.provider === "cloudflare-ai-gateway") { + if (!hasCloudflareAiGatewayCredentials()) return false; + return pair.upstreamApiKeyEnv ? !!process.env[pair.upstreamApiKeyEnv] : true; + } + return !!getEnvApiKey(pair.provider); +} + +function getHeaders(pair: ProviderModelPair): Record | undefined { + if (!pair.upstreamApiKeyEnv) return undefined; + const upstreamApiKey = process.env[pair.upstreamApiKeyEnv]; + return upstreamApiKey ? { Authorization: `Bearer ${upstreamApiKey}` } : undefined; +} + +/** + * Check if any provider has API keys available (for skipIf at describe level) + */ +function hasAnyApiKey(): boolean { + return PROVIDER_MODEL_PAIRS.some((pair) => hasApiKey(pair)); +} + +function dumpFailurePayload(params: { label: string; error: string; payload?: unknown; messages: Message[] }): void { + const filename = `/tmp/pi-handoff-${params.label}-${Date.now()}.json`; + const body = { + label: params.label, + error: params.error, + payload: params.payload, + messages: params.messages, + }; + writeFileSync(filename, JSON.stringify(body, null, 2)); + console.log(`Wrote failure payload to ${filename}`); +} + +/** + * Generate a context from a provider/model pair. + * Makes a real API call to get authentic tool call IDs and thinking blocks. + */ +async function generateContext( + pair: ProviderModelPair, + apiKey: string, +): Promise<{ messages: Message[]; api: Api } | null> { + const baseModel = (getModel as (p: string, m: string) => Model | undefined)(pair.provider, pair.model); + if (!baseModel) { + console.log(` Model not found: ${pair.provider}/${pair.model}`); + return null; + } + + const model: Model = pair.apiOverride ? { ...baseModel, api: pair.apiOverride } : baseModel; + + const userMessage: Message = { + role: "user", + content: "Please double the number 21 using the double_number tool.", + timestamp: Date.now(), + }; + + const supportsReasoning = model.reasoning === true; + const headers = getHeaders(pair); + let lastPayload: unknown; + let assistantResponse: AssistantMessage; + try { + assistantResponse = await completeSimple( + model, + { + systemPrompt: "You are a helpful assistant. Use the provided tool to complete the task.", + messages: [userMessage], + tools: [testTool], + }, + { + apiKey, + reasoning: supportsReasoning ? "high" : undefined, + headers, + onPayload: (payload) => { + lastPayload = payload; + }, + }, + ); + } catch (error) { + const msg = error instanceof Error ? error.message : String(error); + console.log(` Initial request failed: ${msg}`); + dumpFailurePayload({ + label: `${pair.label}-initial`, + error: msg, + payload: lastPayload, + messages: [userMessage], + }); + return null; + } + + if (assistantResponse.stopReason === "error") { + console.log(` Initial request error: ${assistantResponse.errorMessage}`); + dumpFailurePayload({ + label: `${pair.label}-initial`, + error: assistantResponse.errorMessage || "Unknown error", + payload: lastPayload, + messages: [userMessage], + }); + return null; + } + + const toolCall = assistantResponse.content.find((c) => c.type === "toolCall"); + if (!toolCall || toolCall.type !== "toolCall") { + console.log(` No tool call in response (stopReason: ${assistantResponse.stopReason})`); + return { + messages: [userMessage, assistantResponse], + api: model.api, + }; + } + + console.log(` Tool call ID: ${toolCall.id}`); + + const toolResult: ToolResultMessage = { + role: "toolResult", + toolCallId: toolCall.id, + toolName: toolCall.name, + content: [{ type: "text", text: "42" }], + isError: false, + timestamp: Date.now(), + }; + + let finalResponse: AssistantMessage; + const messagesForFinal = [userMessage, assistantResponse, toolResult]; + try { + finalResponse = await completeSimple( + model, + { + systemPrompt: "You are a helpful assistant.", + messages: messagesForFinal, + tools: [testTool], + }, + { + apiKey, + reasoning: supportsReasoning ? "high" : undefined, + headers, + onPayload: (payload) => { + lastPayload = payload; + }, + }, + ); + } catch (error) { + const msg = error instanceof Error ? error.message : String(error); + console.log(` Final request failed: ${msg}`); + dumpFailurePayload({ + label: `${pair.label}-final`, + error: msg, + payload: lastPayload, + messages: messagesForFinal, + }); + return null; + } + + if (finalResponse.stopReason === "error") { + console.log(` Final request error: ${finalResponse.errorMessage}`); + dumpFailurePayload({ + label: `${pair.label}-final`, + error: finalResponse.errorMessage || "Unknown error", + payload: lastPayload, + messages: messagesForFinal, + }); + return null; + } + + return { + messages: [userMessage, assistantResponse, toolResult, finalResponse], + api: model.api, + }; +} + +describe.skipIf(!hasAnyApiKey())("Cross-Provider Handoff", () => { + let contexts: Record; + let availablePairs: ProviderModelPair[]; + + beforeAll(async () => { + contexts = {}; + availablePairs = []; + + console.log("\n=== Generating Fixtures ===\n"); + + for (const pair of PROVIDER_MODEL_PAIRS) { + const apiKey = await getApiKey(pair.provider); + if (!apiKey || !hasApiKey(pair)) { + console.log(`[${pair.label}] Skipping - no auth for ${pair.provider}`); + continue; + } + + console.log(`[${pair.label}] Generating fixture...`); + const result = await generateContext(pair, apiKey); + + if (!result || result.messages.length < 4) { + console.log(`[${pair.label}] Failed to generate fixture, skipping`); + continue; + } + + contexts[pair.label] = { + label: pair.label, + provider: pair.provider, + model: pair.model, + api: result.api, + messages: result.messages, + generatedAt: new Date().toISOString(), + }; + availablePairs.push(pair); + console.log(`[${pair.label}] Generated ${result.messages.length} messages`); + } + + console.log(`\n=== ${availablePairs.length}/${PROVIDER_MODEL_PAIRS.length} contexts available ===\n`); + }, 300000); + + it.skipIf(!hasAnyApiKey())("should have at least 2 fixtures to test handoffs", () => { + expect(Object.keys(contexts).length).toBeGreaterThanOrEqual(2); + }); + + it.skipIf(!hasAnyApiKey())( + "should handle cross-provider handoffs for each target", + async () => { + const contextLabels = Object.keys(contexts); + + if (contextLabels.length < 2) { + console.log("Not enough fixtures for handoff test, skipping"); + return; + } + + console.log("\n=== Testing Cross-Provider Handoffs ===\n"); + + const results: { target: string; success: boolean; error?: string }[] = []; + + for (const targetPair of availablePairs) { + const apiKey = await getApiKey(targetPair.provider); + if (!apiKey || !hasApiKey(targetPair)) { + console.log(`[Target: ${targetPair.label}] Skipping - no auth`); + continue; + } + + // Collect messages from ALL OTHER contexts + const otherMessages: Message[] = []; + for (const [label, ctx] of Object.entries(contexts)) { + if (label === targetPair.label) continue; + otherMessages.push(...ctx.messages); + } + + if (otherMessages.length === 0) { + console.log(`[Target: ${targetPair.label}] Skipping - no other contexts`); + continue; + } + + const allMessages: Message[] = [ + ...otherMessages, + { + role: "user", + content: + "Great, thanks for all that help! Now just say 'Hello, handoff successful!' to confirm you received everything.", + timestamp: Date.now(), + }, + ]; + + const baseModel = (getModel as (p: string, m: string) => Model | undefined)( + targetPair.provider, + targetPair.model, + ); + if (!baseModel) { + console.log(`[Target: ${targetPair.label}] Model not found`); + continue; + } + + const model: Model = targetPair.apiOverride + ? { ...baseModel, api: targetPair.apiOverride } + : baseModel; + const supportsReasoning = model.reasoning === true; + const headers = getHeaders(targetPair); + + console.log( + `[Target: ${targetPair.label}] Testing with ${otherMessages.length} messages from other providers...`, + ); + + let lastPayload: unknown; + try { + const response = await completeSimple( + model, + { + systemPrompt: "You are a helpful assistant.", + messages: allMessages, + tools: [testTool], + }, + { + apiKey, + reasoning: supportsReasoning ? "high" : undefined, + headers, + onPayload: (payload) => { + lastPayload = payload; + }, + }, + ); + + if (response.stopReason === "error") { + console.log(`[Target: ${targetPair.label}] FAILED: ${response.errorMessage}`); + dumpFailurePayload({ + label: targetPair.label, + error: response.errorMessage || "Unknown error", + payload: lastPayload, + messages: allMessages, + }); + results.push({ target: targetPair.label, success: false, error: response.errorMessage }); + } else { + const text = response.content + .filter((c) => c.type === "text") + .map((c) => c.text) + .join(" "); + const preview = text.slice(0, 100).replace(/\n/g, " "); + console.log(`[Target: ${targetPair.label}] SUCCESS: ${preview}...`); + results.push({ target: targetPair.label, success: true }); + } + } catch (error) { + const msg = error instanceof Error ? error.message : String(error); + console.log(`[Target: ${targetPair.label}] EXCEPTION: ${msg}`); + dumpFailurePayload({ + label: targetPair.label, + error: msg, + payload: lastPayload, + messages: allMessages, + }); + results.push({ target: targetPair.label, success: false, error: msg }); + } + } + + console.log("\n=== Results Summary ===\n"); + const successes = results.filter((r) => r.success); + const failures = results.filter((r) => !r.success); + + console.log(`Passed: ${successes.length}/${results.length}`); + if (failures.length > 0) { + console.log("\nFailures:"); + for (const f of failures) { + console.log(` - ${f.target}: ${f.error}`); + } + } + + expect(failures.length).toBe(0); + }, + 600000, + ); +}); diff --git a/packages/ai/test/deferred-tools.test.ts b/packages/ai/test/deferred-tools.test.ts new file mode 100644 index 0000000000000000000000000000000000000000..52cc741db362d9907e9ef42ac66c50069060149b --- /dev/null +++ b/packages/ai/test/deferred-tools.test.ts @@ -0,0 +1,550 @@ +import { Type } from "typebox"; +import { describe, expect, it } from "vitest"; +import { convertMessages } from "../src/api/openai-completions.ts"; +import { getModel, streamSimple } from "../src/compat.ts"; +import type { Api, AssistantMessage, Context, Model, Tool, ToolResultMessage, UserMessage } from "../src/types.ts"; +import { estimateContextTokens } from "../src/utils/estimate.ts"; + +interface AnthropicToolPayload { + name: string; + description?: string; + defer_loading?: boolean; +} + +interface AnthropicContentBlock { + type: string; + text?: string; + tool_use_id?: string; + content?: string | Array<{ type: string; tool_name?: string }>; + source?: { + type: string; + media_type: string; + data: string; + }; +} + +interface AnthropicPayload { + tools?: AnthropicToolPayload[]; + messages: Array<{ + content: string | AnthropicContentBlock[]; + }>; +} + +interface OpenAIToolSearchCall { + type: "tool_search_call"; + call_id?: string | null; + execution?: string; + status?: string | null; +} + +interface OpenAIToolSearchOutput { + type: "tool_search_output"; + call_id?: string | null; + execution?: string; + status?: string | null; + tools: Array<{ type: string; name: string; defer_loading?: boolean }>; +} + +interface OpenAIAdditionalTools { + type: "additional_tools"; + role: "developer"; + tools: Array<{ type: string; name: string; defer_loading?: boolean }>; +} + +interface OpenAIPayload { + tools?: Array<{ name?: string; function?: { name: string } }>; + input?: Array< + OpenAIAdditionalTools | OpenAIToolSearchCall | OpenAIToolSearchOutput | { type?: string; name?: string } + >; +} + +interface KimiTool { + type: "function"; + function: { + name: string; + description?: string; + parameters?: Record; + }; +} + +interface KimiMessage { + role: string; + content?: unknown; + tools?: KimiTool[]; +} + +interface KimiPayload { + tools?: KimiTool[]; + messages: KimiMessage[]; +} + +class PayloadCaptured extends Error {} + +function makeTool(name: string): Tool { + return { + name, + description: `The ${name} tool`, + parameters: Type.Object({ value: Type.String() }), + }; +} + +function makeUserMessage(timestamp: number): UserMessage { + return { role: "user", content: "Hello", timestamp }; +} + +function makeAssistantToolCall(): AssistantMessage { + return { + role: "assistant", + content: [{ type: "toolCall", id: "call_1", name: "base_tool", arguments: {} }], + api: "anthropic-messages", + provider: "anthropic", + model: "claude-opus-4-6", + usage: { + input: 0, + output: 0, + cacheRead: 0, + cacheWrite: 0, + totalTokens: 0, + cost: { input: 0, output: 0, cacheRead: 0, cacheWrite: 0, total: 0 }, + }, + stopReason: "toolUse", + timestamp: 2, + }; +} + +function makeToolResult(addedToolNames: string[]): ToolResultMessage { + return { + role: "toolResult", + toolCallId: "call_1", + toolName: "base_tool", + content: [{ type: "text", text: "done" }], + addedToolNames, + isError: false, + timestamp: 3, + }; +} + +function makeContext(tools: Tool[], addedToolNames = ["late_tool"]): Context { + return { + messages: [makeUserMessage(1), makeAssistantToolCall(), makeToolResult(addedToolNames), makeUserMessage(4)], + tools, + }; +} + +function makeKimiModel(deferredToolsMode?: "kimi"): Model<"openai-completions"> { + return { + id: "deferred-tools-model", + name: "Deferred Tools Model", + api: "openai-completions", + provider: "moonshotai", + baseUrl: "http://127.0.0.1:9/v1", + reasoning: false, + input: ["text"], + cost: { input: 0, output: 0, cacheRead: 0, cacheWrite: 0 }, + contextWindow: 128000, + maxTokens: 4096, + compat: deferredToolsMode ? { deferredToolsMode } : undefined, + }; +} + +async function capturePayload(model: Model, context: Context, apiKey = "fake-key"): Promise { + let captured: T | undefined; + const stream = streamSimple({ ...model, baseUrl: "http://127.0.0.1:9" }, context, { + apiKey, + onPayload: (payload) => { + captured = payload as T; + throw new PayloadCaptured(); + }, + }); + await stream.result(); + if (!captured) throw new Error("Expected payload capture"); + return captured; +} + +function findAnthropicToolResultContent(payload: AnthropicPayload): AnthropicContentBlock[] { + for (const message of payload.messages) { + if (typeof message.content !== "string" && message.content.some((block) => block.type === "tool_result")) { + return message.content; + } + } + throw new Error("No tool result in payload"); +} + +function findAnthropicToolResult(payload: AnthropicPayload): AnthropicContentBlock { + const result = findAnthropicToolResultContent(payload).find((block) => block.type === "tool_result"); + if (!result) throw new Error("No tool result in payload"); + return result; +} + +function openAIToolNames(payload: OpenAIPayload): string[] { + return (payload.tools ?? []).map((tool) => tool.name ?? tool.function?.name ?? ""); +} + +function makeCodexToken(): string { + return `header.${btoa(JSON.stringify({ "https://api.openai.com/auth": { chatgpt_account_id: "account" } }))}.signature`; +} + +describe("deferred tools", () => { + it("loads an Anthropic tool at its tool-result marker", async () => { + const context = makeContext([makeTool("base_tool"), makeTool("late_tool")]); + const payload = await capturePayload(getModel("anthropic", "claude-opus-4-6"), context); + + expect(payload.tools).toMatchObject([{ name: "base_tool" }, { name: "late_tool", defer_loading: true }]); + expect(findAnthropicToolResult(payload).content).toEqual([{ type: "tool_reference", tool_name: "late_tool" }]); + }); + + it("preserves tool output as sibling content after emitting references", async () => { + const context = makeContext([makeTool("base_tool"), makeTool("late_tool")]); + const assistant = context.messages[1] as AssistantMessage; + assistant.content = [ + { type: "toolCall", id: "call_1", name: "base_tool", arguments: {} }, + { type: "toolCall", id: "call_2", name: "base_tool", arguments: {} }, + ]; + const firstResult = context.messages[2] as ToolResultMessage; + firstResult.content = [ + { type: "text", text: "work completed" }, + { type: "image", mimeType: "image/png", data: "aW1hZ2U=" }, + ]; + context.messages.splice(3, 0, { + ...makeToolResult([]), + toolCallId: "call_2", + content: [{ type: "text", text: "second result" }], + }); + + const payload = await capturePayload(getModel("anthropic", "claude-opus-4-6"), context); + + expect(findAnthropicToolResultContent(payload)).toMatchObject([ + { + type: "tool_result", + tool_use_id: "call_1", + content: [{ type: "tool_reference", tool_name: "late_tool" }], + }, + { type: "tool_result", tool_use_id: "call_2", content: "second result" }, + { type: "text", text: "work completed" }, + { + type: "image", + source: { type: "base64", media_type: "image/png", data: "aW1hZ2U=" }, + }, + ]); + }); + + it("loads a tool introduced by OpenAI history after switching to Anthropic", async () => { + const context = makeContext([makeTool("base_tool"), makeTool("late_tool")]); + const assistant = context.messages[1] as AssistantMessage; + assistant.api = "openai-responses"; + assistant.provider = "openai"; + assistant.model = "gpt-5.4"; + + const payload = await capturePayload(getModel("anthropic", "claude-opus-4-8"), context); + + expect(payload.tools).toMatchObject([{ name: "base_tool" }, { name: "late_tool", defer_loading: true }]); + expect(findAnthropicToolResult(payload).content).toEqual([{ type: "tool_reference", tool_name: "late_tool" }]); + }); + + it("does not resurrect a marked tool missing from Context.tools", async () => { + const context = makeContext([makeTool("base_tool")]); + const payload = await capturePayload(getModel("anthropic", "claude-opus-4-6"), context); + + expect(payload.tools?.map((tool) => tool.name)).toEqual(["base_tool"]); + const content = findAnthropicToolResult(payload).content; + expect(Array.isArray(content) && content.some((block) => block.type === "tool_reference")).toBe(false); + }); + + it("keeps a tool immediate when it was used before its marker", async () => { + const context = makeContext([makeTool("base_tool"), makeTool("late_tool")]); + const assistant = context.messages[1] as AssistantMessage; + assistant.content = [{ type: "toolCall", id: "call_1", name: "late_tool", arguments: {} }]; + const payload = await capturePayload(getModel("anthropic", "claude-opus-4-6"), context); + + expect(payload.tools?.map((tool) => tool.name)).toEqual(["base_tool", "late_tool"]); + expect(payload.tools?.every((tool) => !tool.defer_loading)).toBe(true); + }); + + it("normalizes OAuth names before checking prior tool usage", async () => { + const context = makeContext([makeTool("base_tool"), makeTool("read")], ["read"]); + const assistant = context.messages[1] as AssistantMessage; + assistant.content = [{ type: "toolCall", id: "call_1", name: "Read", arguments: {} }]; + const payload = await capturePayload( + getModel("anthropic", "claude-opus-4-6"), + context, + "sk-ant-oat-fake", + ); + + expect(payload.tools?.map((tool) => tool.name)).toEqual(["base_tool", "Read"]); + expect(payload.tools?.every((tool) => !tool.defer_loading)).toBe(true); + const content = findAnthropicToolResult(payload).content; + expect(Array.isArray(content) && content.some((block) => block.type === "tool_reference")).toBe(false); + }); + + it("matches OAuth-canonicalized markers to active tools", async () => { + const context = makeContext([makeTool("base_tool"), makeTool("read")], ["Read"]); + const payload = await capturePayload( + getModel("anthropic", "claude-opus-4-6"), + context, + "sk-ant-oat-fake", + ); + + expect(payload.tools).toMatchObject([{ name: "base_tool" }, { name: "Read", defer_loading: true }]); + const content = findAnthropicToolResult(payload).content; + expect( + Array.isArray(content) && + content.some((block) => block.type === "tool_reference" && block.tool_name === "Read"), + ).toBe(true); + }); + + it("deduplicates active tools after OAuth canonicalization", async () => { + const context: Context = { + messages: [makeUserMessage(1)], + tools: [makeTool("read"), { ...makeTool("Read"), description: "Canonical definition" }], + }; + const payload = await capturePayload( + getModel("anthropic", "claude-opus-4-6"), + context, + "sk-ant-oat-fake", + ); + + expect(payload.tools).toMatchObject([{ name: "Read", description: "Canonical definition" }]); + }); + + it("uses the normal tool list when Anthropic tool references are unsupported", async () => { + const context = makeContext([makeTool("base_tool"), makeTool("late_tool")]); + const models: Model<"anthropic-messages">[] = [ + getModel("anthropic", "claude-haiku-4-5"), + { ...getModel("anthropic", "claude-opus-4-6"), id: "claude-sonnet-4-20250514" }, + ]; + + for (const model of models) { + const payload = await capturePayload(model, context); + expect(payload.tools?.map((tool) => tool.name)).toEqual(["base_tool", "late_tool"]); + expect(payload.tools?.every((tool) => !tool.defer_loading)).toBe(true); + } + }); + + it("keeps one immediate Anthropic tool when every current tool is marked", async () => { + const context = makeContext([makeTool("late_tool")]); + const payload = await capturePayload(getModel("anthropic", "claude-opus-4-6"), context); + + expect(payload.tools).toMatchObject([{ name: "late_tool" }]); + expect(payload.tools?.[0]?.defer_loading).toBeUndefined(); + const content = findAnthropicToolResult(payload).content; + expect(Array.isArray(content) && content.some((block) => block.type === "tool_reference")).toBe(false); + }); + + it("supports explicit Anthropic compatibility overrides", async () => { + const model: Model<"anthropic-messages"> = { + ...getModel("anthropic", "claude-opus-4-6"), + provider: "anthropic-proxy", + compat: { supportsToolReferences: true }, + }; + const context = makeContext([makeTool("base_tool"), makeTool("late_tool")]); + const payload = await capturePayload(model, context); + + expect(payload.tools?.find((tool) => tool.name === "late_tool")?.defer_loading).toBe(true); + }); + + it("serializes Kimi deferred tools as system tool definitions", async () => { + const context = makeContext([makeTool("base_tool"), makeTool("late_tool")]); + const payload = await capturePayload(makeKimiModel("kimi"), context); + + expect(payload.tools?.map((tool) => tool.function.name)).toEqual(["base_tool"]); + const toolResultIndex = payload.messages.findIndex((message) => message.role === "tool"); + const systemToolIndex = payload.messages.findIndex((message) => message.tools !== undefined); + expect(toolResultIndex).toBeGreaterThanOrEqual(0); + expect(systemToolIndex).toBeGreaterThan(toolResultIndex); + expect(payload.messages[systemToolIndex]?.tools?.map((tool) => tool.function.name)).toEqual(["late_tool"]); + }); + + it("emits Kimi deferred schemas after all tool results in a batch", () => { + const context = makeContext([makeTool("base_tool"), makeTool("late_tool"), makeTool("later_tool")]); + context.messages.splice(3, 0, { + ...makeToolResult(["later_tool"]), + toolCallId: "call_2", + }); + + const messages = convertMessages(makeKimiModel("kimi"), context, { + supportsStore: false, + supportsDeveloperRole: false, + supportsReasoningEffort: false, + supportsUsageInStreaming: true, + supportsFinishReason: true, + maxTokensField: "max_tokens", + requiresToolResultName: false, + requiresAssistantAfterToolResult: false, + requiresThinkingAsText: false, + requiresReasoningContentOnAssistantMessages: false, + thinkingFormat: "openai", + openRouterRouting: {}, + vercelGatewayRouting: {}, + chatTemplateKwargs: {}, + chatTemplateArgs: {}, + zaiToolStream: false, + supportsStrictMode: false, + supportsOpenAIGrammarTools: false, + cacheControlFormat: undefined, + sendSessionAffinityHeaders: false, + deferredToolsMode: "kimi", + sessionAffinityFormat: "openai", + supportsLongCacheRetention: false, + }); + + expect(messages.map((message) => message.role)).toEqual(["user", "assistant", "tool", "tool", "system", "user"]); + expect((messages[4] as { tools?: KimiTool[] }).tools?.map((tool) => tool.function.name)).toEqual([ + "late_tool", + "later_tool", + ]); + }); + + it("leaves OpenAI Completions tools unchanged without Kimi mode", async () => { + const context = makeContext([makeTool("base_tool"), makeTool("late_tool")]); + const payload = await capturePayload(makeKimiModel(), context); + + expect(payload.tools?.map((tool) => tool.function.name)).toEqual(["base_tool", "late_tool"]); + expect(payload.messages.some((message) => message.tools !== undefined)).toBe(false); + }); + + it("loads an OpenAI Responses tool through additional_tools", async () => { + const context = makeContext([makeTool("base_tool"), makeTool("late_tool")]); + const payload = await capturePayload(getModel("openai", "gpt-5.4"), context); + const additionalTools = payload.input?.find( + (item): item is OpenAIAdditionalTools => item.type === "additional_tools", + ); + + expect(openAIToolNames(payload)).toEqual(["base_tool"]); + expect(additionalTools).toMatchObject({ role: "developer" }); + expect(additionalTools?.tools).toMatchObject([{ type: "function", name: "late_tool" }]); + expect(additionalTools?.tools.every((tool) => tool.defer_loading === undefined)).toBe(true); + expect(payload.input?.some((item) => item.type === "tool_search_call")).toBe(false); + expect(payload.input?.some((item) => item.type === "tool_search_output")).toBe(false); + }); + + it("preserves an additional_tools marker after the loaded tool is used", async () => { + const context = makeContext([makeTool("base_tool"), makeTool("late_tool")]); + const lateCall: AssistantMessage = { + ...makeAssistantToolCall(), + content: [{ type: "toolCall", id: "call_late|fc_late", name: "late_tool", arguments: {} }], + api: "openai-responses", + provider: "openai", + model: "gpt-5.4", + }; + context.messages.splice(3, 0, lateCall, { + ...makeToolResult(["late_tool"]), + toolCallId: "call_late|fc_late", + toolName: "late_tool", + }); + + const payload = await capturePayload(getModel("openai", "gpt-5.4"), context); + const additionalToolIndexes = (payload.input ?? []).flatMap((item, index) => + item.type === "additional_tools" ? [index] : [], + ); + const lateCallIndex = (payload.input ?? []).findIndex( + (item) => item.type === "function_call" && item.name === "late_tool", + ); + + expect(additionalToolIndexes).toHaveLength(1); + expect(additionalToolIndexes[0]).toBeLessThan(lateCallIndex); + expect(openAIToolNames(payload)).toEqual(["base_tool"]); + }); + + it("falls back to client tool search when additional_tools is unsupported", async () => { + const model: Model<"openai-responses"> = { + ...getModel("openai", "gpt-5.4"), + provider: "openai-proxy", + compat: { supportsAdditionalTools: false, supportsToolSearch: true }, + }; + const context = makeContext([makeTool("base_tool"), makeTool("late_tool")]); + const payload = await capturePayload(model, context); + const searchCall = payload.input?.find((item): item is OpenAIToolSearchCall => item.type === "tool_search_call"); + const searchOutput = payload.input?.find( + (item): item is OpenAIToolSearchOutput => item.type === "tool_search_output", + ); + + expect(openAIToolNames(payload)).toEqual(["base_tool"]); + expect(searchCall).toMatchObject({ execution: "client", status: "completed" }); + expect(searchOutput?.call_id).toBe(searchCall?.call_id); + expect(searchOutput?.tools).toMatchObject([{ type: "function", name: "late_tool", defer_loading: true }]); + expect(payload.input?.some((item) => item.type === "additional_tools")).toBe(false); + }); + + it.each(["gpt-5.2", "gpt-5.4-nano", "gpt-5.5-pro"] as const)( + "uses the normal tool list for unsupported OpenAI model %s", + async (modelId) => { + const context = makeContext([makeTool("base_tool"), makeTool("late_tool")]); + const payload = await capturePayload(getModel("openai", modelId), context); + + expect(openAIToolNames(payload)).toEqual(["base_tool", "late_tool"]); + expect(payload.input?.some((item) => item.type === "tool_search_output")).toBe(false); + }, + ); + + it("uses the normal tool list when OpenAI tool search is explicitly disabled", async () => { + const model: Model<"openai-responses"> = { + ...getModel("openai", "gpt-5.4"), + provider: "openai-proxy", + compat: { supportsToolSearch: false }, + }; + const context = makeContext([makeTool("base_tool"), makeTool("late_tool")]); + const payload = await capturePayload(model, context); + + expect(openAIToolNames(payload)).toEqual(["base_tool", "late_tool"]); + expect(payload.input?.some((item) => item.type === "tool_search_output")).toBe(false); + }); + + it("selects additional tools, tool search, or top-level tools for Codex models", async () => { + const context = makeContext([makeTool("base_tool"), makeTool("late_tool")]); + const additionalTools = await capturePayload( + getModel("openai-codex", "gpt-5.6-sol"), + context, + makeCodexToken(), + ); + const toolSearch = await capturePayload( + getModel("openai-codex", "gpt-5.5"), + context, + makeCodexToken(), + ); + const topLevel = await capturePayload( + getModel("openai-codex", "gpt-5.3-codex-spark"), + context, + makeCodexToken(), + ); + + expect(openAIToolNames(additionalTools)).toEqual(["base_tool"]); + expect(additionalTools.input?.some((item) => item.type === "additional_tools")).toBe(true); + expect(additionalTools.input?.some((item) => item.type === "tool_search_output")).toBe(false); + expect(openAIToolNames(toolSearch)).toEqual(["base_tool"]); + expect(toolSearch.input?.some((item) => item.type === "tool_search_output")).toBe(true); + expect(openAIToolNames(topLevel)).toEqual(["base_tool", "late_tool"]); + expect(topLevel.input?.some((item) => item.type === "additional_tools")).toBe(false); + expect(topLevel.input?.some((item) => item.type === "tool_search_output")).toBe(false); + }); + + it("leaves providers without deferred loading unchanged", async () => { + const context = makeContext([makeTool("base_tool"), makeTool("late_tool")]); + const payload = await capturePayload(getModel("groq", "llama-3.3-70b-versatile"), context); + expect(openAIToolNames(payload)).toEqual(["base_tool", "late_tool"]); + }); + + it("counts definitions marked after the latest usage checkpoint", () => { + const assistant: AssistantMessage = { + ...makeAssistantToolCall(), + content: [{ type: "text", text: "done" }], + usage: { + input: 50, + output: 50, + cacheRead: 0, + cacheWrite: 0, + totalTokens: 100, + cost: { input: 0, output: 0, cacheRead: 0, cacheWrite: 0, total: 0 }, + }, + stopReason: "stop", + }; + const plain = estimateContextTokens({ messages: [assistant, makeUserMessage(4)], tools: [] }); + const lateTool = { ...makeTool("late_tool"), description: "x".repeat(4000) }; + const marked = estimateContextTokens({ + messages: [assistant, makeToolResult(["late_tool"])], + tools: [lateTool], + }); + + expect(marked.tokens).toBeGreaterThan(plain.tokens + 500); + expect(marked.trailingTokens).toBeGreaterThan(plain.trailingTokens + 500); + }); +}); diff --git a/packages/ai/test/empty.test.ts b/packages/ai/test/empty.test.ts new file mode 100644 index 0000000000000000000000000000000000000000..b45ceed68587d9fce5bfdb6d9d280958c787a339 --- /dev/null +++ b/packages/ai/test/empty.test.ts @@ -0,0 +1,822 @@ +import { describe, expect, it } from "vitest"; +import { complete, getModel } from "../src/compat.ts"; +import type { Api, AssistantMessage, Context, Model, StreamOptions, UserMessage } from "../src/types.ts"; + +type StreamOptionsWithExtras = StreamOptions & Record; + +import { hasAzureOpenAICredentials, resolveAzureDeploymentName } from "./azure-utils.ts"; +import { hasBedrockCredentials } from "./bedrock-utils.ts"; +import { hasCloudflareAiGatewayCredentials, hasCloudflareWorkersAICredentials } from "./cloudflare-utils.ts"; +import { resolveApiKey } from "./oauth.ts"; + +// Resolve OAuth tokens at module level (async, runs before tests) +const oauthTokens = await Promise.all([ + resolveApiKey("anthropic"), + resolveApiKey("github-copilot"), + resolveApiKey("openai-codex"), +]); +const [anthropicOAuthToken, githubCopilotToken, openaiCodexToken] = oauthTokens; + +async function testEmptyMessage(llm: Model, options: StreamOptionsWithExtras = {}) { + // Test with completely empty content array + const emptyMessage: UserMessage = { + role: "user", + content: [], + timestamp: Date.now(), + }; + + const context: Context = { + messages: [emptyMessage], + }; + + const response = await complete(llm, context, options); + + // Should either handle gracefully or return an error + expect(response).toBeDefined(); + expect(response.role).toBe("assistant"); + // Should handle empty string gracefully + if (response.stopReason === "error") { + expect(response.errorMessage).toBeDefined(); + } else { + expect(response.content).toBeDefined(); + } +} + +async function testEmptyStringMessage(llm: Model, options: StreamOptionsWithExtras = {}) { + // Test with empty string content + const context: Context = { + messages: [ + { + role: "user", + content: "", + timestamp: Date.now(), + }, + ], + }; + + const response = await complete(llm, context, options); + + expect(response).toBeDefined(); + expect(response.role).toBe("assistant"); + + // Should handle empty string gracefully + if (response.stopReason === "error") { + expect(response.errorMessage).toBeDefined(); + } else { + expect(response.content).toBeDefined(); + } +} + +async function testWhitespaceOnlyMessage(llm: Model, options: StreamOptionsWithExtras = {}) { + // Test with whitespace-only content + const context: Context = { + messages: [ + { + role: "user", + content: " \n\t ", + timestamp: Date.now(), + }, + ], + }; + + const response = await complete(llm, context, options); + + expect(response).toBeDefined(); + expect(response.role).toBe("assistant"); + + // Should handle whitespace-only gracefully + if (response.stopReason === "error") { + expect(response.errorMessage).toBeDefined(); + } else { + expect(response.content).toBeDefined(); + } +} + +async function testEmptyAssistantMessage(llm: Model, options: StreamOptionsWithExtras = {}) { + // Test with empty assistant message in conversation flow + // User -> Empty Assistant -> User + const emptyAssistant: AssistantMessage = { + role: "assistant", + content: [], + api: llm.api, + provider: llm.provider, + model: llm.id, + usage: { + input: 10, + output: 0, + cacheRead: 0, + cacheWrite: 0, + totalTokens: 10, + cost: { input: 0, output: 0, cacheRead: 0, cacheWrite: 0, total: 0 }, + }, + stopReason: "stop", + timestamp: Date.now(), + }; + + const context: Context = { + messages: [ + { + role: "user", + content: "Hello, how are you?", + timestamp: Date.now(), + }, + emptyAssistant, + { + role: "user", + content: "Please respond this time.", + timestamp: Date.now(), + }, + ], + }; + + const response = await complete(llm, context, options); + + expect(response).toBeDefined(); + expect(response.role).toBe("assistant"); + + // Should handle empty assistant message in context gracefully + if (response.stopReason === "error") { + expect(response.errorMessage).toBeDefined(); + } else { + expect(response.content).toBeDefined(); + expect(response.content.length).toBeGreaterThan(0); + } +} + +describe("AI Providers Empty Message Tests", () => { + describe.skipIf(!process.env.GEMINI_API_KEY)("Google Provider Empty Messages", () => { + const llm = getModel("google", "gemini-2.5-flash"); + + it("should handle empty content array", { retry: 3, timeout: 30000 }, async () => { + await testEmptyMessage(llm); + }); + + it("should handle empty string content", { retry: 3, timeout: 30000 }, async () => { + await testEmptyStringMessage(llm); + }); + + it("should handle whitespace-only content", { retry: 3, timeout: 30000 }, async () => { + await testWhitespaceOnlyMessage(llm); + }); + + it("should handle empty assistant message in conversation", { retry: 3, timeout: 30000 }, async () => { + await testEmptyAssistantMessage(llm); + }); + }); + + describe.skipIf(!process.env.OPENAI_API_KEY)("OpenAI Completions Provider Empty Messages", () => { + const llm = getModel("openai", "gpt-4o-mini"); + + it("should handle empty content array", { retry: 3, timeout: 30000 }, async () => { + await testEmptyMessage(llm); + }); + + it("should handle empty string content", { retry: 3, timeout: 30000 }, async () => { + await testEmptyStringMessage(llm); + }); + + it("should handle whitespace-only content", { retry: 3, timeout: 30000 }, async () => { + await testWhitespaceOnlyMessage(llm); + }); + + it("should handle empty assistant message in conversation", { retry: 3, timeout: 30000 }, async () => { + await testEmptyAssistantMessage(llm); + }); + }); + + describe.skipIf(!process.env.OPENAI_API_KEY)("OpenAI Responses Provider Empty Messages", () => { + const llm = getModel("openai", "gpt-5-mini"); + + it("should handle empty content array", { retry: 3, timeout: 30000 }, async () => { + await testEmptyMessage(llm); + }); + + it("should handle empty string content", { retry: 3, timeout: 30000 }, async () => { + await testEmptyStringMessage(llm); + }); + + it("should handle whitespace-only content", { retry: 3, timeout: 30000 }, async () => { + await testWhitespaceOnlyMessage(llm); + }); + + it("should handle empty assistant message in conversation", { retry: 3, timeout: 30000 }, async () => { + await testEmptyAssistantMessage(llm); + }); + }); + + describe.skipIf(!hasAzureOpenAICredentials())("Azure OpenAI Responses Provider Empty Messages", () => { + const llm = getModel("azure-openai-responses", "gpt-4o-mini"); + const azureDeploymentName = resolveAzureDeploymentName(llm.id); + const azureOptions = azureDeploymentName ? { azureDeploymentName } : {}; + + it("should handle empty content array", { retry: 3, timeout: 30000 }, async () => { + await testEmptyMessage(llm, azureOptions); + }); + + it("should handle empty string content", { retry: 3, timeout: 30000 }, async () => { + await testEmptyStringMessage(llm, azureOptions); + }); + + it("should handle whitespace-only content", { retry: 3, timeout: 30000 }, async () => { + await testWhitespaceOnlyMessage(llm, azureOptions); + }); + + it("should handle empty assistant message in conversation", { retry: 3, timeout: 30000 }, async () => { + await testEmptyAssistantMessage(llm, azureOptions); + }); + }); + + describe.skipIf(!process.env.ANTHROPIC_API_KEY)("Anthropic Provider Empty Messages", () => { + const llm = getModel("anthropic", "claude-haiku-4-5"); + + it("should handle empty content array", { retry: 3, timeout: 30000 }, async () => { + await testEmptyMessage(llm); + }); + + it("should handle empty string content", { retry: 3, timeout: 30000 }, async () => { + await testEmptyStringMessage(llm); + }); + + it("should handle whitespace-only content", { retry: 3, timeout: 30000 }, async () => { + await testWhitespaceOnlyMessage(llm); + }); + + it("should handle empty assistant message in conversation", { retry: 3, timeout: 30000 }, async () => { + await testEmptyAssistantMessage(llm); + }); + }); + + describe.skipIf(!process.env.XAI_API_KEY)("xAI Provider Empty Messages", () => { + const llm = getModel("xai", "grok-4.3"); + + it("should handle empty content array", { retry: 3, timeout: 30000 }, async () => { + await testEmptyMessage(llm); + }); + + it("should handle empty string content", { retry: 3, timeout: 30000 }, async () => { + await testEmptyStringMessage(llm); + }); + + it("should handle whitespace-only content", { retry: 3, timeout: 30000 }, async () => { + await testWhitespaceOnlyMessage(llm); + }); + + it("should handle empty assistant message in conversation", { retry: 3, timeout: 30000 }, async () => { + await testEmptyAssistantMessage(llm); + }); + }); + + describe.skipIf(!process.env.GROQ_API_KEY)("Groq Provider Empty Messages", () => { + const llm = getModel("groq", "openai/gpt-oss-20b"); + + it("should handle empty content array", { retry: 3, timeout: 30000 }, async () => { + await testEmptyMessage(llm); + }); + + it("should handle empty string content", { retry: 3, timeout: 30000 }, async () => { + await testEmptyStringMessage(llm); + }); + + it("should handle whitespace-only content", { retry: 3, timeout: 30000 }, async () => { + await testWhitespaceOnlyMessage(llm); + }); + + it("should handle empty assistant message in conversation", { retry: 3, timeout: 30000 }, async () => { + await testEmptyAssistantMessage(llm); + }); + }); + + describe.skipIf(!process.env.CEREBRAS_API_KEY)("Cerebras Provider Empty Messages", () => { + const llm = getModel("cerebras", "gpt-oss-120b"); + + it("should handle empty content array", { retry: 3, timeout: 30000 }, async () => { + await testEmptyMessage(llm); + }); + + it("should handle empty string content", { retry: 3, timeout: 30000 }, async () => { + await testEmptyStringMessage(llm); + }); + + it("should handle whitespace-only content", { retry: 3, timeout: 30000 }, async () => { + await testWhitespaceOnlyMessage(llm); + }); + + it("should handle empty assistant message in conversation", { retry: 3, timeout: 30000 }, async () => { + await testEmptyAssistantMessage(llm); + }); + }); + + describe.skipIf(!hasCloudflareWorkersAICredentials())("Cloudflare Workers AI Provider Empty Messages", () => { + const llm = getModel("cloudflare-workers-ai", "@cf/moonshotai/kimi-k2.6"); + + it("should handle empty content array", { retry: 3, timeout: 30000 }, async () => { + await testEmptyMessage(llm); + }); + + it("should handle empty string content", { retry: 3, timeout: 30000 }, async () => { + await testEmptyStringMessage(llm); + }); + + it("should handle whitespace-only content", { retry: 3, timeout: 30000 }, async () => { + await testWhitespaceOnlyMessage(llm); + }); + + it("should handle empty assistant message in conversation", { retry: 3, timeout: 30000 }, async () => { + await testEmptyAssistantMessage(llm); + }); + }); + + describe.skipIf(!hasCloudflareAiGatewayCredentials())("Cloudflare AI Gateway Provider Empty Messages", () => { + const llm = getModel("cloudflare-ai-gateway", "workers-ai/@cf/moonshotai/kimi-k2.6"); + + it("should handle empty content array", { retry: 3, timeout: 30000 }, async () => { + await testEmptyMessage(llm); + }); + + it("should handle empty string content", { retry: 3, timeout: 30000 }, async () => { + await testEmptyStringMessage(llm); + }); + + it("should handle whitespace-only content", { retry: 3, timeout: 30000 }, async () => { + await testWhitespaceOnlyMessage(llm); + }); + + it("should handle empty assistant message in conversation", { retry: 3, timeout: 30000 }, async () => { + await testEmptyAssistantMessage(llm); + }); + }); + + describe.skipIf(!process.env.HF_TOKEN)("Hugging Face Provider Empty Messages", () => { + const llm = getModel("huggingface", "moonshotai/Kimi-K2.5"); + + it("should handle empty content array", { retry: 3, timeout: 30000 }, async () => { + await testEmptyMessage(llm); + }); + + it("should handle empty string content", { retry: 3, timeout: 30000 }, async () => { + await testEmptyStringMessage(llm); + }); + + it("should handle whitespace-only content", { retry: 3, timeout: 30000 }, async () => { + await testWhitespaceOnlyMessage(llm); + }); + + it("should handle empty assistant message in conversation", { retry: 3, timeout: 30000 }, async () => { + await testEmptyAssistantMessage(llm); + }); + }); + + describe.skipIf(!process.env.TOGETHER_API_KEY)("Together AI Provider Empty Messages", () => { + const llm = getModel("together", "moonshotai/Kimi-K2.6"); + + it("should handle empty content array", { retry: 3, timeout: 30000 }, async () => { + await testEmptyMessage(llm); + }); + + it("should handle empty string content", { retry: 3, timeout: 30000 }, async () => { + await testEmptyStringMessage(llm); + }); + + it("should handle whitespace-only content", { retry: 3, timeout: 30000 }, async () => { + await testWhitespaceOnlyMessage(llm); + }); + + it("should handle empty assistant message in conversation", { retry: 3, timeout: 30000 }, async () => { + await testEmptyAssistantMessage(llm); + }); + }); + + describe.skipIf(!process.env.BASETEN_API_KEY)("Baseten Provider Empty Messages", () => { + const llm = getModel("baseten", "zai-org/GLM-5.2"); + const options = { reasoningEffort: "high" } satisfies StreamOptionsWithExtras; + + it("should handle empty content array", { retry: 3, timeout: 30000 }, async () => { + await testEmptyMessage(llm, options); + }); + + it("should handle empty string content", { retry: 3, timeout: 30000 }, async () => { + await testEmptyStringMessage(llm, options); + }); + + it("should handle whitespace-only content", { retry: 3, timeout: 30000 }, async () => { + await testWhitespaceOnlyMessage(llm, options); + }); + + it("should handle empty assistant message in conversation", { retry: 3, timeout: 30000 }, async () => { + await testEmptyAssistantMessage(llm, options); + }); + }); + + describe.skipIf(!process.env.ZAI_API_KEY)("zAI Provider Empty Messages", () => { + const llm = getModel("zai", "glm-5.2"); + + it("should handle empty content array", { retry: 3, timeout: 30000 }, async () => { + await testEmptyMessage(llm); + }); + + it("should handle empty string content", { retry: 3, timeout: 30000 }, async () => { + await testEmptyStringMessage(llm); + }); + + it("should handle whitespace-only content", { retry: 3, timeout: 30000 }, async () => { + await testWhitespaceOnlyMessage(llm); + }); + + it("should handle empty assistant message in conversation", { retry: 3, timeout: 30000 }, async () => { + await testEmptyAssistantMessage(llm); + }); + }); + + describe.skipIf(!process.env.MISTRAL_API_KEY)("Mistral Provider Empty Messages", () => { + const llm = getModel("mistral", "devstral-medium-latest"); + + it("should handle empty content array", { retry: 3, timeout: 30000 }, async () => { + await testEmptyMessage(llm); + }); + + it("should handle empty string content", { retry: 3, timeout: 30000 }, async () => { + await testEmptyStringMessage(llm); + }); + + it("should handle whitespace-only content", { retry: 3, timeout: 30000 }, async () => { + await testWhitespaceOnlyMessage(llm); + }); + + it("should handle empty assistant message in conversation", { retry: 3, timeout: 30000 }, async () => { + await testEmptyAssistantMessage(llm); + }); + }); + + describe.skipIf(!process.env.MINIMAX_API_KEY)("MiniMax Provider Empty Messages", () => { + const llm = getModel("minimax", "MiniMax-M2.7"); + + it("should handle empty content array", { retry: 3, timeout: 30000 }, async () => { + await testEmptyMessage(llm); + }); + + it("should handle empty string content", { retry: 3, timeout: 30000 }, async () => { + await testEmptyStringMessage(llm); + }); + + it("should handle whitespace-only content", { retry: 3, timeout: 30000 }, async () => { + await testWhitespaceOnlyMessage(llm); + }); + + it("should handle empty assistant message in conversation", { retry: 3, timeout: 30000 }, async () => { + await testEmptyAssistantMessage(llm); + }); + }); + + describe.skipIf(!process.env.XIAOMI_API_KEY)("Xiaomi MiMo (API billing) Provider Empty Messages", () => { + const llm = getModel("xiaomi", "mimo-v2.5-pro"); + + it("should handle empty content array", { retry: 3, timeout: 30000 }, async () => { + await testEmptyMessage(llm); + }); + + it("should handle empty string content", { retry: 3, timeout: 30000 }, async () => { + await testEmptyStringMessage(llm); + }); + + it("should handle whitespace-only content", { retry: 3, timeout: 30000 }, async () => { + await testWhitespaceOnlyMessage(llm); + }); + + it("should handle empty assistant message in conversation", { retry: 3, timeout: 30000 }, async () => { + await testEmptyAssistantMessage(llm); + }); + }); + + describe.skipIf(!process.env.XIAOMI_TOKEN_PLAN_CN_API_KEY)( + "Xiaomi MiMo Token Plan (CN) Provider Empty Messages", + () => { + const llm = getModel("xiaomi-token-plan-cn", "mimo-v2.5-pro"); + + it("should handle empty content array", { retry: 3, timeout: 30000 }, async () => { + await testEmptyMessage(llm); + }); + + it("should handle empty string content", { retry: 3, timeout: 30000 }, async () => { + await testEmptyStringMessage(llm); + }); + + it("should handle whitespace-only content", { retry: 3, timeout: 30000 }, async () => { + await testWhitespaceOnlyMessage(llm); + }); + + it("should handle empty assistant message in conversation", { retry: 3, timeout: 30000 }, async () => { + await testEmptyAssistantMessage(llm); + }); + }, + ); + + describe.skipIf(!process.env.XIAOMI_TOKEN_PLAN_AMS_API_KEY)( + "Xiaomi MiMo Token Plan (AMS) Provider Empty Messages", + () => { + const llm = getModel("xiaomi-token-plan-ams", "mimo-v2.5-pro"); + + it("should handle empty content array", { retry: 3, timeout: 30000 }, async () => { + await testEmptyMessage(llm); + }); + + it("should handle empty string content", { retry: 3, timeout: 30000 }, async () => { + await testEmptyStringMessage(llm); + }); + + it("should handle whitespace-only content", { retry: 3, timeout: 30000 }, async () => { + await testWhitespaceOnlyMessage(llm); + }); + + it("should handle empty assistant message in conversation", { retry: 3, timeout: 30000 }, async () => { + await testEmptyAssistantMessage(llm); + }); + }, + ); + + describe.skipIf(!process.env.XIAOMI_TOKEN_PLAN_SGP_API_KEY)( + "Xiaomi MiMo Token Plan (SGP) Provider Empty Messages", + () => { + const llm = getModel("xiaomi-token-plan-sgp", "mimo-v2.5-pro"); + + it("should handle empty content array", { retry: 3, timeout: 30000 }, async () => { + await testEmptyMessage(llm); + }); + + it("should handle empty string content", { retry: 3, timeout: 30000 }, async () => { + await testEmptyStringMessage(llm); + }); + + it("should handle whitespace-only content", { retry: 3, timeout: 30000 }, async () => { + await testWhitespaceOnlyMessage(llm); + }); + + it("should handle empty assistant message in conversation", { retry: 3, timeout: 30000 }, async () => { + await testEmptyAssistantMessage(llm); + }); + }, + ); + + describe.skipIf(!process.env.QWEN_TOKEN_PLAN_API_KEY)("Qwen Token Plan Provider Empty Messages", () => { + const llm = getModel("qwen-token-plan", "qwen3.7-max"); + + it("should handle empty content array", { retry: 3, timeout: 30000 }, async () => { + await testEmptyMessage(llm); + }); + + it("should handle empty string content", { retry: 3, timeout: 30000 }, async () => { + await testEmptyStringMessage(llm); + }); + + it("should handle whitespace-only content", { retry: 3, timeout: 30000 }, async () => { + await testWhitespaceOnlyMessage(llm); + }); + + it("should handle empty assistant message in conversation", { retry: 3, timeout: 30000 }, async () => { + await testEmptyAssistantMessage(llm); + }); + }); + + describe.skipIf(!process.env.QWEN_TOKEN_PLAN_API_KEY)("Qwen Token Plan Individual Provider Empty Messages", () => { + const llm = getModel("qwen-token-plan-individual", "qwen3.8-max"); + + it("should handle empty content array", { retry: 3, timeout: 30000 }, async () => { + await testEmptyMessage(llm); + }); + + it("should handle empty string content", { retry: 3, timeout: 30000 }, async () => { + await testEmptyStringMessage(llm); + }); + + it("should handle whitespace-only content", { retry: 3, timeout: 30000 }, async () => { + await testWhitespaceOnlyMessage(llm); + }); + + it("should handle empty assistant message in conversation", { retry: 3, timeout: 30000 }, async () => { + await testEmptyAssistantMessage(llm); + }); + }); + + describe.skipIf(!process.env.QWEN_TOKEN_PLAN_CN_API_KEY)("Qwen Token Plan (CN) Provider Empty Messages", () => { + const llm = getModel("qwen-token-plan-cn", "qwen3.7-max"); + + it("should handle empty content array", { retry: 3, timeout: 30000 }, async () => { + await testEmptyMessage(llm); + }); + + it("should handle empty string content", { retry: 3, timeout: 30000 }, async () => { + await testEmptyStringMessage(llm); + }); + + it("should handle whitespace-only content", { retry: 3, timeout: 30000 }, async () => { + await testWhitespaceOnlyMessage(llm); + }); + + it("should handle empty assistant message in conversation", { retry: 3, timeout: 30000 }, async () => { + await testEmptyAssistantMessage(llm); + }); + }); + + describe.skipIf(!process.env.KIMI_API_KEY)("Kimi For Coding Provider Empty Messages", () => { + const llm = getModel("kimi-coding", "kimi-for-coding"); + + it("should handle empty content array", { retry: 3, timeout: 30000 }, async () => { + await testEmptyMessage(llm); + }); + + it("should handle empty string content", { retry: 3, timeout: 30000 }, async () => { + await testEmptyStringMessage(llm); + }); + + it("should handle whitespace-only content", { retry: 3, timeout: 30000 }, async () => { + await testWhitespaceOnlyMessage(llm); + }); + + it("should handle empty assistant message in conversation", { retry: 3, timeout: 30000 }, async () => { + await testEmptyAssistantMessage(llm); + }); + }); + + describe.skipIf(!process.env.AI_GATEWAY_API_KEY)("Vercel AI Gateway Provider Empty Messages", () => { + const llm = getModel("vercel-ai-gateway", "google/gemini-2.5-flash"); + + it("should handle empty content array", { retry: 3, timeout: 30000 }, async () => { + await testEmptyMessage(llm); + }); + + it("should handle empty string content", { retry: 3, timeout: 30000 }, async () => { + await testEmptyStringMessage(llm); + }); + + it("should handle whitespace-only content", { retry: 3, timeout: 30000 }, async () => { + await testWhitespaceOnlyMessage(llm); + }); + + it("should handle empty assistant message in conversation", { retry: 3, timeout: 30000 }, async () => { + await testEmptyAssistantMessage(llm); + }); + }); + + describe.skipIf(!hasBedrockCredentials())("Amazon Bedrock Provider Empty Messages", () => { + const llm = getModel("amazon-bedrock", "global.anthropic.claude-sonnet-4-5-20250929-v1:0"); + + it("should handle empty content array", { retry: 3, timeout: 30000 }, async () => { + await testEmptyMessage(llm); + }); + + it("should handle empty string content", { retry: 3, timeout: 30000 }, async () => { + await testEmptyStringMessage(llm); + }); + + it("should handle whitespace-only content", { retry: 3, timeout: 30000 }, async () => { + await testWhitespaceOnlyMessage(llm); + }); + + it("should handle empty assistant message in conversation", { retry: 3, timeout: 30000 }, async () => { + await testEmptyAssistantMessage(llm); + }); + }); + + // ========================================================================= + // OAuth-based providers (credentials from ~/.pi/agent/oauth.json) + // ========================================================================= + + describe("Anthropic OAuth Provider Empty Messages", () => { + const llm = getModel("anthropic", "claude-haiku-4-5"); + + it.skipIf(!anthropicOAuthToken)("should handle empty content array", { retry: 3, timeout: 30000 }, async () => { + await testEmptyMessage(llm, { apiKey: anthropicOAuthToken }); + }); + + it.skipIf(!anthropicOAuthToken)("should handle empty string content", { retry: 3, timeout: 30000 }, async () => { + await testEmptyStringMessage(llm, { apiKey: anthropicOAuthToken }); + }); + + it.skipIf(!anthropicOAuthToken)( + "should handle whitespace-only content", + { retry: 3, timeout: 30000 }, + async () => { + await testWhitespaceOnlyMessage(llm, { apiKey: anthropicOAuthToken }); + }, + ); + + it.skipIf(!anthropicOAuthToken)( + "should handle empty assistant message in conversation", + { retry: 3, timeout: 30000 }, + async () => { + await testEmptyAssistantMessage(llm, { apiKey: anthropicOAuthToken }); + }, + ); + }); + + describe("GitHub Copilot Provider Empty Messages", () => { + it.skipIf(!githubCopilotToken)( + "claude-haiku-4.5 - should handle empty content array", + { retry: 3, timeout: 30000 }, + async () => { + const llm = getModel("github-copilot", "claude-haiku-4.5"); + await testEmptyMessage(llm, { apiKey: githubCopilotToken }); + }, + ); + + it.skipIf(!githubCopilotToken)( + "claude-haiku-4.5 - should handle empty string content", + { retry: 3, timeout: 30000 }, + async () => { + const llm = getModel("github-copilot", "claude-haiku-4.5"); + await testEmptyStringMessage(llm, { apiKey: githubCopilotToken }); + }, + ); + + it.skipIf(!githubCopilotToken)( + "claude-haiku-4.5 - should handle whitespace-only content", + { retry: 3, timeout: 30000 }, + async () => { + const llm = getModel("github-copilot", "claude-haiku-4.5"); + await testWhitespaceOnlyMessage(llm, { apiKey: githubCopilotToken }); + }, + ); + + it.skipIf(!githubCopilotToken)( + "claude-haiku-4.5 - should handle empty assistant message in conversation", + { retry: 3, timeout: 30000 }, + async () => { + const llm = getModel("github-copilot", "claude-haiku-4.5"); + await testEmptyAssistantMessage(llm, { apiKey: githubCopilotToken }); + }, + ); + + it.skipIf(!githubCopilotToken)( + "claude-sonnet-4 - should handle empty content array", + { retry: 3, timeout: 30000 }, + async () => { + const llm = getModel("github-copilot", "claude-sonnet-4.6"); + await testEmptyMessage(llm, { apiKey: githubCopilotToken }); + }, + ); + + it.skipIf(!githubCopilotToken)( + "claude-sonnet-4 - should handle empty string content", + { retry: 3, timeout: 30000 }, + async () => { + const llm = getModel("github-copilot", "claude-sonnet-4.6"); + await testEmptyStringMessage(llm, { apiKey: githubCopilotToken }); + }, + ); + + it.skipIf(!githubCopilotToken)( + "claude-sonnet-4 - should handle whitespace-only content", + { retry: 3, timeout: 30000 }, + async () => { + const llm = getModel("github-copilot", "claude-sonnet-4.6"); + await testWhitespaceOnlyMessage(llm, { apiKey: githubCopilotToken }); + }, + ); + + it.skipIf(!githubCopilotToken)( + "claude-sonnet-4 - should handle empty assistant message in conversation", + { retry: 3, timeout: 30000 }, + async () => { + const llm = getModel("github-copilot", "claude-sonnet-4.6"); + await testEmptyAssistantMessage(llm, { apiKey: githubCopilotToken }); + }, + ); + }); + + describe("OpenAI Codex Provider Empty Messages", () => { + it.skipIf(!openaiCodexToken)( + "gpt-5.5 - should handle empty content array", + { retry: 3, timeout: 30000 }, + async () => { + const llm = getModel("openai-codex", "gpt-5.5"); + await testEmptyMessage(llm, { apiKey: openaiCodexToken }); + }, + ); + + it.skipIf(!openaiCodexToken)( + "gpt-5.5 - should handle empty string content", + { retry: 3, timeout: 30000 }, + async () => { + const llm = getModel("openai-codex", "gpt-5.5"); + await testEmptyStringMessage(llm, { apiKey: openaiCodexToken }); + }, + ); + + it.skipIf(!openaiCodexToken)( + "gpt-5.5 - should handle whitespace-only content", + { retry: 3, timeout: 30000 }, + async () => { + const llm = getModel("openai-codex", "gpt-5.5"); + await testWhitespaceOnlyMessage(llm, { apiKey: openaiCodexToken }); + }, + ); + + it.skipIf(!openaiCodexToken)( + "gpt-5.5 - should handle empty assistant message in conversation", + { retry: 3, timeout: 30000 }, + async () => { + const llm = getModel("openai-codex", "gpt-5.5"); + await testEmptyAssistantMessage(llm, { apiKey: openaiCodexToken }); + }, + ); + }); +}); diff --git a/packages/ai/test/faux-provider.test.ts b/packages/ai/test/faux-provider.test.ts new file mode 100644 index 0000000000000000000000000000000000000000..557885f23ddcd749a7b7fcacd7d3a17a66899c92 --- /dev/null +++ b/packages/ai/test/faux-provider.test.ts @@ -0,0 +1,616 @@ +import { afterEach, describe, expect, it } from "vitest"; +import { + complete, + fauxAssistantMessage, + fauxText, + fauxThinking, + fauxToolCall, + registerFauxProvider, + stream, + Type, +} from "../src/compat.ts"; +import type { AssistantMessageEvent, Context } from "../src/types.ts"; + +async function collectEvents(streamResult: ReturnType): Promise { + const events: AssistantMessageEvent[] = []; + for await (const event of streamResult) { + events.push(event); + } + return events; +} + +const registrations: Array<{ unregister: () => void }> = []; + +afterEach(() => { + for (const registration of registrations.splice(0)) { + registration.unregister(); + } +}); + +describe("faux provider", () => { + it("registers a custom provider and estimates usage", async () => { + const registration = registerFauxProvider(); + registrations.push(registration); + registration.setResponses([fauxAssistantMessage("hello world")]); + + const context: Context = { + systemPrompt: "Be concise.", + messages: [{ role: "user", content: "hi there", timestamp: Date.now() }], + }; + + const response = await complete(registration.getModel(), context); + expect(response.content).toEqual([{ type: "text", text: "hello world" }]); + expect(response.usage.input).toBeGreaterThan(0); + expect(response.usage.output).toBeGreaterThan(0); + expect(response.usage.totalTokens).toBe(response.usage.input + response.usage.output); + expect(registration.state.callCount).toBe(1); + }); + + it("supports helper blocks for text, thinking, and tool calls", async () => { + const registration = registerFauxProvider(); + registrations.push(registration); + registration.setResponses([ + fauxAssistantMessage([fauxThinking("think"), fauxToolCall("echo", { text: "hi" }), fauxText("done")], { + stopReason: "toolUse", + }), + ]); + + const response = await complete(registration.getModel(), { + messages: [{ role: "user", content: "hi", timestamp: Date.now() }], + }); + + expect(response.content).toEqual([ + { type: "thinking", thinking: "think" }, + { type: "toolCall", id: expect.any(String), name: "echo", arguments: { text: "hi" } }, + { type: "text", text: "done" }, + ]); + expect(response.stopReason).toBe("toolUse"); + }); + + it("supports multiple models with per-model reasoning and model-aware factories", async () => { + const registration = registerFauxProvider({ + models: [ + { id: "faux-fast", name: "Faux Fast", reasoning: false }, + { id: "faux-thinker", name: "Faux Thinker", reasoning: true }, + ], + }); + registrations.push(registration); + registration.setResponses([ + (_context, _options, _state, model) => fauxAssistantMessage(`${model.id}:${String(model.reasoning)}`), + (_context, _options, _state, model) => fauxAssistantMessage(`${model.id}:${String(model.reasoning)}`), + ]); + + expect(registration.models.map((model) => model.id)).toEqual(["faux-fast", "faux-thinker"]); + expect(registration.getModel()).toBe(registration.models[0]); + expect(registration.getModel("faux-fast")?.reasoning).toBe(false); + expect(registration.getModel("faux-thinker")?.reasoning).toBe(true); + + const fast = await complete(registration.getModel("faux-fast")!, { + messages: [{ role: "user", content: "hi", timestamp: Date.now() }], + }); + const thinker = await complete(registration.getModel("faux-thinker")!, { + messages: [{ role: "user", content: "hi", timestamp: Date.now() }], + }); + + expect(fast.content).toEqual([{ type: "text", text: "faux-fast:false" }]); + expect(thinker.content).toEqual([{ type: "text", text: "faux-thinker:true" }]); + }); + + it("rewrites api, provider, and model on returned messages", async () => { + const registration = registerFauxProvider({ + api: "faux:test", + provider: "faux-provider", + models: [{ id: "faux-model" }], + }); + registrations.push(registration); + registration.setResponses([fauxAssistantMessage("hello")]); + + const response = await complete(registration.getModel(), { + messages: [{ role: "user", content: "hi", timestamp: Date.now() }], + }); + + expect(response.api).toBe("faux:test"); + expect(response.provider).toBe("faux-provider"); + expect(response.model).toBe("faux-model"); + }); + + it("consumes queued responses in order and errors when exhausted", async () => { + const registration = registerFauxProvider(); + registrations.push(registration); + registration.setResponses([fauxAssistantMessage("first"), fauxAssistantMessage("second")]); + + const context: Context = { + messages: [{ role: "user", content: "hi", timestamp: Date.now() }], + }; + + const first = await complete(registration.getModel(), context); + const second = await complete(registration.getModel(), context); + const exhausted = await complete(registration.getModel(), context); + + expect(first.content).toEqual([{ type: "text", text: "first" }]); + expect(second.content).toEqual([{ type: "text", text: "second" }]); + expect(exhausted.stopReason).toBe("error"); + expect(exhausted.errorMessage).toBe("No more faux responses queued"); + expect(registration.getPendingResponseCount()).toBe(0); + expect(registration.state.callCount).toBe(3); + }); + + it("can replace and append queued responses", async () => { + const registration = registerFauxProvider(); + registrations.push(registration); + registration.setResponses([fauxAssistantMessage("first")]); + + const context: Context = { + messages: [{ role: "user", content: "hi", timestamp: Date.now() }], + }; + + expect((await complete(registration.getModel(), context)).content).toEqual([{ type: "text", text: "first" }]); + expect(registration.getPendingResponseCount()).toBe(0); + + registration.setResponses([fauxAssistantMessage("second")]); + expect(registration.getPendingResponseCount()).toBe(1); + expect((await complete(registration.getModel(), context)).content).toEqual([{ type: "text", text: "second" }]); + + registration.appendResponses([fauxAssistantMessage("third"), fauxAssistantMessage("fourth")]); + expect(registration.getPendingResponseCount()).toBe(2); + expect((await complete(registration.getModel(), context)).content).toEqual([{ type: "text", text: "third" }]); + expect((await complete(registration.getModel(), context)).content).toEqual([{ type: "text", text: "fourth" }]); + expect(registration.getPendingResponseCount()).toBe(0); + }); + + it("supports async response factories", async () => { + const registration = registerFauxProvider(); + registrations.push(registration); + registration.setResponses([ + async (context, _options, state) => fauxAssistantMessage(`${context.messages.length}:${state.callCount}`), + ]); + + const response = await complete(registration.getModel(), { + messages: [{ role: "user", content: "hi", timestamp: Date.now() }], + }); + + expect(response.content).toEqual([{ type: "text", text: "1:1" }]); + }); + + it("emits an error when a response factory throws", async () => { + const registration = registerFauxProvider(); + registrations.push(registration); + registration.setResponses([ + () => { + throw new Error("boom"); + }, + ]); + + const events = await collectEvents( + stream(registration.getModel(), { messages: [{ role: "user", content: "hi", timestamp: Date.now() }] }), + ); + + expect(events).toHaveLength(1); + expect(events[0].type).toBe("error"); + if (events[0].type === "error") { + expect(events[0].error.stopReason).toBe("error"); + expect(events[0].error.errorMessage).toBe("boom"); + } + }); + + it("rejects a queued response without a terminal stop reason", async () => { + const registration = registerFauxProvider(); + registrations.push(registration); + registration.setResponses([fauxAssistantMessage("partial", { stopReason: "pending" })]); + + const events = await collectEvents( + stream(registration.getModel(), { messages: [{ role: "user", content: "hi", timestamp: Date.now() }] }), + ); + + expect(events.some((event) => event.type === "done")).toBe(false); + const terminal = events.at(-1); + expect(terminal?.type).toBe("error"); + if (terminal?.type === "error") { + expect(terminal.error.stopReason).toBe("error"); + expect(terminal.error.errorMessage).toBe("Faux response ended without a stop reason"); + } + }); + + it("estimates prompt and output tokens from serialized context", async () => { + const registration = registerFauxProvider(); + registrations.push(registration); + registration.setResponses([fauxAssistantMessage("done")]); + + const tool = { + name: "echo", + description: "Echo back text", + parameters: Type.Object({ text: Type.String() }), + }; + const context: Context = { + systemPrompt: "sys", + messages: [ + { + role: "user", + content: [ + { type: "text", text: "hello" }, + { type: "image", mimeType: "image/png", data: "abcd" }, + ], + timestamp: 1, + }, + fauxAssistantMessage("prior"), + { + role: "toolResult", + toolCallId: "tool-1", + toolName: "echo", + content: [{ type: "text", text: "tool out" }], + isError: false, + timestamp: 2, + }, + ], + tools: [tool], + }; + + const response = await complete(registration.getModel(), context); + const promptText = [ + "system:sys", + "user:hello\n[image:image/png:4]", + "assistant:prior", + "toolResult:echo\ntool out", + `tools:${JSON.stringify([tool])}`, + ].join("\n\n"); + const expectedPromptTokens = Math.ceil(promptText.length / 4); + const expectedOutputTokens = Math.ceil("done".length / 4); + + expect(response.usage.input).toBe(expectedPromptTokens); + expect(response.usage.output).toBe(expectedOutputTokens); + expect(response.usage.cacheRead).toBe(0); + expect(response.usage.cacheWrite).toBe(0); + expect(response.usage.totalTokens).toBe(expectedPromptTokens + expectedOutputTokens); + }); + + it("does not share cache across sessions or requests without sessionId", async () => { + const registration = registerFauxProvider(); + registrations.push(registration); + registration.setResponses([ + fauxAssistantMessage("first"), + fauxAssistantMessage("second"), + fauxAssistantMessage("third"), + ]); + + const context: Context = { + messages: [{ role: "user", content: "hello", timestamp: Date.now() }], + }; + + const first = await complete(registration.getModel(), context, { + sessionId: "session-1", + cacheRetention: "short", + }); + expect(first.usage.cacheWrite).toBeGreaterThan(0); + context.messages.push(first); + context.messages.push({ role: "user", content: "follow up", timestamp: Date.now() + 1 }); + + const second = await complete(registration.getModel(), context, { + sessionId: "session-2", + cacheRetention: "short", + }); + expect(second.usage.cacheRead).toBe(0); + expect(second.usage.cacheWrite).toBeGreaterThan(0); + + const third = await complete(registration.getModel(), context); + expect(third.usage.cacheRead).toBe(0); + expect(third.usage.cacheWrite).toBe(0); + }); + + it("simulates prompt caching per sessionId", async () => { + const registration = registerFauxProvider(); + registrations.push(registration); + registration.setResponses([fauxAssistantMessage("first"), fauxAssistantMessage("second")]); + + const context: Context = { + systemPrompt: "Be concise.", + messages: [{ role: "user", content: "hello", timestamp: Date.now() }], + }; + + const first = await complete(registration.getModel(), context, { + sessionId: "session-1", + cacheRetention: "short", + }); + expect(first.usage.cacheRead).toBe(0); + expect(first.usage.cacheWrite).toBeGreaterThan(0); + + context.messages.push(first); + context.messages.push({ role: "user", content: "follow up", timestamp: Date.now() + 1 }); + + const second = await complete(registration.getModel(), context, { + sessionId: "session-1", + cacheRetention: "short", + }); + expect(second.usage.cacheRead).toBeGreaterThan(0); + expect(second.usage.input + second.usage.cacheRead).toBeGreaterThan(second.usage.input); + }); + + it("does not simulate caching when cacheRetention is none", async () => { + const registration = registerFauxProvider(); + registrations.push(registration); + registration.setResponses([fauxAssistantMessage("first"), fauxAssistantMessage("second")]); + + const context: Context = { + messages: [{ role: "user", content: "hello", timestamp: Date.now() }], + }; + + await complete(registration.getModel(), context, { sessionId: "session-1", cacheRetention: "none" }); + context.messages.push(fauxAssistantMessage("first")); + context.messages.push({ role: "user", content: "follow up", timestamp: Date.now() + 1 }); + const second = await complete(registration.getModel(), context, { + sessionId: "session-1", + cacheRetention: "none", + }); + expect(second.usage.cacheRead).toBe(0); + expect(second.usage.cacheWrite).toBe(0); + }); + + it("streams thinking, text, and partial tool call deltas", async () => { + const registration = registerFauxProvider(); + registrations.push(registration); + registration.setResponses([ + fauxAssistantMessage( + [ + fauxThinking("thinking text"), + fauxText("answer text"), + fauxToolCall("echo", { text: "hi", count: 12 }, { id: "tool-1" }), + ], + { stopReason: "toolUse" }, + ), + ]); + + const events: string[] = []; + const toolCallDeltas: string[] = []; + const s = stream(registration.getModel(), { messages: [{ role: "user", content: "hi", timestamp: Date.now() }] }); + for await (const event of s) { + events.push(event.type); + if (event.type === "toolcall_delta") { + toolCallDeltas.push(event.delta); + } + } + + expect(events).toContain("thinking_start"); + expect(events).toContain("thinking_delta"); + expect(events).toContain("text_start"); + expect(events).toContain("text_delta"); + expect(events).toContain("toolcall_start"); + expect(events).toContain("toolcall_delta"); + expect(events).toContain("toolcall_end"); + expect(toolCallDeltas.length).toBeGreaterThan(1); + expect(JSON.parse(toolCallDeltas.join(""))).toEqual({ text: "hi", count: 12 }); + }); + + it("streams an exact event order for fixed-size chunks", async () => { + const registration = registerFauxProvider({ tokenSize: { min: 1, max: 1 } }); + registrations.push(registration); + registration.setResponses([ + fauxAssistantMessage([fauxThinking("go"), fauxText("ok"), fauxToolCall("echo", {}, { id: "tool-1" })], { + stopReason: "toolUse", + }), + ]); + + const events = await collectEvents( + stream(registration.getModel(), { messages: [{ role: "user", content: "hi", timestamp: Date.now() }] }), + ); + + expect(events[0]).toMatchObject({ type: "start", partial: { stopReason: "pending" } }); + expect(events.map((event) => event.type)).toEqual([ + "start", + "thinking_start", + "thinking_delta", + "thinking_end", + "text_start", + "text_delta", + "text_end", + "toolcall_start", + "toolcall_delta", + "toolcall_end", + "done", + ]); + }); + + it("streams multiple tool calls in one message", async () => { + const registration = registerFauxProvider(); + registrations.push(registration); + registration.setResponses([ + fauxAssistantMessage( + [ + fauxToolCall("echo", { text: "one" }, { id: "tool-1" }), + fauxToolCall("echo", { text: "two" }, { id: "tool-2" }), + ], + { stopReason: "toolUse" }, + ), + ]); + + const events = await collectEvents( + stream(registration.getModel(), { messages: [{ role: "user", content: "hi", timestamp: Date.now() }] }), + ); + + expect(events.filter((event) => event.type === "toolcall_start")).toHaveLength(2); + expect(events.filter((event) => event.type === "toolcall_end")).toHaveLength(2); + }); + + it("streams an explicit assistant error message as a terminal error", async () => { + const registration = registerFauxProvider({ tokenSize: { min: 2, max: 2 } }); + registrations.push(registration); + registration.setResponses([ + { + ...fauxAssistantMessage("partial"), + stopReason: "error", + errorMessage: "upstream failed", + }, + ]); + + const events = await collectEvents( + stream(registration.getModel(), { messages: [{ role: "user", content: "hi", timestamp: Date.now() }] }), + ); + + expect(events.map((event) => event.type)).toEqual(["start", "text_start", "text_delta", "text_end", "error"]); + const terminal = events[events.length - 1]; + expect(terminal.type).toBe("error"); + if (terminal.type === "error") { + expect(terminal.reason).toBe("error"); + expect(terminal.error.stopReason).toBe("error"); + expect(terminal.error.errorMessage).toBe("upstream failed"); + } + }); + + it("streams an explicit assistant aborted message as a terminal error", async () => { + const registration = registerFauxProvider({ tokenSize: { min: 2, max: 2 } }); + registrations.push(registration); + registration.setResponses([ + { + ...fauxAssistantMessage("partial"), + stopReason: "aborted", + errorMessage: "Request was aborted", + }, + ]); + + const events = await collectEvents( + stream(registration.getModel(), { messages: [{ role: "user", content: "hi", timestamp: Date.now() }] }), + ); + + expect(events.map((event) => event.type)).toEqual(["start", "text_start", "text_delta", "text_end", "error"]); + const terminal = events[events.length - 1]; + expect(terminal.type).toBe("error"); + if (terminal.type === "error") { + expect(terminal.reason).toBe("aborted"); + expect(terminal.error.stopReason).toBe("aborted"); + expect(terminal.error.errorMessage).toBe("Request was aborted"); + } + }); + + it("supports aborting before the first chunk", async () => { + const registration = registerFauxProvider({ tokensPerSecond: 50, tokenSize: { min: 3, max: 3 } }); + registrations.push(registration); + registration.setResponses([fauxAssistantMessage("abcdefghijklmnopqrstuvwxyz")]); + + const controller = new AbortController(); + controller.abort(); + const events = await collectEvents( + stream( + registration.getModel(), + { messages: [{ role: "user", content: "hi", timestamp: Date.now() }] }, + { signal: controller.signal }, + ), + ); + + expect(events).toHaveLength(1); + expect(events[0].type).toBe("error"); + if (events[0].type === "error") { + expect(events[0].reason).toBe("aborted"); + expect(events[0].error.stopReason).toBe("aborted"); + } + }); + + it("supports aborting mid-text stream when paced", async () => { + const registration = registerFauxProvider({ tokensPerSecond: 100, tokenSize: { min: 3, max: 3 } }); + registrations.push(registration); + registration.setResponses([fauxAssistantMessage("abcdefghijklmnopqrstuvwxyz")]); + + const controller = new AbortController(); + const events: string[] = []; + let textDeltaCount = 0; + const s = stream( + registration.getModel(), + { messages: [{ role: "user", content: "hi", timestamp: Date.now() }] }, + { signal: controller.signal }, + ); + for await (const event of s) { + events.push(event.type); + if (event.type === "text_delta") { + textDeltaCount++; + controller.abort(); + } + } + + expect(textDeltaCount).toBe(1); + expect(events).toContain("text_start"); + expect(events).toContain("text_delta"); + expect(events).toContain("error"); + expect(events).not.toContain("text_end"); + }); + + it("supports aborting mid-thinking stream when paced", async () => { + const registration = registerFauxProvider({ tokensPerSecond: 100, tokenSize: { min: 3, max: 3 } }); + registrations.push(registration); + registration.setResponses([ + { + ...fauxAssistantMessage("ignored"), + content: [{ type: "thinking", thinking: "abcdefghijklmnopqrstuvwxyz" }], + }, + ]); + + const controller = new AbortController(); + const events: string[] = []; + let thinkingDeltaCount = 0; + const s = stream( + registration.getModel(), + { messages: [{ role: "user", content: "hi", timestamp: Date.now() }] }, + { signal: controller.signal }, + ); + for await (const event of s) { + events.push(event.type); + if (event.type === "thinking_delta") { + thinkingDeltaCount++; + controller.abort(); + } + } + + expect(thinkingDeltaCount).toBe(1); + expect(events).toContain("thinking_start"); + expect(events).toContain("thinking_delta"); + expect(events).toContain("error"); + expect(events).not.toContain("thinking_end"); + }); + + it("supports aborting mid-toolcall stream when paced", async () => { + const registration = registerFauxProvider({ tokensPerSecond: 100, tokenSize: { min: 3, max: 3 } }); + registrations.push(registration); + registration.setResponses([ + { + ...fauxAssistantMessage("done"), + content: [ + { + type: "toolCall", + id: "tool-1", + name: "echo", + arguments: { text: "abcdefghijklmnopqrstuvwxyz", count: 123456789 }, + }, + ], + stopReason: "toolUse", + }, + ]); + + const controller = new AbortController(); + const events: string[] = []; + let toolCallDeltaCount = 0; + const s = stream( + registration.getModel(), + { messages: [{ role: "user", content: "hi", timestamp: Date.now() }] }, + { signal: controller.signal }, + ); + for await (const event of s) { + events.push(event.type); + if (event.type === "toolcall_delta") { + toolCallDeltaCount++; + controller.abort(); + } + } + + expect(toolCallDeltaCount).toBe(1); + expect(events).toContain("toolcall_start"); + expect(events).toContain("toolcall_delta"); + expect(events).toContain("error"); + expect(events).not.toContain("toolcall_end"); + }); + + it("unregisters the provider", async () => { + const registration = registerFauxProvider(); + registration.setResponses([fauxAssistantMessage("hello")]); + registration.unregister(); + + await expect( + complete(registration.getModel(), { messages: [{ role: "user", content: "hi", timestamp: Date.now() }] }), + ).rejects.toThrow(`No API provider registered for api: ${registration.api}`); + }); +}); diff --git a/packages/ai/test/fireworks-models.test.ts b/packages/ai/test/fireworks-models.test.ts new file mode 100644 index 0000000000000000000000000000000000000000..c5d1dbead186d3f9fd5860f7ebcc900dd3cf0ed1 --- /dev/null +++ b/packages/ai/test/fireworks-models.test.ts @@ -0,0 +1,450 @@ +import { createServer, type IncomingMessage, type ServerResponse } from "node:http"; +import type { AddressInfo } from "node:net"; +import { Type } from "typebox"; +import { afterEach, describe, expect, it } from "vitest"; +import { stream as streamAnthropic } from "../src/api/anthropic-messages.ts"; +import { getModel, getModels, streamSimple } from "../src/compat.ts"; +import { findEnvKeys, getEnvApiKey } from "../src/env-api-keys.ts"; +import { getSupportedThinkingLevels } from "../src/models.ts"; +import type { Context, Model, Tool } from "../src/types.ts"; + +const originalFireworksApiKey = process.env.FIREWORKS_API_KEY; + +afterEach(() => { + if (originalFireworksApiKey === undefined) { + delete process.env.FIREWORKS_API_KEY; + } else { + process.env.FIREWORKS_API_KEY = originalFireworksApiKey; + } +}); + +describe("Fireworks models", () => { + it("enables native tool references only on Messages models", () => { + for (const model of getModels("fireworks")) { + if (model.api === "anthropic-messages") { + expect(model.compat).toMatchObject({ supportsToolReferences: true }); + } else { + expect(model.compat).not.toHaveProperty("supportsToolReferences"); + } + } + }); + + it("registers the default Kimi K2.6 model via Anthropic-compatible Messages API", () => { + const model = getModel("fireworks", "accounts/fireworks/models/kimi-k2p6"); + + expect(model).toBeDefined(); + expect(model.api).toBe("anthropic-messages"); + expect(model.provider).toBe("fireworks"); + expect(model.baseUrl).toBe("https://api.fireworks.ai/inference"); + expect(model.reasoning).toBe(true); + expect(model.input).toEqual(["text", "image"]); + expect(model.contextWindow).toBe(262000); + expect(model.maxTokens).toBe(262000); + expect(model.cost).toEqual({ + input: 0.95, + output: 4, + cacheRead: 0.16, + cacheWrite: 0, + }); + }); + + it("aligns GLM 5.2 Fast with GLM 5.2's OpenAI-compatible config", () => { + const base = getModel("fireworks", "accounts/fireworks/models/glm-5p2"); + const fast = getModel("fireworks", "accounts/fireworks/routers/glm-5p2-fast"); + + expect(fast.api).toBe(base.api); + expect(fast.baseUrl).toBe(base.baseUrl); + expect(fast.compat).toEqual(base.compat); + expect(fast.thinkingLevelMap).toEqual(base.thinkingLevelMap); + }); + + it.each(["accounts/fireworks/models/glm-5p2", "accounts/fireworks/routers/glm-5p2-fast"] as const)( + "omits unsupported long cache retention for %s", + async (modelId) => { + const model = getModel("fireworks", modelId); + let payload: Record | undefined; + const response = streamSimple( + model, + { messages: [{ role: "user", content: "test", timestamp: 0 }] }, + { + apiKey: "test-fireworks-key", + cacheRetention: "long", + sessionId: "test-fireworks-session", + onPayload: (value) => { + payload = value as Record; + throw new Error("payload captured"); + }, + }, + ); + await response.result(); + + expect(payload).toBeDefined(); + expect(payload?.prompt_cache_retention).toBeUndefined(); + }, + ); + + it("routes Kimi K3 through the OpenAI-compatible API with native effort controls", async () => { + const base = getModel("fireworks", "accounts/fireworks/models/kimi-k3"); + const fast = getModel("fireworks", "accounts/fireworks/routers/kimi-k3-fast"); + const compat = { + supportsStore: false, + supportsDeveloperRole: false, + requiresReasoningContentOnAssistantMessages: true, + thinkingFormat: "openai", + deferredToolsMode: "kimi", + sendSessionAffinityHeaders: true, + supportsLongCacheRetention: false, + }; + const thinkingLevelMap = { + off: null, + minimal: null, + low: "low", + medium: null, + high: "high", + xhigh: null, + max: "max", + }; + + expect(base.api).toBe("openai-completions"); + expect(base.baseUrl).toBe("https://api.fireworks.ai/inference/v1"); + expect(base.compat).toEqual(compat); + expect(base.thinkingLevelMap).toEqual(thinkingLevelMap); + expect(fast.api).toBe(base.api); + expect(fast.baseUrl).toBe(base.baseUrl); + expect(fast.compat).toEqual(compat); + expect(fast.thinkingLevelMap).toEqual(thinkingLevelMap); + + let payload: Record | undefined; + const response = streamSimple( + base, + { messages: [{ role: "user", content: "test", timestamp: 0 }] }, + { + apiKey: "test-fireworks-key", + reasoning: "max", + onPayload: (value) => { + payload = value as Record; + throw new Error("payload captured"); + }, + }, + ); + await response.result(); + + expect(payload?.reasoning_effort).toBe("max"); + }); + + // Regression for #9323: native effort must reach Messages without budget-based fallback. + it.each([ + ["accounts/fireworks/models/deepseek-v4-flash-0731", ["off", "low", "high", "max"]], + ["accounts/fireworks/models/deepseek-v4-flash-vision-exp", ["off", "low", "high", "max"]], + ["accounts/fireworks/models/deepseek-v4-pro-0813", ["off", "low", "high", "max"]], + ["accounts/fireworks/models/qwen3p8-max", ["off", "low", "medium", "xhigh"]], + ["accounts/fireworks/models/qwen3p8-2p4t-a95b", ["off", "low", "medium", "xhigh"]], + ] as const)("sends native Messages effort levels for %s", async (modelId, levels) => { + const model = getModel("fireworks", modelId); + expect(model.api).toBe("anthropic-messages"); + expect(model.compat?.forceAdaptiveThinking).toBe(true); + expect(getSupportedThinkingLevels(model)).toEqual(levels); + + for (const level of levels) { + let payload: Record | undefined; + await streamSimple( + model, + { messages: [{ role: "user", content: "test", timestamp: 0 }] }, + { + apiKey: "test-fireworks-key", + reasoning: level === "off" ? undefined : level, + onPayload: (value) => { + payload = value as Record; + throw new Error("payload captured"); + }, + }, + ).result(); + expect(payload).toBeDefined(); + expect(payload?.thinking).toEqual( + level === "off" ? { type: "disabled" } : { type: "adaptive", display: "summarized" }, + ); + expect(payload?.output_config).toEqual(level === "off" ? undefined : { effort: level }); + } + }); + + // Regression for #9323: accepted aliases are not distinct native effort levels. + it.each([ + ["accounts/fireworks/models/glm-5p2", ["off", "high", "max"]], + ["accounts/fireworks/routers/glm-5p2-fast", ["off", "high", "max"]], + ["accounts/fireworks/models/kimi-k3", ["low", "high", "max"]], + ["accounts/fireworks/routers/kimi-k3-fast", ["low", "high", "max"]], + ] as const)("exposes distinct native effort levels for %s", (modelId, levels) => { + expect(getSupportedThinkingLevels(getModel("fireworks", modelId))).toEqual(levels); + }); + + it("keeps toggle-only Messages models without a verified fallback on budget-based thinking", async () => { + const model = getModel("fireworks", "accounts/fireworks/models/kimi-k2p6"); + expect(model.compat?.forceAdaptiveThinking).toBeUndefined(); + let payload: Record | undefined; + await streamSimple( + model, + { messages: [{ role: "user", content: "test", timestamp: 0 }] }, + { + apiKey: "test-fireworks-key", + reasoning: "high", + onPayload: (value) => { + payload = value as Record; + throw new Error("payload captured"); + }, + }, + ).result(); + expect(payload?.thinking).toEqual({ type: "enabled", budget_tokens: 16384, display: "summarized" }); + expect(payload?.output_config).toBeUndefined(); + }); + + it("resolves FIREWORKS_API_KEY from the environment", () => { + process.env.FIREWORKS_API_KEY = "test-fireworks-key"; + + expect(findEnvKeys("fireworks")).toEqual(["FIREWORKS_API_KEY"]); + expect(getEnvApiKey("fireworks")).toBe("test-fireworks-key"); + }); + + it("sets Fireworks-specific compat for session affinity and unsupported tool fields", () => { + const model = getModel("fireworks", "accounts/fireworks/models/kimi-k2p6"); + + expect(model.compat).toBeDefined(); + expect(model.compat?.sendSessionAffinityHeaders).toBe(true); + expect(model.compat?.supportsEagerToolInputStreaming).toBe(false); + expect(model.compat?.supportsCacheControlOnTools).toBe(false); + expect(model.compat?.supportsLongCacheRetention).toBe(false); + expect(model.compat?.allowEmptySignature).toBe(true); + expect(model.compat?.supportsToolReferences).toBe(true); + }); +}); + +// --- Integration tests for Fireworks Anthropic session affinity and tool compat --- + +interface CapturedRequest { + headers: IncomingMessage["headers"]; + body: Record; +} + +const tool: Tool = { + name: "lookup", + description: "Look up a value", + parameters: Type.Object({ value: Type.String() }), +}; + +const FIREWORKS_ANTHROPIC_COMPAT = { + supportsToolReferences: true, + allowEmptySignature: true, + sendSessionAffinityHeaders: true, + supportsEagerToolInputStreaming: false, + supportsCacheControlOnTools: false, + supportsLongCacheRetention: false, +} satisfies NonNullable["compat"]>; + +function createFireworksModel( + compat: Model<"anthropic-messages">["compat"] = FIREWORKS_ANTHROPIC_COMPAT, +): Model<"anthropic-messages"> { + return { + id: "accounts/fireworks/models/kimi-k2p6", + name: "Kimi K2.6", + api: "anthropic-messages", + provider: "fireworks", + baseUrl: "http://127.0.0.1:0", // overridden by captureAnthropicRequest + reasoning: true, + input: ["text", "image"], + cost: { input: 0.95, output: 4, cacheRead: 0.16, cacheWrite: 0 }, + contextWindow: 262000, + maxTokens: 262000, + compat, + }; +} + +function createAnthropicModel(): Model<"anthropic-messages"> { + return { + id: "claude-opus-4-8", + name: "Claude Opus 4.8", + api: "anthropic-messages", + provider: "anthropic", + baseUrl: "http://127.0.0.1:0", // overridden by captureAnthropicRequest + reasoning: true, + input: ["text"], + cost: { input: 0, output: 0, cacheRead: 0, cacheWrite: 0 }, + contextWindow: 200000, + maxTokens: 32000, + }; +} + +function createOpenRouterModel(): Model<"anthropic-messages"> { + return { + ...createAnthropicModel(), + id: "anthropic/claude-opus-4.8", + provider: "openrouter", + baseUrl: "https://openrouter.ai/api", + }; +} + +function createContext(tools: Tool[] = [tool]): Context { + return { + messages: [{ role: "user", content: "Use the tool", timestamp: Date.now() }], + ...(tools.length > 0 ? { tools } : {}), + }; +} + +async function readRequestBody(request: IncomingMessage): Promise> { + const chunks: Buffer[] = []; + for await (const chunk of request) { + chunks.push(Buffer.isBuffer(chunk) ? chunk : Buffer.from(chunk)); + } + return JSON.parse(Buffer.concat(chunks).toString("utf8")) as Record; +} + +function writeEmptySseResponse(response: ServerResponse): void { + response.writeHead(200, { "content-type": "text/event-stream" }); + response.end(); +} + +async function captureAnthropicRequest( + model: Model<"anthropic-messages">, + context: Context, + options?: { sessionId?: string; cacheRetention?: string }, +): Promise { + let capturedRequest: CapturedRequest | undefined; + + const server = createServer(async (request, response) => { + capturedRequest = { + headers: request.headers, + body: await readRequestBody(request), + }; + writeEmptySseResponse(response); + }); + + await new Promise((resolve) => server.listen(0, "127.0.0.1", resolve)); + const address = server.address() as AddressInfo; + + try { + // Override the model's baseUrl to point to the local test server + const localModel = { ...model, baseUrl: `http://127.0.0.1:${address.port}` }; + + const stream = streamAnthropic(localModel, context, { + apiKey: "test-key", + cacheRetention: (options?.cacheRetention as "none" | "short" | "long") ?? "short", + sessionId: options?.sessionId, + }); + + for await (const event of stream) { + if (event.type === "done" || event.type === "error") break; + } + } finally { + await new Promise((resolve, reject) => { + server.close((error) => (error ? reject(error) : resolve())); + }); + } + + if (!capturedRequest) { + throw new Error("Anthropic request was not captured"); + } + return capturedRequest; +} + +function getTools(body: Record): Record[] { + const tools = body.tools; + if (!Array.isArray(tools)) { + throw new Error("Expected tools in request body"); + } + return tools as Record[]; +} + +describe("Anthropic-compatible session affinity and tool compat", () => { + it("sends x-session-affinity header for Fireworks models", async () => { + const model = createFireworksModel(); + // Need a real port, capture will assign one + const request = await captureAnthropicRequest(model, createContext(), { + sessionId: "fireworks-session-1", + }); + + expect(request.headers["x-session-affinity"]).toBe("fireworks-session-1"); + }); + + it("omits x-session-affinity header for native Anthropic models", async () => { + const model = createAnthropicModel(); + const request = await captureAnthropicRequest(model, createContext(), { + sessionId: "anthropic-session-1", + }); + + expect(request.headers["x-session-affinity"]).toBeUndefined(); + }); + + it("omits x-session-affinity header when cacheRetention is none", async () => { + const model = createFireworksModel(); + const request = await captureAnthropicRequest(model, createContext(), { + sessionId: "fireworks-session-2", + cacheRetention: "none", + }); + + expect(request.headers["x-session-affinity"]).toBeUndefined(); + }); + + // Regression test for https://github.com/earendil-works/pi/issues/9102 + it("sends only x-session-id for OpenRouter models", async () => { + const request = await captureAnthropicRequest(createOpenRouterModel(), createContext(), { + sessionId: "openrouter-session-1", + }); + + expect(request.headers["x-session-id"]).toBe("openrouter-session-1"); + expect(request.headers["x-session-affinity"]).toBeUndefined(); + }); + + it("omits OpenRouter session headers when cacheRetention is none", async () => { + const request = await captureAnthropicRequest(createOpenRouterModel(), createContext(), { + sessionId: "openrouter-session-2", + cacheRetention: "none", + }); + + expect(request.headers["x-session-id"]).toBeUndefined(); + expect(request.headers["x-session-affinity"]).toBeUndefined(); + }); + + it("allows OpenRouter session headers to be disabled", async () => { + const model = { ...createOpenRouterModel(), compat: { sendSessionAffinityHeaders: false } }; + const request = await captureAnthropicRequest(model, createContext(), { + sessionId: "openrouter-session-3", + }); + + expect(request.headers["x-session-id"]).toBeUndefined(); + }); + + it("omits cache_control on tools for Fireworks models", async () => { + const model = createFireworksModel(); + const request = await captureAnthropicRequest(model, createContext()); + + const tools = getTools(request.body); + const lastTool = tools[tools.length - 1]; + expect(lastTool.cache_control).toBeUndefined(); + }); + + it("omits eager_input_streaming on tools for Fireworks models", async () => { + const model = createFireworksModel(); + const request = await captureAnthropicRequest(model, createContext()); + + const tools = getTools(request.body); + for (const t of tools) { + expect(t.eager_input_streaming).toBeUndefined(); + } + }); + + it("sends cache_control on tools for native Anthropic models", async () => { + const model = createAnthropicModel(); + const request = await captureAnthropicRequest(model, createContext()); + + const tools = getTools(request.body); + const lastTool = tools[tools.length - 1]; + expect(lastTool.cache_control).toBeDefined(); + expect((lastTool.cache_control as { type: string }).type).toBe("ephemeral"); + }); + + it("sends eager_input_streaming on tools for native Anthropic models", async () => { + const model = createAnthropicModel(); + const request = await captureAnthropicRequest(model, createContext()); + + const tools = getTools(request.body); + expect(tools[0].eager_input_streaming).toBe(true); + }); +}); diff --git a/packages/ai/test/github-copilot-anthropic.test.ts b/packages/ai/test/github-copilot-anthropic.test.ts new file mode 100644 index 0000000000000000000000000000000000000000..e2894037fadeca020ecff0f38f1022c51e72fd35 --- /dev/null +++ b/packages/ai/test/github-copilot-anthropic.test.ts @@ -0,0 +1,125 @@ +import { describe, expect, it, vi } from "vitest"; +import { stream as streamAnthropic } from "../src/api/anthropic-messages.ts"; +import { getModel } from "../src/compat.ts"; +import { getSupportedThinkingLevels } from "../src/models.ts"; +import type { Context } from "../src/types.ts"; + +const mockState = vi.hoisted(() => ({ + constructorOpts: undefined as Record | undefined, + createParams: undefined as Record | undefined, +})); + +vi.mock("@anthropic-ai/sdk", () => { + function createSseResponse(): Response { + const body = [ + `event: message_start\ndata: ${JSON.stringify({ + type: "message_start", + message: { + id: "msg_test", + usage: { input_tokens: 10, output_tokens: 0 }, + }, + })}\n`, + `event: message_delta\ndata: ${JSON.stringify({ + type: "message_delta", + delta: { stop_reason: "end_turn" }, + usage: { output_tokens: 5 }, + })}\n`, + ].join("\n"); + + return new Response(body, { + status: 200, + headers: { "content-type": "text/event-stream" }, + }); + } + + class FakeAnthropic { + constructor(opts: Record) { + mockState.constructorOpts = opts; + } + beta = { + messages: { + create: (params: Record) => { + mockState.createParams = params; + return { + asResponse: async () => createSseResponse(), + }; + }, + }, + }; + } + + return { default: FakeAnthropic }; +}); + +describe("Copilot Claude via Anthropic Messages", () => { + const context: Context = { + systemPrompt: "You are a helpful assistant.", + messages: [{ role: "user", content: "Hello", timestamp: Date.now() }], + }; + + it("applies Copilot-specific adaptive thinking effort overrides", () => { + const opus47 = getModel("github-copilot", "claude-opus-4.7"); + expect(opus47.thinkingLevelMap).toMatchObject({ minimal: "low", xhigh: "xhigh", max: "max" }); + expect(getSupportedThinkingLevels(opus47)).toContain("xhigh"); + expect(getSupportedThinkingLevels(opus47)).toContain("max"); + + const opus5 = getModel("github-copilot", "claude-opus-5"); + expect(opus5.api).toBe("anthropic-messages"); + expect(opus5.contextWindow).toBe(1000000); + expect(opus5.thinkingLevelMap).toMatchObject({ minimal: "low", xhigh: "xhigh", max: "max" }); + expect(getSupportedThinkingLevels(opus5)).toContain("xhigh"); + expect(getSupportedThinkingLevels(opus5)).toContain("max"); + + const sonnet46 = getModel("github-copilot", "claude-sonnet-4.6"); + expect(sonnet46.thinkingLevelMap).toMatchObject({ minimal: "low", max: "max" }); + expect(getSupportedThinkingLevels(sonnet46)).toContain("max"); + expect(getSupportedThinkingLevels(sonnet46)).not.toContain("xhigh"); + }); + + it("uses Bearer auth, Copilot headers, and valid Anthropic Messages payload", async () => { + const model = getModel("github-copilot", "claude-sonnet-4.6"); + expect(model.api).toBe("anthropic-messages"); + + const s = streamAnthropic(model, context, { apiKey: "tid_copilot_session_test_token" }); + for await (const event of s) { + if (event.type === "error") break; + } + + const opts = mockState.constructorOpts!; + expect(opts).toBeDefined(); + + // Auth: apiKey null, authToken for Bearer + expect(opts.apiKey).toBeNull(); + expect(opts.authToken).toBe("tid_copilot_session_test_token"); + const headers = opts.defaultHeaders as Record; + + // Copilot static headers from model.headers + expect(headers["User-Agent"]).toContain("GitHubCopilotChat"); + expect(headers["Copilot-Integration-Id"]).toBe("vscode-chat"); + + // Dynamic headers + expect(headers["X-Initiator"]).toBe("user"); + expect(headers["Openai-Intent"]).toBe("conversation-edits"); + + // Payload is valid Anthropic Messages format + const params = mockState.createParams!; + expect(params.betas ?? []).not.toContain("fine-grained-tool-streaming-2025-05-14"); + expect(params.model).toBe("claude-sonnet-4.6"); + expect(params.stream).toBe(true); + expect(params.max_tokens).toBe(model.maxTokens); + expect(Array.isArray(params.messages)).toBe(true); + }); + + it("omits interleaved-thinking beta for adaptive-thinking models", async () => { + const model = getModel("github-copilot", "claude-sonnet-4.6"); + const s = streamAnthropic(model, context, { + apiKey: "tid_copilot_session_test_token", + interleavedThinking: true, + }); + for await (const event of s) { + if (event.type === "error") break; + } + + expect(mockState.createParams?.betas ?? []).not.toContain("interleaved-thinking-2025-05-14"); + }); +}); diff --git a/packages/ai/test/google-shared-convert-tools.test.ts b/packages/ai/test/google-shared-convert-tools.test.ts new file mode 100644 index 0000000000000000000000000000000000000000..82876e5fe53a4f33e9eb38743b0a05dd49fe31bd --- /dev/null +++ b/packages/ai/test/google-shared-convert-tools.test.ts @@ -0,0 +1,203 @@ +import { describe, expect, it } from "vitest"; +import { + convertTools, + resolveGoogleFunctionCallingMode, + supportsGoogleStrictToolSampling, +} from "../src/api/google-shared.ts"; +import type { Tool } from "../src/types.ts"; + +function makeTool(parameters: Record): Tool { + return { + name: "test_tool", + description: "A test tool", + parameters: parameters as Tool["parameters"], + }; +} + +describe("google-shared convertTools", () => { + it("strips JSON Schema meta keys from parameters when useParameters=true", () => { + const tools = [ + makeTool({ + $schema: "http://json-schema.org/draft-07/schema#", + $id: "urn:bash-tool", + $comment: "A bash tool for demonstration", + $defs: { + commandDef: { type: "string" }, + }, + definitions: { + legacyDef: { type: "number" }, + }, + type: "object", + properties: { + command: { type: "string" }, + }, + required: ["command"], + }), + ]; + + const result = convertTools(tools, true); + const decl = result?.[0]?.functionDeclarations?.[0]; + + expect(decl).toBeDefined(); + expect(decl?.parameters).toEqual({ + type: "object", + properties: { + command: { type: "string" }, + }, + required: ["command"], + }); + expect(decl?.parameters).not.toHaveProperty("$schema"); + expect(decl?.parameters).not.toHaveProperty("$id"); + expect(decl?.parameters).not.toHaveProperty("$comment"); + expect(decl?.parameters).not.toHaveProperty("$defs"); + expect(decl?.parameters).not.toHaveProperty("definitions"); + }); + + it("recursively strips nested JSON Schema meta keys", () => { + const tools = [ + makeTool({ + $schema: "http://json-schema.org/draft-07/schema#", + type: "object", + properties: { + deep: { + $schema: "http://json-schema.org/draft-07/schema#", + $id: "urn:nested", + type: "string", + }, + }, + }), + ]; + + const result = convertTools(tools, true); + const decl = result?.[0]?.functionDeclarations?.[0]; + + expect(decl).toBeDefined(); + expect(decl?.parameters).toEqual({ + type: "object", + properties: { + deep: { + type: "string", + }, + }, + }); + }); + + it("preserves $ref while stripping meta keys", () => { + const tools = [ + makeTool({ + $schema: "http://json-schema.org/draft-07/schema#", + type: "object", + properties: { + refProp: { + $ref: "#/$defs/someDef", + type: "string", + }, + }, + }), + ]; + + const result = convertTools(tools, true); + const decl = result?.[0]?.functionDeclarations?.[0]; + + expect(decl).toBeDefined(); + expect(decl?.parameters).toEqual({ + type: "object", + properties: { + refProp: { + $ref: "#/$defs/someDef", + type: "string", + }, + }, + }); + }); + + it("does not mutate the original Tool.parameters object", () => { + const originalParameters = { + $schema: "http://json-schema.org/draft-07/schema#", + type: "object", + properties: { + command: { type: "string" }, + }, + required: ["command"], + }; + const tools = [makeTool(originalParameters)]; + + convertTools(tools, true); + + expect(originalParameters).toEqual({ + $schema: "http://json-schema.org/draft-07/schema#", + type: "object", + properties: { + command: { type: "string" }, + }, + required: ["command"], + }); + }); + + it("preserves $schema in parametersJsonSchema when useParameters=false", () => { + const tools = [ + makeTool({ + $schema: "http://json-schema.org/draft-07/schema#", + type: "object", + properties: { + command: { type: "string" }, + }, + required: ["command"], + }), + ]; + + const result = convertTools(tools, false); + const decl = result?.[0]?.functionDeclarations?.[0]; + + expect(decl).toBeDefined(); + expect(decl?.parametersJsonSchema).toEqual({ + $schema: "http://json-schema.org/draft-07/schema#", + type: "object", + properties: { + command: { type: "string" }, + }, + required: ["command"], + }); + }); + + it("handles tools without $schema gracefully", () => { + const tools = [ + makeTool({ + type: "object", + properties: { + path: { type: "string" }, + }, + required: ["path"], + }), + ]; + + const result = convertTools(tools, true); + const decl = result?.[0]?.functionDeclarations?.[0]; + + expect(decl).toBeDefined(); + expect(decl?.parameters).toEqual({ + type: "object", + properties: { + path: { type: "string" }, + }, + required: ["path"], + }); + }); + + it("uses validated function calling for strict tools on Gemini 3", () => { + const tool = makeTool({ type: "object", properties: {} }); + tool.constrainedSampling = { type: "json_schema", strict: "require" }; + + expect(supportsGoogleStrictToolSampling("gemini-3.1-pro-preview")).toBe(true); + expect(supportsGoogleStrictToolSampling("gemini-2.5-pro")).toBe(false); + expect(resolveGoogleFunctionCallingMode([tool], undefined, true)).toBe("VALIDATED"); + expect(() => resolveGoogleFunctionCallingMode([tool], undefined, false)).toThrow( + 'Tool "test_tool" requires JSON-schema constrained sampling', + ); + }); + + it("returns undefined for empty tool list", () => { + expect(convertTools([])).toBeUndefined(); + expect(convertTools([], true)).toBeUndefined(); + }); +}); diff --git a/packages/ai/test/google-shared-image-tool-result-routing.test.ts b/packages/ai/test/google-shared-image-tool-result-routing.test.ts new file mode 100644 index 0000000000000000000000000000000000000000..8ba5660cb5f28c652868e7772c79a001524579cf --- /dev/null +++ b/packages/ai/test/google-shared-image-tool-result-routing.test.ts @@ -0,0 +1,102 @@ +import { describe, expect, it } from "vitest"; +import { convertMessages } from "../src/api/google-shared.ts"; +import type { Context, Model } from "../src/types.ts"; + +function makeModel( + api: TApi, + provider: Model["provider"], + id: string, +): Model { + return { + id, + name: id, + api, + provider, + baseUrl: "https://example.com", + reasoning: true, + input: ["text", "image"], + cost: { input: 0, output: 0, cacheRead: 0, cacheWrite: 0 }, + contextWindow: 128000, + maxTokens: 8192, + }; +} + +function makeContext(model: { api: string; provider: string; id: string }): Context { + const now = Date.now(); + return { + messages: [ + { role: "user", content: "read the files", timestamp: now }, + { + role: "assistant", + content: [ + { type: "toolCall", id: "call_a", name: "read", arguments: { path: "a.txt" } }, + { type: "toolCall", id: "call_img", name: "read", arguments: { path: "image.png" } }, + { type: "toolCall", id: "call_b", name: "read", arguments: { path: "b.txt" } }, + ], + api: model.api, + provider: model.provider, + model: model.id, + usage: { + input: 0, + output: 0, + cacheRead: 0, + cacheWrite: 0, + totalTokens: 0, + cost: { input: 0, output: 0, cacheRead: 0, cacheWrite: 0, total: 0 }, + }, + stopReason: "toolUse", + timestamp: now, + }, + { + role: "toolResult", + toolCallId: "call_a", + toolName: "read", + content: [{ type: "text", text: "alpha text" }], + isError: false, + timestamp: now, + }, + { + role: "toolResult", + toolCallId: "call_img", + toolName: "read", + content: [{ type: "image", data: "abc", mimeType: "image/png" }], + isError: false, + timestamp: now, + }, + { + role: "toolResult", + toolCallId: "call_b", + toolName: "read", + content: [{ type: "text", text: "beta text" }], + isError: false, + timestamp: now, + }, + ], + }; +} + +describe("google-shared image tool result routing", () => { + it("keeps separate synthetic image turn for Gemini 2.x Google API models", () => { + const model = makeModel("google-generative-ai", "google", "gemini-2.5-flash"); + const contents = convertMessages(model, makeContext(model)); + + expect(contents).toHaveLength(5); + expect(contents[2].parts?.every((part) => part.functionResponse)).toBe(true); + expect(contents[3].parts?.[0]?.text).toBe("Tool result image:"); + expect(contents[3].parts?.[1]?.inlineData).toBeTruthy(); + expect(contents[4].parts?.[0]?.functionResponse).toBeTruthy(); + }); + + it("nests image tool results for Gemini 3 Google API models", () => { + const model = makeModel("google-generative-ai", "google", "gemini-3-pro-preview"); + const contents = convertMessages(model, makeContext(model)); + + expect(contents).toHaveLength(3); + const toolResultTurn = contents[2]; + expect(toolResultTurn.parts).toHaveLength(3); + const imageResponse = toolResultTurn.parts?.[1]?.functionResponse; + expect(imageResponse).toBeTruthy(); + expect(imageResponse?.parts).toHaveLength(1); + expect(imageResponse?.parts?.[0]?.inlineData).toBeTruthy(); + }); +}); diff --git a/packages/ai/test/google-thinking-disable.test.ts b/packages/ai/test/google-thinking-disable.test.ts new file mode 100644 index 0000000000000000000000000000000000000000..3f6e36c7168e21d55f0b6c8ba5e8498d8eb26917 --- /dev/null +++ b/packages/ai/test/google-thinking-disable.test.ts @@ -0,0 +1,160 @@ +import { describe, expect, it } from "vitest"; +import { getModel, streamSimple } from "../src/compat.ts"; +import type { Api, Context, Model, SimpleStreamOptions } from "../src/types.ts"; + +type SimpleOptionsWithExtras = SimpleStreamOptions & Record; + +interface RunResult { + thinkingEventCount: number; + thinkingCharCount: number; + text: string; + outputTokens: number; + contentTypes: string[]; +} + +interface DisableExpectations { + requestOptions?: SimpleOptionsWithExtras; + minPongs?: number; + maxOutputTokens?: number; +} + +function makeContext(): Context { + return { + systemPrompt: "You are a precise assistant. Follow the requested output format exactly.", + messages: [ + { + role: "user", + content: + "Before replying, carefully solve 36863 * 5279 internally. Then reply with the word pong repeated exactly 40 times, separated by single spaces. Do not add any other text.", + timestamp: Date.now(), + }, + ], + }; +} + +function countPongs(text: string): number { + return text.match(/\bpong\b/gi)?.length ?? 0; +} + +async function runWithoutReasoning( + model: Model, + options: SimpleOptionsWithExtras = {}, +): Promise { + const s = streamSimple(model, makeContext(), { + maxTokens: 160, + temperature: 0, + ...options, + }); + + let thinkingEventCount = 0; + let thinkingCharCount = 0; + + for await (const event of s) { + if (event.type === "thinking_start" || event.type === "thinking_end") { + thinkingEventCount += 1; + } + if (event.type === "thinking_delta") { + thinkingEventCount += 1; + thinkingCharCount += event.delta.length; + } + } + + const response = await s.result(); + expect(response.stopReason, response.errorMessage).toBe("stop"); + + const text = response.content + .filter((block) => block.type === "text") + .map((block) => block.text) + .join("") + .trim(); + + return { + thinkingEventCount, + thinkingCharCount, + text, + outputTokens: response.usage.output, + contentTypes: response.content.map((block) => block.type), + }; +} + +async function expectThinkingDisabledE2E(model: Model, expectations: DisableExpectations = {}) { + const result = await runWithoutReasoning(model, expectations.requestOptions); + + expect(result.thinkingEventCount).toBe(0); + expect(result.thinkingCharCount).toBe(0); + expect(result.contentTypes).not.toContain("thinking"); + expect(countPongs(result.text)).toBeGreaterThanOrEqual(expectations.minPongs ?? 35); + if (expectations.maxOutputTokens !== undefined) { + expect(result.outputTokens).toBeLessThan(expectations.maxOutputTokens); + } +} + +describe.skipIf(!process.env.ANTHROPIC_API_KEY)("Anthropic thinking disable E2E", () => { + it("disables thinking for budget-based reasoning models", { retry: 2, timeout: 30000 }, async () => { + await expectThinkingDisabledE2E(getModel("anthropic", "claude-sonnet-4-5"), { + requestOptions: { maxTokens: 320, temperature: 0 }, + }); + }); + + it("disables thinking for adaptive reasoning models", { retry: 2, timeout: 30000 }, async () => { + await expectThinkingDisabledE2E(getModel("anthropic", "claude-sonnet-4-6"), { + requestOptions: { maxTokens: 320, temperature: 0 }, + }); + }); +}); + +describe.skipIf(!process.env.GEMINI_API_KEY)("Google thinking disable E2E", () => { + it("disables thinking for Gemini 2.5", { retry: 2, timeout: 30000 }, async () => { + await expectThinkingDisabledE2E(getModel("google", "gemini-2.5-flash")); + }); + + it("disables thinking for Gemini 3.x", { retry: 2, timeout: 30000 }, async () => { + await expectThinkingDisabledE2E(getModel("google", "gemini-3-flash-preview")); + }); + + it("does not error when thinking is off for Gemini 3.1 Pro", { retry: 2, timeout: 30000 }, async () => { + await expectThinkingDisabledE2E(getModel("google", "gemini-3.1-pro-preview"), { + requestOptions: { maxTokens: 512 }, + minPongs: 20, + }); + }); +}); + +describe("Google Vertex thinking disable E2E", () => { + const vertexProject = process.env.GOOGLE_CLOUD_PROJECT || process.env.GCLOUD_PROJECT; + const vertexLocation = process.env.GOOGLE_CLOUD_LOCATION; + const vertexApiKey = process.env.GOOGLE_CLOUD_API_KEY; + const vertexOptions = vertexApiKey + ? ({ apiKey: vertexApiKey } satisfies SimpleOptionsWithExtras) + : vertexProject && vertexLocation + ? ({ project: vertexProject, location: vertexLocation } satisfies SimpleOptionsWithExtras) + : undefined; + + it.skipIf(!vertexOptions)("disables thinking for Gemini 2.5", { retry: 2, timeout: 30000 }, async () => { + await expectThinkingDisabledE2E(getModel("google-vertex", "gemini-2.5-flash"), { + requestOptions: vertexOptions, + }); + }); + + it.skipIf(!vertexOptions)("disables thinking for Gemini 3.x", { retry: 2, timeout: 30000 }, async () => { + await expectThinkingDisabledE2E(getModel("google-vertex", "gemini-3-flash-preview"), { + requestOptions: vertexOptions, + }); + }); +}); + +describe.skipIf(!process.env.OPENAI_API_KEY)("OpenAI thinking disable E2E", () => { + it("disables thinking for Responses reasoning models", { retry: 2, timeout: 30000 }, async () => { + await expectThinkingDisabledE2E(getModel("openai", "gpt-5.4-mini"), { + requestOptions: { temperature: undefined }, + }); + }); +}); + +describe.skipIf(!process.env.OPENROUTER_API_KEY)("OpenRouter thinking disable E2E", () => { + it("disables thinking for Qwen 3.5 reasoning models", { retry: 2, timeout: 30000 }, async () => { + await expectThinkingDisabledE2E(getModel("openrouter", "qwen/qwen3.5-plus-02-15"), { + maxOutputTokens: 100, + }); + }); +}); diff --git a/packages/ai/test/google-thinking-level-map.test.ts b/packages/ai/test/google-thinking-level-map.test.ts new file mode 100644 index 0000000000000000000000000000000000000000..957c9b14b22d09f088d2338527de2c74050bb091 --- /dev/null +++ b/packages/ai/test/google-thinking-level-map.test.ts @@ -0,0 +1,170 @@ +import type { GenerateContentParameters } from "@google/genai"; +import { describe, expect, it } from "vitest"; +import { streamSimple as streamSimpleGoogle } from "../src/api/google-generative-ai.ts"; +import { resolveGoogleThinkingLevel } from "../src/api/google-shared.ts"; +import { streamSimple as streamSimpleVertex } from "../src/api/google-vertex.ts"; +import type { + Context, + Model, + ModelThinkingLevel, + ThinkingBudgets, + ThinkingLevel, + ThinkingLevelMap, +} from "../src/types.ts"; + +const context: Context = { + messages: [{ role: "user", content: "Hello", timestamp: 0 }], +}; + +function googleModel(id: string, thinkingLevelMap: ThinkingLevelMap): Model<"google-generative-ai"> { + return { + id, + name: id, + api: "google-generative-ai", + provider: "test-google", + baseUrl: "https://example.invalid/v1beta", + reasoning: true, + thinkingLevelMap, + input: ["text"], + cost: { input: 0, output: 0, cacheRead: 0, cacheWrite: 0 }, + contextWindow: 128000, + maxTokens: 4096, + }; +} + +function vertexModel(id: string, thinkingLevelMap: ThinkingLevelMap): Model<"google-vertex"> { + return { + id, + name: id, + api: "google-vertex", + provider: "test-vertex", + baseUrl: "https://example.invalid/v1", + reasoning: true, + thinkingLevelMap, + input: ["text"], + cost: { input: 0, output: 0, cacheRead: 0, cacheWrite: 0 }, + contextWindow: 128000, + maxTokens: 4096, + }; +} + +async function captureGooglePayload( + model: Model<"google-generative-ai">, + reasoning: ThinkingLevel, + thinkingBudgets?: ThinkingBudgets, +): Promise { + let payload: GenerateContentParameters | undefined; + const result = await streamSimpleGoogle(model, context, { + apiKey: "test", + reasoning, + thinkingBudgets, + onPayload: (request) => { + payload = request as GenerateContentParameters; + throw new Error("payload captured"); + }, + }).result(); + + expect(result.errorMessage).toContain("payload captured"); + if (!payload) throw new Error("Google payload was not captured"); + return payload; +} + +async function captureVertexPayload( + model: Model<"google-vertex">, + reasoning: ThinkingLevel, + thinkingBudgets?: ThinkingBudgets, +): Promise { + let payload: GenerateContentParameters | undefined; + const result = await streamSimpleVertex(model, context, { + apiKey: "test", + reasoning, + thinkingBudgets, + onPayload: (request) => { + payload = request as GenerateContentParameters; + throw new Error("payload captured"); + }, + }).result(); + + expect(result.errorMessage).toContain("payload captured"); + if (!payload) throw new Error("Vertex payload was not captured"); + return payload; +} + +describe("Google thinking level maps", () => { + it("exhaustively resolves supported logical levels and mapping values", () => { + const defaultExpectations = { + off: "high", + minimal: "minimal", + low: "low", + medium: "medium", + high: "high", + } as const satisfies Partial>; + for (const [level, expected] of Object.entries(defaultExpectations)) { + expect(resolveGoogleThinkingLevel(googleModel("gemini-3.7-flash", {}), level as ModelThinkingLevel)).toBe( + expected, + ); + } + + const mappedExpectations = { + minimal: "minimal", + low: "low", + medium: "medium", + high: "high", + MINIMAL: "minimal", + LOW: "low", + MEDIUM: "medium", + HIGH: "high", + } as const; + for (const [mapped, expected] of Object.entries(mappedExpectations)) { + const model = googleModel("gemini-3.7-flash", { high: mapped, xhigh: mapped, max: mapped }); + expect(resolveGoogleThinkingLevel(model, "high")).toBe(expected); + expect(resolveGoogleThinkingLevel(model, "xhigh")).toBe(expected); + expect(resolveGoogleThinkingLevel(model, "max")).toBe(expected); + } + + const invalidModel = googleModel("gemini-3.7-flash", { xhigh: "extreme" }); + expect(() => resolveGoogleThinkingLevel(invalidModel, "xhigh")).toThrow( + "Unsupported Google thinking level mapping for test-google/gemini-3.7-flash: xhigh -> extreme", + ); + expect(() => resolveGoogleThinkingLevel(googleModel("gemini-3.7-flash", {}), "max")).toThrow( + "Unsupported Google thinking level mapping for test-google/gemini-3.7-flash: max -> undefined", + ); + }); + + it.each(["xhigh", "max"] as const)("maps Google Generative AI %s to a supported level", async (reasoning) => { + const payload = await captureGooglePayload( + googleModel("gemini-3.7-flash", { xhigh: "high", max: "high" }), + reasoning, + ); + + expect(payload).toMatchObject({ config: { thinkingConfig: { includeThoughts: true, thinkingLevel: "HIGH" } } }); + }); + + it("honors uppercase provider values for standard Google Generative AI levels", async () => { + const payload = await captureGooglePayload(googleModel("gemini-3.7-flash", { high: "LOW" }), "high"); + + expect(payload).toMatchObject({ config: { thinkingConfig: { thinkingLevel: "LOW" } } }); + }); + + it("uses mapped Google Generative AI levels for token budgets", async () => { + const payload = await captureGooglePayload(googleModel("gemini-2.5-flash", { xhigh: "high" }), "xhigh", { + high: 1234, + }); + + expect(payload).toMatchObject({ config: { thinkingConfig: { thinkingBudget: 1234 } } }); + }); + + it("maps Google Vertex extended levels", async () => { + const payload = await captureVertexPayload(vertexModel("gemini-3.7-flash", { xhigh: "high" }), "xhigh"); + + expect(payload).toMatchObject({ config: { thinkingConfig: { includeThoughts: true, thinkingLevel: "HIGH" } } }); + }); + + it("uses mapped Google Vertex levels for token budgets", async () => { + const payload = await captureVertexPayload(vertexModel("gemini-2.5-flash", { max: "high" }), "max", { + high: 4321, + }); + + expect(payload).toMatchObject({ config: { thinkingConfig: { thinkingBudget: 4321 } } }); + }); +}); diff --git a/packages/ai/test/google-vertex-api-key-resolution.test.ts b/packages/ai/test/google-vertex-api-key-resolution.test.ts new file mode 100644 index 0000000000000000000000000000000000000000..3937b3feb24bfe1fab7841fab4e1493d3ef7554c --- /dev/null +++ b/packages/ai/test/google-vertex-api-key-resolution.test.ts @@ -0,0 +1,242 @@ +import { arch, platform, release } from "node:os"; +import { afterEach, beforeEach, describe, expect, it, vi } from "vitest"; + +const googleGenAiMock = vi.hoisted(() => ({ + constructorCalls: [] as Array>, +})); + +vi.mock("@google/genai", () => { + class GoogleGenAI { + models = { + generateContentStream: async function* () { + yield { + responseId: "vertex-response-id", + candidates: [ + { + content: { parts: [{ text: "ok" }] }, + finishReason: "STOP", + }, + ], + usageMetadata: { + promptTokenCount: 1, + candidatesTokenCount: 1, + totalTokenCount: 2, + }, + }; + }, + }; + + constructor(config: Record) { + googleGenAiMock.constructorCalls.push(config); + } + } + + return { + GoogleGenAI, + ResourceScope: { + COLLECTION: "COLLECTION", + }, + ThinkingLevel: { + THINKING_LEVEL_UNSPECIFIED: "THINKING_LEVEL_UNSPECIFIED", + MINIMAL: "MINIMAL", + LOW: "LOW", + MEDIUM: "MEDIUM", + HIGH: "HIGH", + }, + }; +}); + +import { stream as streamGoogleVertex } from "../src/api/google-vertex.ts"; +import { getModel } from "../src/compat.ts"; +import type { Context, Model } from "../src/types.ts"; + +const PI_USER_AGENT = `pi (${platform()} ${release()}; ${arch()})`; +const model = getModel("google-vertex", "gemini-3-flash-preview"); +const context: Context = { + messages: [{ role: "user", content: "hello", timestamp: Date.now() }], +}; + +const originalGoogleCloudApiKey = process.env.GOOGLE_CLOUD_API_KEY; + +beforeEach(() => { + googleGenAiMock.constructorCalls.length = 0; + delete process.env.GOOGLE_CLOUD_API_KEY; +}); + +afterEach(() => { + if (originalGoogleCloudApiKey === undefined) { + delete process.env.GOOGLE_CLOUD_API_KEY; + } else { + process.env.GOOGLE_CLOUD_API_KEY = originalGoogleCloudApiKey; + } +}); + +describe("google-vertex api key resolution", () => { + it("falls back to ADC when options.apiKey is a placeholder marker", async () => { + const stream = streamGoogleVertex(model, context, { + apiKey: "", + project: "test-project", + location: "us-central1", + }); + + await stream.result(); + + expect(googleGenAiMock.constructorCalls).toHaveLength(1); + expect(googleGenAiMock.constructorCalls[0]).toMatchObject({ + vertexai: true, + project: "test-project", + location: "us-central1", + apiVersion: "v1", + }); + expect(googleGenAiMock.constructorCalls[0]).not.toHaveProperty("apiKey"); + }); + + it("falls back to ADC when options.apiKey is the gcp-vertex-credentials marker", async () => { + const stream = streamGoogleVertex(model, context, { + apiKey: "gcp-vertex-credentials", + project: "test-project", + location: "us-central1", + }); + + await stream.result(); + + expect(googleGenAiMock.constructorCalls).toHaveLength(1); + expect(googleGenAiMock.constructorCalls[0]).toMatchObject({ + vertexai: true, + project: "test-project", + location: "us-central1", + apiVersion: "v1", + }); + expect(googleGenAiMock.constructorCalls[0]).not.toHaveProperty("apiKey"); + }); + + it("falls back to ADC when GOOGLE_CLOUD_API_KEY is a placeholder marker", async () => { + process.env.GOOGLE_CLOUD_API_KEY = ""; + + const stream = streamGoogleVertex(model, context, { + project: "test-project", + location: "us-central1", + }); + + await stream.result(); + + expect(googleGenAiMock.constructorCalls).toHaveLength(1); + expect(googleGenAiMock.constructorCalls[0]).toMatchObject({ + vertexai: true, + project: "test-project", + location: "us-central1", + apiVersion: "v1", + }); + expect(googleGenAiMock.constructorCalls[0]).not.toHaveProperty("apiKey"); + }); + + it("still uses the API key client for real API keys", async () => { + const stream = streamGoogleVertex(model, context, { + apiKey: "AIzaSyExampleRealisticLookingApiKey123456", + }); + + await stream.result(); + + expect(googleGenAiMock.constructorCalls).toHaveLength(1); + expect(googleGenAiMock.constructorCalls[0]).toMatchObject({ + vertexai: true, + apiKey: "AIzaSyExampleRealisticLookingApiKey123456", + apiVersion: "v1", + }); + expect(googleGenAiMock.constructorCalls[0]).not.toHaveProperty("project"); + expect(googleGenAiMock.constructorCalls[0]).not.toHaveProperty("location"); + }); + + it("does not forward generated Vertex base URL placeholders", async () => { + const stream = streamGoogleVertex(model, context, { + project: "test-project", + location: "us-central1", + }); + + await stream.result(); + + expect(googleGenAiMock.constructorCalls).toHaveLength(1); + expect(googleGenAiMock.constructorCalls[0]?.httpOptions).toEqual({ + headers: { "User-Agent": PI_USER_AGENT }, + }); + }); + + it("lets explicit headers override the default User-Agent", async () => { + const stream = streamGoogleVertex(model, context, { + project: "test-project", + location: "us-central1", + headers: { "User-Agent": "custom-agent" }, + }); + + await stream.result(); + + expect(googleGenAiMock.constructorCalls).toHaveLength(1); + expect(googleGenAiMock.constructorCalls[0]?.httpOptions).toEqual({ + headers: { "User-Agent": "custom-agent" }, + }); + }); + + it("forwards custom baseUrl to the ADC client", async () => { + const customModel: Model<"google-vertex"> = { ...model, baseUrl: "https://proxy.example.com" }; + const stream = streamGoogleVertex(customModel, context, { + project: "test-project", + location: "us-central1", + }); + + await stream.result(); + + expect(googleGenAiMock.constructorCalls).toHaveLength(1); + expect(googleGenAiMock.constructorCalls[0]).toMatchObject({ + vertexai: true, + project: "test-project", + location: "us-central1", + apiVersion: "v1", + httpOptions: { + baseUrl: "https://proxy.example.com", + baseUrlResourceScope: "COLLECTION", + }, + }); + }); + + it("forwards custom baseUrl to the API key client", async () => { + const customModel: Model<"google-vertex"> = { ...model, baseUrl: "https://proxy.example.com" }; + const stream = streamGoogleVertex(customModel, context, { + apiKey: "AIzaSyExampleRealisticLookingApiKey123456", + }); + + await stream.result(); + + expect(googleGenAiMock.constructorCalls).toHaveLength(1); + expect(googleGenAiMock.constructorCalls[0]).toMatchObject({ + vertexai: true, + apiKey: "AIzaSyExampleRealisticLookingApiKey123456", + apiVersion: "v1", + httpOptions: { + baseUrl: "https://proxy.example.com", + baseUrlResourceScope: "COLLECTION", + }, + }); + }); + + it("does not append apiVersion when custom baseUrl already includes one", async () => { + const customModel: Model<"google-vertex"> = { + ...model, + baseUrl: "https://proxy.example.com/v1/projects/test-project/locations/global", + }; + const stream = streamGoogleVertex(customModel, context, { + project: "test-project", + location: "us-central1", + }); + + await stream.result(); + + expect(googleGenAiMock.constructorCalls).toHaveLength(1); + expect(googleGenAiMock.constructorCalls[0]).toMatchObject({ + httpOptions: { + baseUrl: "https://proxy.example.com/v1/projects/test-project/locations/global", + baseUrlResourceScope: "COLLECTION", + apiVersion: "", + }, + }); + }); +}); diff --git a/packages/ai/test/image-model-data.test.ts b/packages/ai/test/image-model-data.test.ts new file mode 100644 index 0000000000000000000000000000000000000000..21d4680509e773613a3ee648a75b424cdbebce48 --- /dev/null +++ b/packages/ai/test/image-model-data.test.ts @@ -0,0 +1,47 @@ +import { describe, expect, it } from "vitest"; +import { parseOpenRouterImageModels } from "../scripts/generate-image-models.ts"; + +const validImageModel = { + id: "example/image-model", + name: "Example Image Model", + architecture: { + input_modalities: ["text", "image"], + output_modalities: ["image"], + }, + pricing: { + prompt: "0.000001", + completion: "0.000002", + }, +}; + +describe("OpenRouter image model parsing", () => { + it.each([{}, { data: [] }, { data: "invalid" }])("rejects a missing or empty strict catalog", (payload) => { + expect(() => parseOpenRouterImageModels(payload, true)).toThrow("missing or empty image model list"); + }); + + it("rejects a strict catalog with no usable image models", () => { + expect(() => + parseOpenRouterImageModels( + { + data: [ + { + ...validImageModel, + architecture: { input_modalities: ["text"], output_modalities: ["text"] }, + }, + ], + }, + true, + ), + ).toThrow("no usable image models"); + }); + + it("parses a non-empty image model catalog", () => { + expect(parseOpenRouterImageModels({ data: [validImageModel] }, true)).toEqual([ + expect.objectContaining({ + id: "example/image-model", + input: ["text", "image"], + output: ["image"], + }), + ]); + }); +}); diff --git a/packages/ai/test/images-models.test.ts b/packages/ai/test/images-models.test.ts new file mode 100644 index 0000000000000000000000000000000000000000..a9dd04077fde4da0e0b76e178e06ad3c6e38135b --- /dev/null +++ b/packages/ai/test/images-models.test.ts @@ -0,0 +1,209 @@ +import { describe, expect, it } from "vitest"; +import type { AuthContext } from "../src/auth/types.ts"; +import { createImagesModels, createImagesProvider, type ImagesProvider } from "../src/images-models.ts"; +import { builtinImagesModels } from "../src/providers/all.ts"; +import type { AssistantImages, ImagesApi, ImagesContext, ImagesModel, ImagesOptions } from "../src/types.ts"; + +function fakeAuthContext(env: Record): AuthContext { + return { + env: async (name) => env[name], + fileExists: async () => false, + }; +} + +function testImageModel(provider: string, id: string): ImagesModel { + return { + id, + name: id, + api: "test-images", + provider, + baseUrl: "https://example.test/v1", + input: ["text"], + output: ["image"], + cost: { input: 0, output: 0, cacheRead: 0, cacheWrite: 0 }, + }; +} + +function okResult(model: ImagesModel): AssistantImages { + return { + api: model.api, + provider: model.provider, + model: model.id, + output: [{ type: "image", data: "aGk=", mimeType: "image/png" }], + stopReason: "stop", + timestamp: Date.now(), + }; +} + +interface GenerateCall { + model: ImagesModel; + options: ImagesOptions | undefined; +} + +function testProvider(input: { + id: string; + models?: ImagesModel[]; + envVar?: string; + calls?: GenerateCall[]; +}): ImagesProvider { + return createImagesProvider({ + id: input.id, + auth: { + apiKey: { + name: "Test key", + resolve: async ({ ctx, credential }) => { + if (!input.envVar) return { auth: {} }; + const key = credential?.key ?? (await ctx.env(input.envVar)); + return key ? { auth: { apiKey: key }, source: credential ? "stored" : input.envVar } : undefined; + }, + }, + }, + models: input.models ?? [testImageModel(input.id, "model-a")], + api: { + generateImages: async (model, _context, options) => { + input.calls?.push({ model, options }); + return okResult(model); + }, + }, + }); +} + +const context: ImagesContext = { input: [{ type: "text", text: "a red circle" }] }; + +describe("ImagesModels", () => { + it("registers providers and reads models synchronously", () => { + const models = createImagesModels(); + models.setProvider(testProvider({ id: "p1", models: [testImageModel("p1", "m1"), testImageModel("p1", "m2")] })); + models.setProvider(testProvider({ id: "p2", models: [testImageModel("p2", "m3")] })); + + expect(models.getProviders().map((p) => p.id)).toEqual(["p1", "p2"]); + expect(models.getModels().map((m) => m.id)).toEqual(["m1", "m2", "m3"]); + expect(models.getModels("p1").map((m) => m.id)).toEqual(["m1", "m2"]); + expect(models.getModel("p2", "m3")?.id).toBe("m3"); + expect(models.getModel("p2", "missing")).toBeUndefined(); + + models.deleteProvider("p1"); + expect(models.getProvider("p1")).toBeUndefined(); + }); + + it("resolves auth through the provider and merges it into requests; explicit options win", async () => { + const calls: GenerateCall[] = []; + const models = createImagesModels({ authContext: fakeAuthContext({ TEST_KEY: "env-key" }) }); + models.setProvider(testProvider({ id: "p1", envVar: "TEST_KEY", calls })); + const model = models.getModel("p1", "model-a")!; + + expect((await models.getAuth(model))?.auth.apiKey).toBe("env-key"); + expect((await models.getAuth(model.provider))?.auth.apiKey).toBe("env-key"); + expect((await models.getAuth(model, { apiKey: "explicit-key" }))?.auth.apiKey).toBe("explicit-key"); + + const result = await models.generateImages(model, context); + expect(result.stopReason).toBe("stop"); + expect(calls[0].options?.apiKey).toBe("env-key"); + + await models.generateImages(model, context, { apiKey: "explicit" }); + expect(calls[1].options?.apiKey).toBe("explicit"); + }); + + it("merges provider-resolved env into image options", async () => { + const calls: GenerateCall[] = []; + const models = createImagesModels(); + models.setProvider( + createImagesProvider({ + id: "p1", + auth: { + apiKey: { + name: "Test key", + resolve: async () => ({ + auth: { apiKey: "provider-key" }, + env: { PROVIDER_ONLY: "provider", SHARED: "provider" }, + }), + }, + }, + models: [testImageModel("p1", "model-a")], + api: { + generateImages: async (model, _context, options) => { + calls.push({ model, options }); + return okResult(model); + }, + }, + }), + ); + const model = models.getModel("p1", "model-a")!; + + await models.generateImages(model, context, { + apiKey: "request-key", + env: { REQUEST_ONLY: "request", SHARED: "request" }, + }); + + expect(calls[0].options?.apiKey).toBe("request-key"); + expect(calls[0].options?.env).toEqual({ + PROVIDER_ONLY: "provider", + REQUEST_ONLY: "request", + SHARED: "request", + }); + }); + + it("returns an error result for unknown providers and unconfigured auth rejections", async () => { + const models = createImagesModels({ authContext: fakeAuthContext({}) }); + const ghost = await models.generateImages(testImageModel("ghost", "m"), context); + expect(ghost.stopReason).toBe("error"); + expect(ghost.errorMessage).toContain("Unknown provider: ghost"); + + // unconfigured (resolve -> undefined) still dispatches; provider decides what to do + const calls: GenerateCall[] = []; + models.setProvider(testProvider({ id: "p1", envVar: "MISSING", calls })); + const model = models.getModel("p1", "model-a")!; + expect(await models.getAuth(model)).toBeUndefined(); + await models.generateImages(model, context); + expect(calls[0].options?.apiKey).toBeUndefined(); + }); + + it("supports dynamic providers via refresh with in-flight dedupe", async () => { + let fetches = 0; + const provider = createImagesProvider({ + id: "dyn", + auth: { apiKey: { name: "Test", resolve: async () => ({ auth: {} }) } }, + models: [], + refreshModels: async () => { + fetches++; + await new Promise((resolve) => setTimeout(resolve, 5)); + return [testImageModel("dyn", "listed")]; + }, + api: { generateImages: async (model) => okResult(model) }, + }); + const models = createImagesModels(); + models.setProvider(provider); + + expect(models.getModels("dyn")).toEqual([]); + await Promise.all([models.refresh("dyn"), models.refresh("dyn")]); + expect(fetches).toBe(1); + expect(models.getModel("dyn", "listed")).toBeDefined(); + + // failures reject with ModelsError for a single provider + models.setProvider( + createImagesProvider({ + id: "flaky", + auth: { apiKey: { name: "Test", resolve: async () => ({ auth: {} }) } }, + models: [], + refreshModels: async () => { + throw new Error("fetch failed"); + }, + api: { generateImages: async (model) => okResult(model) }, + }), + ); + await expect(models.refresh("flaky")).rejects.toMatchObject({ code: "model_source" }); + await expect(models.refresh()).resolves.toBeUndefined(); + }); + + it("builtinImagesModels registers the openrouter provider with its catalog", async () => { + const models = builtinImagesModels({ authContext: fakeAuthContext({ OPENROUTER_API_KEY: "or-key" }) }); + const providers = models.getProviders(); + expect(providers.map((p) => p.id)).toEqual(["openrouter"]); + + const list = models.getModels("openrouter"); + expect(list.length).toBeGreaterThan(0); + expect(list.every((m) => m.api === "openrouter-images")).toBe(true); + + expect((await models.getAuth(list[0]))?.auth.apiKey).toBe("or-key"); + }); +}); diff --git a/packages/ai/test/images.test.ts b/packages/ai/test/images.test.ts new file mode 100644 index 0000000000000000000000000000000000000000..c45cc215ef4f5802f57542f6c3dc1bf906d33cf3 --- /dev/null +++ b/packages/ai/test/images.test.ts @@ -0,0 +1,90 @@ +import { readFileSync } from "node:fs"; +import { dirname, join } from "node:path"; +import { fileURLToPath } from "node:url"; +import { describe, expect, it } from "vitest"; +import { getImageModel } from "../src/image-models.ts"; +import { generateImages } from "../src/images.ts"; +import type { ImageContent, ImagesContext, ImagesModel, ProviderImagesOptions } from "../src/types.ts"; + +const __filename = fileURLToPath(import.meta.url); +const __dirname = dirname(__filename); + +type ImagesOptionsWithExtras = ProviderImagesOptions & Record; + +async function basicImageGeneration(model: ImagesModel, options?: ImagesOptionsWithExtras) { + const context: ImagesContext = { + input: [{ type: "text", text: "Generate a simple red circle on a plain white background. No text." }], + }; + + const response = await generateImages(model, context, options); + + expect(response.stopReason, `Error: ${response.errorMessage}`).toBe("stop"); + expect(response.errorMessage).toBeFalsy(); + expect(response.output.some((item) => item.type === "image")).toBe(true); + expect(response.timestamp).toBeGreaterThan(0); +} + +async function handleTextAndImageOutput( + model: ImagesModel, + options?: ImagesOptionsWithExtras, +) { + if (!model.output.includes("text")) { + console.log(`Skipping text+image output test - model ${model.id} doesn't support text output`); + return; + } + + const context: ImagesContext = { + input: [{ type: "text", text: "Generate a red circle and include a brief description of the image." }], + }; + + const response = await generateImages(model, context, options); + + expect(response.stopReason, `Error: ${response.errorMessage}`).toBe("stop"); + expect(response.output.some((item) => item.type === "image")).toBe(true); + expect(response.output.some((item) => item.type === "text" && item.text.trim().length > 0)).toBe(true); +} + +async function handleImageInput(model: ImagesModel, options?: ImagesOptionsWithExtras) { + if (!model.input.includes("image")) { + console.log(`Skipping image input test - model ${model.id} doesn't support image input`); + return; + } + + const imagePath = join(__dirname, "data", "red-circle.png"); + const imageBuffer = readFileSync(imagePath); + const imageContent: ImageContent = { + type: "image", + data: imageBuffer.toString("base64"), + mimeType: "image/png", + }; + + const context: ImagesContext = { + input: [{ type: "text", text: "Create a variation of this image with a blue background." }, imageContent], + }; + + const response = await generateImages(model, context, options); + + expect(response.stopReason, `Error: ${response.errorMessage}`).toBe("stop"); + expect(response.output.some((item) => item.type === "image")).toBe(true); +} + +describe("Images E2E Tests", () => { + describe.skipIf(!process.env.OPENROUTER_API_KEY)( + "OpenRouter Images Provider (google/gemini-2.5-flash-image)", + () => { + const model = getImageModel("openrouter", "google/gemini-2.5-flash-image"); + + it("should generate a basic image", { retry: 3 }, async () => { + await basicImageGeneration(model); + }); + + it("should handle text plus image output", { retry: 3 }, async () => { + await handleTextAndImageOutput(model); + }); + + it("should handle image input", { retry: 3 }, async () => { + await handleImageInput(model); + }); + }, + ); +}); diff --git a/packages/ai/test/lazy-module-load.test.ts b/packages/ai/test/lazy-module-load.test.ts new file mode 100644 index 0000000000000000000000000000000000000000..cb0875c9ee73e012206348a9b295b20cfc25a970 --- /dev/null +++ b/packages/ai/test/lazy-module-load.test.ts @@ -0,0 +1,113 @@ +import { spawnSync } from "node:child_process"; +import { dirname, resolve } from "node:path"; +import { fileURLToPath } from "node:url"; +import { describe, expect, it } from "vitest"; + +const packageRoot = resolve(dirname(fileURLToPath(import.meta.url)), ".."); +const aiEntryUrl = new URL("../src/index.ts", import.meta.url).href; +const compatEntryUrl = new URL("../src/compat.ts", import.meta.url).href; +const providersAllUrl = new URL("../src/providers/all.ts", import.meta.url).href; + +const SDK_SPECIFIERS = ["@anthropic-ai/sdk", "openai", "@google/genai", "@aws-sdk/client-bedrock-runtime"] as const; + +type ProbeResult = { + loadedSpecifiers: string[]; +}; + +function runProbe(action: string): ProbeResult { + const script = ` + import { registerHooks } from "node:module"; + + const targets = new Set(${JSON.stringify(SDK_SPECIFIERS)}); + const loaded = []; + + registerHooks({ + resolve(specifier, context, nextResolve) { + if (targets.has(specifier)) { + loaded.push(specifier); + } + return nextResolve(specifier, context); + }, + }); + + const mod = await import(${JSON.stringify(aiEntryUrl)}); + ${action} + console.log(JSON.stringify({ loadedSpecifiers: [...new Set(loaded)] })); + `; + + const result = spawnSync(process.execPath, ["--input-type=module", "--eval", script], { + cwd: packageRoot, + encoding: "utf8", + }); + + if (result.status !== 0) { + throw new Error(`Probe failed (exit ${result.status})\nSTDOUT:\n${result.stdout}\nSTDERR:\n${result.stderr}`); + } + + const stdoutLines = result.stdout + .split(/\r?\n/) + .map((line) => line.trim()) + .filter((line) => line.length > 0); + const lastLine = stdoutLines.at(-1); + if (!lastLine) { + throw new Error(`Probe produced no output\nSTDERR:\n${result.stderr}`); + } + + return JSON.parse(lastLine) as ProbeResult; +} + +describe("lazy provider module loading", () => { + it("does not load provider SDKs when importing the root barrel", () => { + const result = runProbe(""); + expect(result.loadedSpecifiers).toEqual([]); + }); + + it("does not load provider SDKs when building all builtin providers", () => { + const result = runProbe(` + const all = await import(${JSON.stringify(providersAllUrl)}); + const models = all.builtinModels(); + models.getModels(); + `); + expect(result.loadedSpecifiers).toEqual([]); + }); + + it("does not load provider SDKs when importing the compat entrypoint", () => { + const result = runProbe(` + await import(${JSON.stringify(compatEntryUrl)}); + `); + expect(result.loadedSpecifiers).toEqual([]); + }); + + it("loads only the Anthropic SDK when streaming through the lazy API wrapper", () => { + const result = runProbe(` + const compat = await import(${JSON.stringify(compatEntryUrl)}); + const model = { + id: "claude-sonnet-4-6", + name: "Claude Sonnet 4", + api: "anthropic-messages", + provider: "anthropic", + baseUrl: "https://api.anthropic.com", + reasoning: true, + input: ["text"], + cost: { input: 0, output: 0, cacheRead: 0, cacheWrite: 0 }, + contextWindow: 200000, + maxTokens: 8192, + }; + const context = { messages: [{ role: "user", content: "hi" }] }; + await compat.anthropicMessagesApi().streamSimple(model, context).result(); + `); + + expect(result.loadedSpecifiers).toEqual(["@anthropic-ai/sdk"]); + }); + + it("loads only the Anthropic SDK when dispatching through streamSimple", () => { + const result = runProbe(` + const compat = await import(${JSON.stringify(compatEntryUrl)}); + const model = compat.getModel("anthropic", "claude-sonnet-4-6"); + const context = { messages: [{ role: "user", content: "hi" }] }; + await compat.streamSimple(model, context).result(); + `); + + expect(result.loadedSpecifiers).toEqual(["@anthropic-ai/sdk"]); + }); +}); diff --git a/packages/ai/test/max-thinking.test.ts b/packages/ai/test/max-thinking.test.ts new file mode 100644 index 0000000000000000000000000000000000000000..5620941b4c8140dbc59545cb8dc51bf91430931f --- /dev/null +++ b/packages/ai/test/max-thinking.test.ts @@ -0,0 +1,89 @@ +import { describe, expect, it } from "vitest"; +import { streamSimple as streamSimpleOpenAICodexResponses } from "../src/api/openai-codex-responses.ts"; +import { clampThinkingLevel, getModel, getSupportedThinkingLevels } from "../src/compat.ts"; +import type { Context, Model } from "../src/types.ts"; + +function mockToken(): string { + const payload = Buffer.from( + JSON.stringify({ "https://api.openai.com/auth": { chatgpt_account_id: "acc_test" } }), + "utf8", + ).toString("base64"); + return `aaa.${payload}.bbb`; +} + +describe("max thinking level", () => { + it("is opt-in for ordinary reasoning models", () => { + const model: Model<"openai-completions"> = { + id: "ordinary-reasoning", + name: "Ordinary Reasoning", + api: "openai-completions", + provider: "test", + baseUrl: "https://example.com/v1", + reasoning: true, + input: ["text"], + cost: { input: 0, output: 0, cacheRead: 0, cacheWrite: 0 }, + contextWindow: 128000, + maxTokens: 4096, + }; + + expect(getSupportedThinkingLevels(model)).toEqual(["off", "minimal", "low", "medium", "high"]); + expect(clampThinkingLevel(model, "max")).toBe("high"); + }); + + it.each(["gpt-5.6-luna", "gpt-5.6-sol", "gpt-5.6-terra"] as const)( + "exposes xhigh and max for openai-codex/%s", + (modelId) => { + const model = getModel("openai-codex", modelId); + expect(model).toBeDefined(); + expect(model?.thinkingLevelMap).toMatchObject({ xhigh: "xhigh", max: "max" }); + expect(getSupportedThinkingLevels(model!)).toEqual([ + "off", + "minimal", + "low", + "medium", + "high", + "xhigh", + "max", + ]); + }, + ); + + it("supports a hole between high and max", () => { + const model: Model<"openai-completions"> = { + id: "high-and-max", + name: "High and Max", + api: "openai-completions", + provider: "test", + baseUrl: "https://example.com/v1", + reasoning: true, + thinkingLevelMap: { xhigh: null, max: "max" }, + input: ["text"], + cost: { input: 0, output: 0, cacheRead: 0, cacheWrite: 0 }, + contextWindow: 128000, + maxTokens: 4096, + }; + + expect(getSupportedThinkingLevels(model)).toEqual(["off", "minimal", "low", "medium", "high", "max"]); + expect(clampThinkingLevel(model, "xhigh")).toBe("max"); + }); + + it.each(["gpt-5.6-sol", "gpt-6-astra"] as const)("sends max to the Codex Responses API for %s", async (modelId) => { + const model = getModel("openai-codex", modelId)!; + const context: Context = { + systemPrompt: "You are a helpful assistant.", + messages: [{ role: "user", content: "Hello", timestamp: Date.now() }], + }; + let payload: unknown; + + await streamSimpleOpenAICodexResponses(model, context, { + apiKey: mockToken(), + reasoning: "max", + onPayload: (request) => { + payload = request; + throw new Error("payload captured"); + }, + }).result(); + + expect(payload).toMatchObject({ reasoning: { effort: "max", summary: "auto" } }); + }); +}); diff --git a/packages/ai/test/mistral-http-transport.test.ts b/packages/ai/test/mistral-http-transport.test.ts new file mode 100644 index 0000000000000000000000000000000000000000..b4282ffe75b9b43e6a3f740d79d40fd879ef6f85 --- /dev/null +++ b/packages/ai/test/mistral-http-transport.test.ts @@ -0,0 +1,431 @@ +import { arch, platform, release } from "node:os"; +import { Type } from "typebox"; +import { describe, expect, it } from "vitest"; +import { stream as streamMistral } from "../src/api/mistral-conversations.ts"; +import { getModel } from "../src/compat.ts"; +import type { Context, FetchFunction, ProviderResponse } from "../src/types.ts"; + +const PI_USER_AGENT = `pi (${platform()} ${release()}; ${arch()})`; + +function createSseResponse(events: unknown[], headers?: Record): Response { + const body = `${events.map((event) => `data: ${JSON.stringify(event)}`).join("\r\n\r\n")}\r\n\r\ndata: [DONE]\r\n\r\n`; + return new Response(body, { + headers: { "content-type": "text/event-stream", ...headers }, + }); +} + +function createBytewiseSseResponse(event: unknown): Response { + const bytes = new TextEncoder().encode(`data: ${JSON.stringify(event)}\r\n\r\ndata: [DONE]\r\n\r\n`); + return new Response( + new ReadableStream({ + start(controller) { + for (const byte of bytes) controller.enqueue(Uint8Array.of(byte)); + controller.close(); + }, + }), + { headers: { "content-type": "text/event-stream" } }, + ); +} + +function createTerminalEvent(finishReason = "stop") { + return { + id: "mistral-response-id", + model: "mistral-large-latest", + choices: [{ index: 0, finish_reason: finishReason, delta: {} }], + usage: { prompt_tokens: 1, completion_tokens: 1, total_tokens: 2 }, + }; +} + +describe("Mistral HTTP transport", () => { + it("serializes SDK-style payloads to the Mistral wire format", async () => { + const model = getModel("mistral", "mistral-large-latest"); + const context: Context = { + systemPrompt: "Be precise", + messages: [ + { + role: "user", + content: [ + { type: "text", text: "describe" }, + { type: "image", data: "aGVsbG8=", mimeType: "image/png" }, + ], + timestamp: 1, + }, + ], + tools: [ + { + name: "lookup", + description: "Look something up", + parameters: Type.Object({ query: Type.String() }), + }, + ], + }; + let requestUrl: string | undefined; + let requestInit: RequestInit | undefined; + let callbackPayload: Record | undefined; + let callbackResponse: ProviderResponse | undefined; + const fetch: FetchFunction = async (input, init) => { + requestUrl = String(input); + requestInit = init; + return createSseResponse([createTerminalEvent()], { "x-request-id": "request-1" }); + }; + + const message = await streamMistral(model, context, { + apiKey: "secret", + fetch, + headers: { "x-custom": "value" }, + maxTokens: 123, + promptMode: "reasoning", + reasoningEffort: "high", + toolChoice: { type: "function", function: { name: "lookup" } }, + sessionId: "session-1", + onPayload: (payload) => { + callbackPayload = payload as Record; + return { + ...callbackPayload, + topP: 0.9, + randomSeed: 42, + responseFormat: { + type: "json_schema", + jsonSchema: { + name: "result", + schemaDefinition: { + type: "object", + properties: { maxTokens: { type: "number" } }, + }, + }, + }, + presencePenalty: 0.1, + frequencyPenalty: 0.2, + parallelToolCalls: true, + safePrompt: true, + }; + }, + onResponse: (response) => { + callbackResponse = response; + }, + }).result(); + + expect(message.stopReason).toBe("stop"); + expect(requestUrl).toBe("https://api.mistral.ai/v1/chat/completions"); + const headers = new Headers(requestInit?.headers); + expect(headers.get("authorization")).toBe("Bearer secret"); + expect(headers.get("accept")).toBe("text/event-stream"); + expect(headers.get("x-affinity")).toBe("session-1"); + expect(headers.get("x-custom")).toBe("value"); + expect(headers.get("user-agent")).toBe(PI_USER_AGENT); + expect(callbackPayload?.maxTokens).toBe(123); + expect(callbackPayload?.promptMode).toBe("reasoning"); + expect(callbackPayload?.promptCacheKey).toBe("session-1"); + expect(callbackResponse).toEqual({ + status: 200, + headers: { "content-type": "text/event-stream", "x-request-id": "request-1" }, + }); + + const wirePayload = JSON.parse(String(requestInit?.body)) as Record; + expect(wirePayload.max_tokens).toBe(123); + expect(wirePayload.prompt_mode).toBe("reasoning"); + expect(wirePayload.reasoning_effort).toBe("high"); + expect(wirePayload.tool_choice).toEqual({ type: "function", function: { name: "lookup" } }); + expect(wirePayload.prompt_cache_key).toBe("session-1"); + expect(wirePayload.top_p).toBe(0.9); + expect(wirePayload.random_seed).toBe(42); + expect(wirePayload.presence_penalty).toBe(0.1); + expect(wirePayload.frequency_penalty).toBe(0.2); + expect(wirePayload.parallel_tool_calls).toBe(true); + expect(wirePayload.safe_prompt).toBe(true); + expect(wirePayload.response_format).toEqual({ + type: "json_schema", + json_schema: { + name: "result", + schema: { + type: "object", + properties: { maxTokens: { type: "number" } }, + }, + }, + }); + expect(wirePayload).not.toHaveProperty("maxTokens"); + expect(wirePayload).not.toHaveProperty("promptMode"); + expect(wirePayload).not.toHaveProperty("promptCacheKey"); + expect(wirePayload.messages).toEqual([ + { role: "system", content: "Be precise" }, + { + role: "user", + content: [ + { type: "text", text: "describe" }, + { type: "image_url", image_url: "data:image/png;base64,aGVsbG8=" }, + ], + }, + ]); + }); + + it("serializes assistant thinking, tool calls, and tool results for replay", async () => { + const model = getModel("mistral", "mistral-large-latest"); + const context: Context = { + messages: [ + { + role: "assistant", + api: "mistral-conversations", + provider: "mistral", + model: model.id, + content: [ + { type: "thinking", thinking: "reason" }, + { type: "text", text: "answer" }, + { type: "toolCall", id: "abc123456", name: "lookup", arguments: { query: "pi" } }, + ], + usage: { + input: 0, + output: 0, + cacheRead: 0, + cacheWrite: 0, + totalTokens: 0, + cost: { input: 0, output: 0, cacheRead: 0, cacheWrite: 0, total: 0 }, + }, + stopReason: "toolUse", + timestamp: 1, + }, + { + role: "toolResult", + toolCallId: "abc123456", + toolName: "lookup", + content: [ + { type: "text", text: "found" }, + { type: "image", data: "aGVsbG8=", mimeType: "image/png" }, + ], + isError: false, + timestamp: 2, + }, + ], + }; + let requestInit: RequestInit | undefined; + const fetch: FetchFunction = async (_input, init) => { + requestInit = init; + return createSseResponse([createTerminalEvent()]); + }; + + const message = await streamMistral(model, context, { apiKey: "test", fetch }).result(); + + expect(message.stopReason).toBe("stop"); + const wirePayload = JSON.parse(String(requestInit?.body)) as { messages: unknown[] }; + expect(wirePayload.messages).toEqual([ + { + role: "assistant", + prefix: false, + content: [ + { type: "thinking", thinking: [{ type: "text", text: "reason" }] }, + { type: "text", text: "answer" }, + ], + tool_calls: [ + { + id: "abc123456", + type: "function", + function: { name: "lookup", arguments: '{"query":"pi"}' }, + index: 0, + }, + ], + }, + { + role: "tool", + tool_call_id: "abc123456", + name: "lookup", + content: [ + { type: "text", text: "found" }, + { type: "image_url", image_url: "data:image/png;base64,aGVsbG8=" }, + ], + }, + ]); + }); + + it("parses native thinking, text, fragmented tool calls, and cached-token usage", async () => { + const model = getModel("mistral", "mistral-large-latest"); + const context: Context = { + messages: [{ role: "user", content: "hello", timestamp: 1 }], + }; + const events = [ + { + id: "response-1", + model: model.id, + choices: [ + { + index: 0, + finish_reason: null, + delta: { content: [{ type: "thinking", thinking: [{ type: "text", text: "reason" }] }] }, + }, + ], + }, + { + id: "response-1", + model: model.id, + choices: [ + { + index: 0, + finish_reason: null, + delta: { content: [{ type: "text", text: "answer" }] }, + }, + ], + }, + { + id: "response-1", + model: model.id, + choices: [ + { + index: 0, + finish_reason: null, + delta: { + tool_calls: [ + { + id: "abc123456", + index: 0, + function: { name: "lookup", arguments: '{"query":' }, + }, + ], + }, + }, + ], + }, + { + id: "response-1", + model: model.id, + choices: [ + { + index: 0, + finish_reason: "tool_calls", + delta: { + tool_calls: [ + { + index: 0, + function: { name: "", arguments: '"pi"}' }, + }, + ], + }, + }, + ], + usage: { + prompt_tokens: 10, + completion_tokens: 4, + total_tokens: 14, + prompt_tokens_details: { cached_tokens: 3 }, + }, + }, + ]; + const fetch: FetchFunction = async () => createSseResponse(events); + + const message = await streamMistral(model, context, { apiKey: "test", fetch }).result(); + + expect(message.stopReason).toBe("toolUse"); + expect(message.rawStopReason).toBe("tool_calls"); + expect(message.responseId).toBe("response-1"); + expect(message.content).toEqual([ + { type: "thinking", thinking: "reason" }, + { type: "text", text: "answer" }, + { type: "toolCall", id: "abc123456", name: "lookup", arguments: { query: "pi" } }, + ]); + expect(message.usage).toMatchObject({ input: 7, output: 4, cacheRead: 3, cacheWrite: 0, totalTokens: 14 }); + }); + + it("parses SSE and UTF-8 sequences split across transport chunks", async () => { + const model = getModel("mistral", "mistral-large-latest"); + const context: Context = { + messages: [{ role: "user", content: "hello", timestamp: 1 }], + }; + const fetch: FetchFunction = async () => + createBytewiseSseResponse({ + id: "response-bytewise", + model: model.id, + choices: [{ index: 0, finish_reason: "stop", delta: { content: "héllo 🌍" } }], + usage: { prompt_tokens: 1, completion_tokens: 2, total_tokens: 3 }, + }); + + const message = await streamMistral(model, context, { apiKey: "test", fetch }).result(); + + expect(message.stopReason).toBe("stop"); + expect(message.content).toEqual([{ type: "text", text: "héllo 🌍" }]); + }); + + it("honors case-insensitive header overrides and explicit affinity suppression", async () => { + const model = { + ...getModel("mistral", "mistral-large-latest"), + headers: { Authorization: "Bearer model-key", "X-Affinity": "model-affinity" }, + }; + const context: Context = { + messages: [{ role: "user", content: "hello", timestamp: 1 }], + }; + let requestHeaders: Headers | undefined; + const fetch: FetchFunction = async (_input, init) => { + requestHeaders = new Headers(init?.headers); + return createSseResponse([createTerminalEvent()]); + }; + + await streamMistral(model, context, { + apiKey: "request-key", + fetch, + sessionId: "automatic-affinity", + headers: { authorization: null, "x-affinity": null, "User-Agent": "custom-agent" }, + }).result(); + + expect(requestHeaders?.has("authorization")).toBe(false); + expect(requestHeaders?.has("x-affinity")).toBe(false); + expect(requestHeaders?.get("user-agent")).toBe("custom-agent"); + }); + + it("aborts while waiting for an SSE chunk", async () => { + const model = getModel("mistral", "mistral-large-latest"); + const context: Context = { + messages: [{ role: "user", content: "hello", timestamp: 1 }], + }; + const controller = new AbortController(); + const fetch: FetchFunction = async () => + new Response( + new ReadableStream({ + start() {}, + }), + { headers: { "content-type": "text/event-stream" } }, + ); + + const result = streamMistral(model, context, { + apiKey: "test", + fetch, + signal: controller.signal, + }).result(); + controller.abort(); + const message = await result; + + expect(message.stopReason).toBe("aborted"); + }); + + it("applies the request timeout while waiting for an SSE chunk", async () => { + const model = getModel("mistral", "mistral-large-latest"); + const context: Context = { + messages: [{ role: "user", content: "hello", timestamp: 1 }], + }; + const fetch: FetchFunction = async () => + new Response( + new ReadableStream({ + start() {}, + }), + { headers: { "content-type": "text/event-stream" } }, + ); + + const message = await streamMistral(model, context, { + apiKey: "test", + fetch, + timeoutMs: 5, + }).result(); + + expect(message.stopReason).toBe("error"); + expect(message.errorMessage).toMatch(/timeout/i); + }); + + it("preserves HTTP status and response bodies in errors", async () => { + const model = getModel("mistral", "mistral-large-latest"); + const context: Context = { + messages: [{ role: "user", content: "hello", timestamp: 1 }], + }; + const fetch: FetchFunction = async () => + new Response('{"message":"blocked by gateway"}', { status: 403, statusText: "Forbidden" }); + + const message = await streamMistral(model, context, { apiKey: "test", fetch }).result(); + + expect(message.stopReason).toBe("error"); + expect(message.errorMessage).toBe('Mistral API error (403): {"message":"blocked by gateway"}'); + }); +}); diff --git a/packages/ai/test/mistral-reasoning-mode.test.ts b/packages/ai/test/mistral-reasoning-mode.test.ts new file mode 100644 index 0000000000000000000000000000000000000000..2d93a389e44a60047c4b8ec0fd93c1f7f29cb7ad --- /dev/null +++ b/packages/ai/test/mistral-reasoning-mode.test.ts @@ -0,0 +1,135 @@ +import { describe, expect, it } from "vitest"; +import { streamSimple } from "../src/api/mistral-conversations.ts"; +import type { Context, Model, SimpleStreamOptions } from "../src/types.ts"; + +interface MistralPayload { + promptMode?: "reasoning"; + reasoningEffort?: "none" | "high"; + promptCacheKey?: string; +} + +function makeModel(id: string, reasoning: boolean): Model<"mistral-conversations"> { + return { + id, + name: id, + api: "mistral-conversations", + provider: "mistral", + baseUrl: "http://127.0.0.1:9", + reasoning, + input: ["text"], + cost: { input: 0, output: 0, cacheRead: 0, cacheWrite: 0 }, + contextWindow: 128000, + maxTokens: 16384, + }; +} + +function makeContext(): Context { + return { + messages: [{ role: "user", content: "Hello", timestamp: Date.now() }], + }; +} + +async function capturePayload( + model: Model<"mistral-conversations">, + options?: SimpleStreamOptions, +): Promise { + let capturedPayload: MistralPayload | undefined; + const stream = streamSimple(model, makeContext(), { + ...options, + apiKey: "fake-key", + onPayload: (payload) => { + capturedPayload = payload as MistralPayload; + return payload; + }, + }); + + await stream.result(); + + if (!capturedPayload) { + throw new Error("Expected payload to be captured before request failure"); + } + + return capturedPayload; +} + +describe("Mistral reasoning mode selection", () => { + it("uses reasoning_effort for Mistral Small 4", async () => { + const payload = await capturePayload(makeModel("mistral-small-2603", true), { reasoning: "medium" }); + + expect(payload.reasoningEffort).toBe("high"); + expect(payload.promptMode).toBeUndefined(); + }); + + it("omits reasoning controls for Mistral Small 4 when thinking is off", async () => { + const payload = await capturePayload(makeModel("mistral-small-2603", true)); + + expect(payload.reasoningEffort).toBeUndefined(); + expect(payload.promptMode).toBeUndefined(); + }); + + it("uses prompt_mode for Magistral reasoning models", async () => { + const payload = await capturePayload(makeModel("magistral-medium-latest", true), { reasoning: "medium" }); + + expect(payload.promptMode).toBe("reasoning"); + expect(payload.reasoningEffort).toBeUndefined(); + }); + + // Regression for #9375: Mistral-hosted GLM-5.2 ignores prompt_mode. + describe("zai-glm-5-2", () => { + it("uses reasoning_effort when thinking is enabled", async () => { + const payload = await capturePayload(makeModel("zai-glm-5-2", true), { reasoning: "medium" }); + + expect(payload.reasoningEffort).toBe("high"); + expect(payload.promptMode).toBeUndefined(); + }); + + it("omits reasoning controls when thinking is off", async () => { + const payload = await capturePayload(makeModel("zai-glm-5-2", true)); + + expect(payload.reasoningEffort).toBeUndefined(); + expect(payload.promptMode).toBeUndefined(); + }); + }); + + // Regression for #8700: Medium aliases must use reasoning_effort, not Magistral's prompt_mode. + describe.each(["mistral-medium-2604", "mistral-medium-latest"] as const)("%s", (modelId) => { + it("uses reasoning_effort when thinking is enabled", async () => { + const payload = await capturePayload(makeModel(modelId, true), { reasoning: "medium" }); + + expect(payload.reasoningEffort).toBe("high"); + expect(payload.promptMode).toBeUndefined(); + }); + + it("omits reasoning controls when thinking is off", async () => { + const payload = await capturePayload(makeModel(modelId, true)); + + expect(payload.reasoningEffort).toBeUndefined(); + expect(payload.promptMode).toBeUndefined(); + }); + }); + + // Regression for #8700: the Medium prefix must still respect the model's reasoning capability. + it("omits reasoning controls for non-reasoning Medium models", async () => { + const payload = await capturePayload(makeModel("mistral-medium-2505", false), { reasoning: "medium" }); + + expect(payload.reasoningEffort).toBeUndefined(); + expect(payload.promptMode).toBeUndefined(); + }); + + it("uses the session id as prompt cache key", async () => { + const payload = await capturePayload(makeModel("mistral-large-latest", false), { + sessionId: "session-123", + }); + + expect(payload.promptCacheKey).toBe("session-123"); + }); + + it("omits prompt cache key when cache retention is disabled", async () => { + const payload = await capturePayload(makeModel("mistral-large-latest", false), { + sessionId: "session-123", + cacheRetention: "none", + }); + + expect(payload.promptCacheKey).toBeUndefined(); + }); +}); diff --git a/packages/ai/test/model-catalog-types.test.ts b/packages/ai/test/model-catalog-types.test.ts new file mode 100644 index 0000000000000000000000000000000000000000..57f02e0f2265831bf2b3cc527269bd8a2573f0f3 --- /dev/null +++ b/packages/ai/test/model-catalog-types.test.ts @@ -0,0 +1,25 @@ +import { expect, expectTypeOf, it } from "vitest"; +import { GITHUB_COPILOT_MODELS } from "../src/providers/github-copilot.models.ts"; +import { XAI_MODELS } from "../src/providers/xai.models.ts"; + +it("derives model API, ID, and provider literals from grouped model data", () => { + expectTypeOf(XAI_MODELS["grok-4.5"].api).toEqualTypeOf<"openai-responses">(); + expectTypeOf(XAI_MODELS["grok-4.5"].id).toEqualTypeOf<"grok-4.5">(); + expectTypeOf(XAI_MODELS["grok-4.5"].provider).toEqualTypeOf<"xai">(); + expectTypeOf(XAI_MODELS["grok-4.6"].api).toEqualTypeOf<"openai-responses">(); + expectTypeOf(XAI_MODELS["grok-4.6"].id).toEqualTypeOf<"grok-4.6">(); + expectTypeOf(XAI_MODELS["grok-4.3"].api).toEqualTypeOf<"openai-responses">(); +}); + +it("routes GitHub Copilot Grok 4.5 through the Responses API", () => { + expectTypeOf(GITHUB_COPILOT_MODELS["grok-4.5"].api).toEqualTypeOf<"openai-responses">(); + expect(GITHUB_COPILOT_MODELS["grok-4.5"].api).toBe("openai-responses"); +}); + +// Regression test for https://github.com/earendil-works/pi/issues/9209 +it("routes all GitHub Copilot GPT models through the Responses API", () => { + const gptModels = Object.values(GITHUB_COPILOT_MODELS).filter((model) => model.id.startsWith("gpt-")); + expect(gptModels.length).toBeGreaterThan(0); + expect(gptModels.every((model) => model.api === "openai-responses")).toBe(true); + expectTypeOf(GITHUB_COPILOT_MODELS["gpt-6-astra"].api).toEqualTypeOf<"openai-responses">(); +}); diff --git a/packages/ai/test/model-data-validation.test.ts b/packages/ai/test/model-data-validation.test.ts new file mode 100644 index 0000000000000000000000000000000000000000..f0e1b9fbcfc951e72c3a937080a709b6d64fc351 --- /dev/null +++ b/packages/ai/test/model-data-validation.test.ts @@ -0,0 +1,177 @@ +import { mkdirSync, mkdtempSync, readFileSync, rmSync, writeFileSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { afterEach, describe, expect, it } from "vitest"; +import { + assertExactModelIds, + createModelDataManifest, + MODEL_DATA_MANIFEST_FILE, + MODEL_DATA_SCHEMA_VERSION, + type ModelDataStructure, + readModelDataStructure, + validateModelDataDirectory, +} from "../scripts/model-data.ts"; + +const GENERATED_AT = "2026-07-23T10:00:00.000Z"; +const temporaryRoots: string[] = []; + +afterEach(() => { + for (const root of temporaryRoots.splice(0)) rmSync(root, { force: true, recursive: true }); +}); + +function createFixture(): { + dataDir: string; + packageRoot: string; + structure: ModelDataStructure; + values: Record; +} { + const packageRoot = mkdtempSync(join(tmpdir(), "pi-model-data-")); + temporaryRoots.push(packageRoot); + const providersDir = join(packageRoot, "src", "providers"); + const dataDir = join(providersDir, "data"); + mkdirSync(dataDir, { recursive: true }); + writeFileSync( + join(packageRoot, "src", "models.generated.ts"), + 'import { TEST_PROVIDER_MODELS } from "./providers/test-provider.models.ts";\n', + ); + writeFileSync( + join(providersDir, "test-provider.models.ts"), + 'import values from "./data/test-provider.json" with { type: "json" };\nimport { flattenModelCatalog, type ModelCatalog } from "../model-catalog.ts";\n\nexport const TEST_PROVIDER_MODELS: ModelCatalog =\n\tflattenModelCatalog("test-provider", values);\n', + ); + + const structure: ModelDataStructure = { + "test-provider": { + "model-a": "openai-completions", + }, + }; + const values: Record = { + "model-a": { + id: "model-a", + name: "Model A", + api: "openai-completions", + provider: "test-provider", + baseUrl: "https://example.test/v1", + reasoning: false, + input: ["text"], + cost: { input: 1, output: 2, cacheRead: 0, cacheWrite: 0 }, + contextWindow: 1000, + maxTokens: 100, + }, + }; + writeFixtureData(dataDir, structure, values); + return { dataDir, packageRoot, structure, values }; +} + +function writeFixtureData( + dataDir: string, + structure: ModelDataStructure, + values: Record, + manifestSchemaVersion = MODEL_DATA_SCHEMA_VERSION, + apiGroup = "openai-completions", +): void { + const filename = "test-provider.json"; + const content = `${JSON.stringify({ [apiGroup]: values })}\n`; + writeFileSync(join(dataDir, filename), content); + const manifest = createModelDataManifest(structure, { [filename]: content }, GENERATED_AT); + manifest.schemaVersion = manifestSchemaVersion; + writeFileSync(join(dataDir, MODEL_DATA_MANIFEST_FILE), `${JSON.stringify(manifest)}\n`); +} + +describe("generated model data validation", () => { + it("rejects a missing upstream model from an exact generated allowlist", () => { + expect(() => assertExactModelIds("qwen-token-plan-individual", ["model-a", "model-b"], ["model-a"])).toThrow( + "qwen-token-plan-individual model IDs do not match (missing: model-b)", + ); + }); + + it("rejects an unexpected model from an exact generated allowlist", () => { + expect(() => assertExactModelIds("test-provider", ["model-a"], ["model-a", "model-b"])).toThrow( + "test-provider model IDs do not match (extra: model-b)", + ); + }); + + it("reads and validates API-grouped model data", () => { + const { dataDir, packageRoot, structure } = createFixture(); + expect(readModelDataStructure(packageRoot)).toEqual(structure); + expect(() => validateModelDataDirectory(structure, dataDir)).not.toThrow(); + }); + + it("rejects a missing model data directory", () => { + const { dataDir, structure } = createFixture(); + rmSync(dataDir, { recursive: true }); + expect(() => validateModelDataDirectory(structure, dataDir)).toThrow("does not exist"); + }); + + it.each([ + ["id", "wrong-id", "has id"], + ["provider", "wrong-provider", "has provider"], + ["api", "anthropic-messages", "has api"], + ] as const)("rejects a wrong model %s", (field, value, expectedMessage) => { + const fixture = createFixture(); + const model = fixture.values["model-a"] as Record; + model[field] = value; + writeFixtureData(fixture.dataDir, fixture.structure, fixture.values); + expect(() => validateModelDataDirectory(fixture.structure, fixture.dataDir)).toThrow(expectedMessage); + }); + + it("rejects a model in the wrong API group", () => { + const fixture = createFixture(); + writeFixtureData( + fixture.dataDir, + fixture.structure, + fixture.values, + MODEL_DATA_SCHEMA_VERSION, + "anthropic-messages", + ); + expect(() => validateModelDataDirectory(fixture.structure, fixture.dataDir)).toThrow("grouped under API"); + }); + + it("rejects duplicate model IDs across API groups", () => { + const fixture = createFixture(); + const filename = "test-provider.json"; + const content = `${JSON.stringify({ + "openai-completions": fixture.values, + "anthropic-messages": fixture.values, + })}\n`; + writeFileSync(join(fixture.dataDir, filename), content); + const manifest = createModelDataManifest(fixture.structure, { [filename]: content }, GENERATED_AT); + writeFileSync(join(fixture.dataDir, MODEL_DATA_MANIFEST_FILE), `${JSON.stringify(manifest)}\n`); + expect(() => validateModelDataDirectory(fixture.structure, fixture.dataDir)).toThrow("more than one API group"); + }); + + it("rejects missing model IDs and stale file hashes", () => { + const fixture = createFixture(); + writeFileSync(join(fixture.dataDir, "test-provider.json"), "{}\n"); + expect(() => validateModelDataDirectory(fixture.structure, fixture.dataDir)).toThrow(/manifest hash|model IDs/); + }); + + it("rejects incompatible schema and generation stamps", () => { + const fixture = createFixture(); + writeFixtureData(fixture.dataDir, fixture.structure, fixture.values, MODEL_DATA_SCHEMA_VERSION + 1); + expect(() => validateModelDataDirectory(fixture.structure, fixture.dataDir)).toThrow("model data schema"); + + const manifestPath = join(fixture.dataDir, MODEL_DATA_MANIFEST_FILE); + const manifest = JSON.parse(readFileSync(manifestPath, "utf8")) as Record; + manifest.structureHash = "stale"; + writeFileSync(manifestPath, `${JSON.stringify(manifest)}\n`); + expect(() => validateModelDataDirectory(fixture.structure, fixture.dataDir)).toThrow("generation stamp"); + }); + + it("rejects an invalid generation timestamp", () => { + const fixture = createFixture(); + const manifestPath = join(fixture.dataDir, MODEL_DATA_MANIFEST_FILE); + const manifest = JSON.parse(readFileSync(manifestPath, "utf8")) as Record; + manifest.generatedAt = "invalid"; + writeFileSync(manifestPath, `${JSON.stringify(manifest)}\n`); + expect(() => validateModelDataDirectory(fixture.structure, fixture.dataDir)).toThrow("generation timestamp"); + }); + + it("rejects missing provider shards imported by the aggregator", () => { + const { packageRoot } = createFixture(); + writeFileSync( + join(packageRoot, "src", "models.generated.ts"), + 'import { TEST_PROVIDER_MODELS } from "./providers/test-provider.models.ts";\nimport { MISSING_MODELS } from "./providers/missing.models.ts";\n', + ); + expect(() => readModelDataStructure(packageRoot)).toThrow("aggregator and provider shards do not match"); + }); +}); diff --git a/packages/ai/test/models-runtime.test.ts b/packages/ai/test/models-runtime.test.ts new file mode 100644 index 0000000000000000000000000000000000000000..ffbcf6af3facd42026cc31ea54bb08976c2173b2 --- /dev/null +++ b/packages/ai/test/models-runtime.test.ts @@ -0,0 +1,1159 @@ +import { describe, expect, it, vi } from "vitest"; +import { InMemoryCredentialStore } from "../src/auth/credential-store.ts"; +import type { ApiKeyAuth, CredentialStore, OAuthAuth, OAuthCredential, ProviderAuth } from "../src/auth/types.ts"; +import { calculateCost, createModels, createProvider, hasApi, type Provider } from "../src/models.ts"; +import { InMemoryModelsStore, type ModelsStore, type ModelsStoreEntry } from "../src/models-store.ts"; +import type { Api, AssistantMessage, Context, Model, SimpleStreamOptions, StreamOptions, Usage } from "../src/types.ts"; +import { AssistantMessageEventStream } from "../src/utils/event-stream.ts"; + +function testModel(provider: string, id: string): Model { + return { + id, + name: id, + api: "test-api", + provider, + baseUrl: "https://example.test/v1", + reasoning: false, + input: ["text"], + cost: { input: 0, output: 0, cacheRead: 0, cacheWrite: 0 }, + contextWindow: 10000, + maxTokens: 1000, + }; +} + +function doneMessage(model: Model, text: string): AssistantMessage { + return { + role: "assistant", + content: [{ type: "text", text }], + api: model.api, + provider: model.provider, + model: model.id, + usage: { + input: 0, + output: 0, + cacheRead: 0, + cacheWrite: 0, + totalTokens: 0, + cost: { input: 0, output: 0, cacheRead: 0, cacheWrite: 0, total: 0 }, + }, + stopReason: "stop", + timestamp: Date.now(), + }; +} + +interface ProviderCall { + model: Model; + options: StreamOptions | undefined; +} + +/** Ambient auth for keyless test providers; reports "configured" with no auth values. */ +const ambientAuth: ApiKeyAuth = { + name: "Ambient", + resolve: async () => ({ auth: {} }), +}; + +function testProvider(input: { + id: string; + models?: Model[]; + auth?: ProviderAuth; + getModels?: () => readonly Model[]; + refreshModels?: Provider["refreshModels"]; + calls?: ProviderCall[]; +}): Provider { + const models = input.models ?? [testModel(input.id, "model-a")]; + const respond = (model: Model, options: StreamOptions | undefined) => { + input.calls?.push({ model, options }); + const stream = new AssistantMessageEventStream(); + const message = doneMessage(model, "ok"); + stream.push({ type: "start", partial: message }); + stream.push({ type: "done", reason: "stop", message }); + stream.end(message); + return stream; + }; + return { + id: input.id, + name: input.id, + auth: input.auth ?? { apiKey: ambientAuth }, + getModels: input.getModels ?? (() => models), + refreshModels: input.refreshModels, + stream: (model, _context, options) => respond(model, options as StreamOptions | undefined), + streamSimple: (model, _context, options) => respond(model, options as SimpleStreamOptions | undefined), + }; +} + +const context: Context = { messages: [{ role: "user", content: "hi", timestamp: Date.now() }] }; + +function envKeyAuth(key: string | undefined): ApiKeyAuth { + return { + name: "Test API key", + resolve: async ({ credential }) => { + const resolved = credential?.key ?? key; + if (!resolved) return undefined; + return { auth: { apiKey: resolved }, source: credential ? "stored" : "env" }; + }, + }; +} + +function testOAuth(overrides?: Partial): OAuthAuth { + return { + name: "Test OAuth", + login: async () => { + throw new Error("not used"); + }, + refresh: async (credential) => credential, + toAuth: async (credential) => ({ apiKey: credential.access }), + ...overrides, + }; +} + +describe("Models runtime", () => { + it("enumerates credential metadata without exposing secrets", async () => { + const credentials = new InMemoryCredentialStore(); + await credentials.modify("api-provider", async () => ({ type: "api_key", key: "secret" })); + await credentials.modify("oauth-provider", async () => ({ + type: "oauth", + access: "access", + refresh: "refresh", + expires: Date.now() + 60_000, + })); + + expect(await credentials.list()).toEqual([ + { providerId: "api-provider", type: "api_key" }, + { providerId: "oauth-provider", type: "oauth" }, + ]); + }); + + it("applies request-wide pricing tiers above the configured input threshold", () => { + const model = testModel("openai", "gpt-5.6-sol"); + model.cost = { + input: 5, + output: 30, + cacheRead: 0.5, + cacheWrite: 6.25, + tiers: [ + { + inputTokensAbove: 272000, + input: 10, + output: 45, + cacheRead: 1, + cacheWrite: 12.5, + }, + ], + }; + const createUsage = (cacheWrite: number): Usage => ({ + input: 200000, + output: 100000, + cacheRead: 72000, + cacheWrite, + totalTokens: 372000 + cacheWrite, + cost: { input: 0, output: 0, cacheRead: 0, cacheWrite: 0, total: 0 }, + }); + + const short = calculateCost(model, createUsage(0)); + expect(short).toMatchObject({ input: 1, output: 3, cacheRead: 0.036, cacheWrite: 0 }); + + const long = calculateCost(model, createUsage(1)); + expect(long.input).toBe(2); + expect(long.output).toBe(4.5); + expect(long.cacheRead).toBe(0.072); + expect(long.cacheWrite).toBe(0.0000125); + }); + + it("registers, replaces, and deletes providers", () => { + const models = createModels(); + models.setProvider(testProvider({ id: "p1" })); + models.setProvider(testProvider({ id: "p2" })); + expect(models.getProviders().map((p) => p.id)).toEqual(["p1", "p2"]); + + const replacement = testProvider({ id: "p1" }); + models.setProvider(replacement); + expect(models.getProvider("p1")).toBe(replacement); + expect(models.getProviders()).toHaveLength(2); + + models.deleteProvider("p1"); + expect(models.getProvider("p1")).toBeUndefined(); + + models.clearProviders(); + expect(models.getProviders()).toHaveLength(0); + }); + + it("lists and finds models per provider", async () => { + const models = createModels(); + models.setProvider(testProvider({ id: "p1", models: [testModel("p1", "m1"), testModel("p1", "m2")] })); + models.setProvider(testProvider({ id: "p2", models: [testModel("p2", "m3")] })); + + expect(models.getModels().map((m) => m.id)).toEqual(["m1", "m2", "m3"]); + expect(models.getModels("p1").map((m) => m.id)).toEqual(["m1", "m2"]); + expect(models.getModels("nope").length).toBe(0); + expect(models.getModel("p2", "m3")?.id).toBe("m3"); + expect(models.getModel("p2", "missing")).toBeUndefined(); + + // hasApi() narrows dynamically looked-up models with a runtime check + const found = models.getModel("p2", "m3"); + expect(found && hasApi(found, "openai-completions")).toBe(false); + expect(found && hasApi(found, "test-api")).toBe(true); + if (found && hasApi(found, "test-api")) { + const _typed: Model<"test-api"> = found; + expect(_typed.id).toBe("m3"); + } + }); + + it("swallows provider source failures for both all-provider and single-provider listing", () => { + const models = createModels(); + models.setProvider( + testProvider({ + id: "broken", + getModels: () => { + throw new Error("boom"); + }, + }), + ); + models.setProvider(testProvider({ id: "ok", models: [testModel("ok", "m1")] })); + + expect(models.getModels().map((m) => m.id)).toEqual(["m1"]); + expect(models.getModels("broken")).toEqual([]); + // precise failures come from the provider directly + expect(() => models.getProvider("broken")?.getModels()).toThrow("boom"); + }); + + it("refresh() updates every configured dynamic provider and reports failures", async () => { + let list = [testModel("dyn", "before")]; + let refreshes = 0; + const models = createModels(); + models.setProvider( + testProvider({ + id: "dyn", + getModels: () => list, + refreshModels: async (refresh) => { + if (!refresh.allowNetwork) return; + refreshes++; + await refresh.publish({ + update: () => { + list = [testModel("dyn", "after")]; + }, + }); + }, + }), + ); + models.setProvider(testProvider({ id: "static", models: [testModel("static", "s1")] })); + + expect(models.getModel("dyn", "before")).toBeDefined(); + const first = await models.refresh(); + expect(first.errors.size).toBe(0); + expect(refreshes).toBe(1); + expect(models.getModel("dyn", "after")).toBeDefined(); + expect(models.getModel("dyn", "before")).toBeUndefined(); + + models.setProvider( + testProvider({ + id: "flaky", + refreshModels: async ({ allowNetwork }) => { + if (allowNetwork) throw new Error("fetch failed"); + }, + }), + ); + const second = await models.refresh(); + expect(refreshes).toBe(2); + expect(second.errors.get("flaky")?.message).toBe("fetch failed"); + }); + + it("restricts refresh work to selected providers", async () => { + const calls: string[] = []; + const models = createModels(); + for (const id of ["one", "two"]) { + models.setProvider( + testProvider({ + id, + refreshModels: async ({ allowNetwork }) => { + calls.push(`${id}:${allowNetwork ? "network" : "cache"}`); + }, + }), + ); + } + + const result = await models.refresh({ providers: ["two", "unknown"] }); + + expect(result.errors.size).toBe(0); + expect(calls).toEqual(["two:cache", "two:network"]); + }); + + it("restores cached models before waiting for network auth", async () => { + const store = new InMemoryModelsStore(); + await store.write("dynamic", { models: [testModel("dynamic", "cached")] }); + let markAuthStarted: (() => void) | undefined; + let finishAuth: (() => void) | undefined; + const authStarted = new Promise((resolve) => { + markAuthStarted = resolve; + }); + const blockedAuth = new Promise((resolve) => { + finishAuth = resolve; + }); + const provider = createProvider({ + id: "dynamic", + auth: { + apiKey: { + name: "Blocked auth", + resolve: async () => { + markAuthStarted?.(); + await blockedAuth; + return { auth: { apiKey: "key" } }; + }, + }, + }, + models: [], + fetchModels: async () => { + throw new Error("must not fetch"); + }, + api: { + stream: () => new AssistantMessageEventStream(), + streamSimple: () => new AssistantMessageEventStream(), + }, + }); + const models = createModels({ modelsStore: store }); + models.setProvider(provider); + const controller = new AbortController(); + const pending = models.refresh({ providers: ["dynamic"], signal: controller.signal }); + await authStarted; + + expect(models.getModel("dynamic", "cached")).toBeDefined(); + controller.abort(); + expect(await pending).toMatchObject({ aborted: true }); + finishAuth?.(); + }); + + it("lets providers choose persistent deletion and ephemeral publication atomically", async () => { + let entry: ModelsStoreEntry | undefined = { models: [testModel("dynamic", "stored")] }; + const store: ModelsStore = { + read: async () => entry, + write: async (_providerId, next) => { + entry = next; + }, + delete: async () => { + entry = undefined; + }, + }; + let state = "initial"; + const models = createModels({ modelsStore: store }); + models.setProvider( + testProvider({ + id: "dynamic", + refreshModels: async (context) => { + expect(context.stored?.models[0]?.id).toBe("stored"); + await context.publish({ + persist: null, + update: () => { + expect(entry).toBeUndefined(); + state = "deleted"; + }, + }); + await context.publish({ + update: () => { + state = "ephemeral"; + }, + }); + }, + }), + ); + + const result = await models.refresh({ allowNetwork: false }); + + expect(result.errors.size).toBe(0); + expect(entry).toBeUndefined(); + expect(state).toBe("ephemeral"); + }); + + it("persists dynamic catalogs and restores them without network access", async () => { + const credentials = new InMemoryCredentialStore(); + const modelsStore = new InMemoryModelsStore(); + await credentials.modify("dynamic", async () => ({ type: "api_key", key: "key" })); + const createDynamicProvider = (fetchModels: (() => Promise[]>) | undefined) => + createProvider({ + id: "dynamic", + auth: { apiKey: envKeyAuth(undefined) }, + models: [], + fetchModels: fetchModels ? () => fetchModels() : undefined, + api: { + stream: () => new AssistantMessageEventStream(), + streamSimple: () => new AssistantMessageEventStream(), + }, + }); + + const online = createModels({ credentials, modelsStore }); + online.setProvider(createDynamicProvider(async () => [testModel("dynamic", "fetched")])); + expect((await online.refresh()).errors.size).toBe(0); + expect(online.getModel("dynamic", "fetched")).toBeDefined(); + + const offline = createModels({ credentials, modelsStore }); + offline.setProvider( + createDynamicProvider(async () => { + throw new Error("must not fetch"); + }), + ); + expect((await offline.refresh({ allowNetwork: false })).errors.size).toBe(0); + expect(offline.getModel("dynamic", "fetched")).toBeDefined(); + }); + + it("passes effective API-key credentials and refresh options while skipping unconfigured providers", async () => { + let effectiveCredential: unknown; + let forceRefresh: boolean | undefined; + let unconfiguredRefreshes = 0; + const models = createModels(); + models.setProvider( + testProvider({ + id: "configured", + auth: { apiKey: envKeyAuth("ambient-key") }, + refreshModels: async (context) => { + if (!context.allowNetwork) return; + effectiveCredential = context.credential; + forceRefresh = context.force; + }, + }), + ); + models.setProvider( + testProvider({ + id: "unconfigured", + auth: { apiKey: envKeyAuth(undefined) }, + refreshModels: async ({ allowNetwork }) => { + if (allowNetwork) unconfiguredRefreshes++; + }, + }), + ); + + await models.refresh({ force: true }); + expect(effectiveCredential).toEqual({ type: "api_key", key: "ambient-key", env: undefined }); + expect(forceRefresh).toBe(true); + expect(unconfiguredRefreshes).toBe(0); + }); + + it("refreshes expired OAuth before refreshing models", async () => { + const credentials = new InMemoryCredentialStore(); + let modelRefreshCredential: unknown; + await credentials.modify("oauth-dynamic", async () => ({ + type: "oauth", + access: "expired", + refresh: "refresh", + expires: 0, + })); + const models = createModels({ credentials }); + models.setProvider( + testProvider({ + id: "oauth-dynamic", + auth: { + oauth: testOAuth({ + refresh: async () => ({ + type: "oauth", + access: "fresh", + refresh: "rotated", + expires: Date.now() + 60_000, + }), + }), + }, + refreshModels: async (context) => { + if (context.allowNetwork) modelRefreshCredential = context.credential; + }, + }), + ); + + expect((await models.refresh()).errors.size).toBe(0); + expect(modelRefreshCredential).toMatchObject({ type: "oauth", access: "fresh", refresh: "rotated" }); + expect(await credentials.read("oauth-dynamic")).toMatchObject({ access: "fresh", refresh: "rotated" }); + }); + + it("always gives providers a concrete signal", async () => { + let receivedSignal: AbortSignal | undefined; + const models = createModels(); + models.setProvider( + testProvider({ + id: "dynamic", + refreshModels: async ({ signal }) => { + receivedSignal = signal; + }, + }), + ); + + const result = await models.refresh(); + expect(result.aborted).toBe(false); + expect(receivedSignal).toBeInstanceOf(AbortSignal); + expect(receivedSignal?.aborted).toBe(false); + }); + + it("binds model-store waits to the provider refresh signal", async () => { + const storageSignals: (AbortSignal | undefined)[] = []; + const store: ModelsStore = { + read: async (_providerId, options) => { + storageSignals.push(options?.signal); + return undefined; + }, + write: async (_providerId, _entry, options) => { + storageSignals.push(options?.signal); + }, + delete: async (_providerId, options) => { + storageSignals.push(options?.signal); + }, + }; + let providerSignal: AbortSignal | undefined; + const models = createModels({ modelsStore: store }); + models.setProvider( + testProvider({ + id: "dynamic", + auth: { apiKey: envKeyAuth("key") }, + refreshModels: async (context) => { + providerSignal = context.signal; + if (!context.allowNetwork) return; + await context.publish({ persist: { models: [testModel("dynamic", "fresh")] } }); + }, + }), + ); + + const result = await models.refresh({ providers: ["dynamic"] }); + + expect(result.errors.size).toBe(0); + expect(storageSignals).toHaveLength(3); + expect(storageSignals.every((signal) => signal === providerSignal)).toBe(true); + }); + + it("returns aborted state without reporting cancellation as a provider error", async () => { + const controller = new AbortController(); + const models = createModels(); + models.setProvider( + testProvider({ + id: "dynamic", + refreshModels: async ({ signal }) => { + controller.abort(); + if (signal.aborted) return; + }, + }), + ); + + const result = await models.refresh({ signal: controller.signal }); + expect(result.aborted).toBe(true); + expect(result.errors.size).toBe(0); + }); + + it("stops waiting on abort when a provider ignores its signal", async () => { + const controller = new AbortController(); + let markStarted: (() => void) | undefined; + let rejectRefresh: ((error: Error) => void) | undefined; + const started = new Promise((resolve) => { + markStarted = resolve; + }); + const stalled = new Promise((_resolve, reject) => { + rejectRefresh = reject; + }); + let calls = 0; + const models = createModels(); + models.setProvider( + testProvider({ + id: "dynamic", + refreshModels: async () => { + calls++; + if (calls !== 1) return; + markStarted?.(); + await stalled; + }, + }), + ); + + const pending = models.refresh({ signal: controller.signal }); + await started; + controller.abort(); + + const result = await pending; + expect(result.aborted).toBe(true); + expect(result.errors.size).toBe(0); + + rejectRefresh?.(new Error("late provider failure")); + await new Promise((resolve) => setTimeout(resolve, 0)); + expect(result.errors.size).toBe(0); + }); + + it("rejects late publication from a superseded non-cooperative provider", async () => { + const store = new InMemoryModelsStore(); + let state = "initial"; + let calls = 0; + let markFirstStarted: (() => void) | undefined; + let finishFirst: (() => void) | undefined; + const firstStarted = new Promise((resolve) => { + markFirstStarted = resolve; + }); + const firstBlocked = new Promise((resolve) => { + finishFirst = resolve; + }); + const models = createModels({ modelsStore: store }); + models.setProvider( + testProvider({ + id: "dynamic", + refreshModels: async (context) => { + if (!context.allowNetwork) return; + calls++; + const current = calls; + if (current === 1) { + markFirstStarted?.(); + await firstBlocked; + } + const value = `generation-${current}`; + await context.publish({ + persist: { models: [testModel("dynamic", value)] }, + update: () => { + state = value; + }, + }); + }, + }), + ); + + const first = models.refresh({ providers: ["dynamic"] }); + await firstStarted; + const second = models.refresh({ providers: ["dynamic"] }); + await second; + await first; + finishFirst?.(); + await new Promise((resolve) => setTimeout(resolve, 0)); + + expect(state).toBe("generation-2"); + expect((await store.read("dynamic"))?.models[0]?.id).toBe("generation-2"); + }); + + it("passes caller signals to provider auth callbacks", async () => { + const controller = new AbortController(); + const received: AbortSignal[] = []; + const apiKey: ApiKeyAuth = { + name: "Signal auth", + login: async (interaction) => { + received.push(interaction.signal); + return { type: "api_key", key: "saved" }; + }, + check: async ({ signal }) => { + received.push(signal); + return { type: "api_key" }; + }, + resolve: async ({ signal }) => { + received.push(signal); + return { auth: { apiKey: "resolved" } }; + }, + }; + const models = createModels(); + models.setProvider(testProvider({ id: "p1", auth: { apiKey } })); + + await models.checkAuth("p1", { signal: controller.signal }); + await models.getAuth("p1", { signal: controller.signal }); + await models.login("p1", "api_key", { + signal: controller.signal, + prompt: async () => "unused", + notify: () => {}, + }); + + expect(received).toEqual([controller.signal, controller.signal, controller.signal]); + }); + + it("stops waiting for non-cooperative auth callbacks", async () => { + let startCheck: (() => void) | undefined; + let finishCheck: (() => void) | undefined; + const checkStarted = new Promise((resolve) => { + startCheck = resolve; + }); + const blockedCheck = new Promise((resolve) => { + finishCheck = resolve; + }); + let startResolve: (() => void) | undefined; + let finishResolve: (() => void) | undefined; + const resolveStarted = new Promise((resolve) => { + startResolve = resolve; + }); + const blockedResolve = new Promise((resolve) => { + finishResolve = resolve; + }); + const models = createModels(); + models.setProvider( + testProvider({ + id: "p1", + auth: { + apiKey: { + name: "Blocked auth", + check: async () => { + startCheck?.(); + await blockedCheck; + return { type: "api_key" }; + }, + resolve: async () => { + startResolve?.(); + await blockedResolve; + return { auth: { apiKey: "key" } }; + }, + }, + }, + }), + ); + + const availableController = new AbortController(); + const available = models.getAvailable(undefined, { signal: availableController.signal }); + await checkStarted; + availableController.abort(); + await expect(available).rejects.toMatchObject({ name: "AbortError" }); + + const authController = new AbortController(); + const auth = models.getAuth("p1", { signal: authController.signal }); + await resolveStarted; + authController.abort(); + await expect(auth).rejects.toMatchObject({ name: "AbortError" }); + + finishCheck?.(); + finishResolve?.(); + }); + + it("cancels queued credential mutations without running them later", async () => { + const credentials = new InMemoryCredentialStore(); + let finishFirst: (() => void) | undefined; + const firstBlocked = new Promise((resolve) => { + finishFirst = resolve; + }); + let secondRan = false; + const first = credentials.modify("p1", async () => { + await firstBlocked; + return { type: "api_key", key: "first" }; + }); + const controller = new AbortController(); + const second = credentials.modify( + "p1", + async () => { + secondRan = true; + return { type: "api_key", key: "second" }; + }, + { signal: controller.signal }, + ); + + controller.abort(); + await expect(second).rejects.toMatchObject({ name: "AbortError" }); + finishFirst?.(); + await first; + await new Promise((resolve) => setTimeout(resolve, 0)); + + expect(secondRan).toBe(false); + expect(await credentials.read("p1")).toEqual({ type: "api_key", key: "first" }); + }); + + it("passes cancellation to OAuth refresh and preserves the previous credential", async () => { + const credentials = new InMemoryCredentialStore(); + const previous: OAuthCredential = { type: "oauth", access: "old", refresh: "old-refresh", expires: 0 }; + await credentials.modify("p1", async () => previous); + let startRefresh: (() => void) | undefined; + let finishRefresh: ((credential: typeof previous) => void) | undefined; + const refreshStarted = new Promise((resolve) => { + startRefresh = resolve; + }); + const blockedRefresh = new Promise((resolve) => { + finishRefresh = resolve; + }); + let receivedSignal: AbortSignal | undefined; + const models = createModels({ credentials }); + models.setProvider( + testProvider({ + id: "p1", + auth: { + oauth: testOAuth({ + refresh: async (_credential, signal) => { + receivedSignal = signal; + startRefresh?.(); + return blockedRefresh; + }, + }), + }, + }), + ); + const controller = new AbortController(); + const auth = models.getAuth("p1", { signal: controller.signal }); + await refreshStarted; + controller.abort(); + + await expect(auth).rejects.toMatchObject({ name: "AbortError" }); + expect(receivedSignal).toBeInstanceOf(AbortSignal); + expect(receivedSignal?.aborted).toBe(true); + expect(receivedSignal?.reason).toBe(controller.signal.reason); + finishRefresh?.({ ...previous, access: "new", expires: Date.now() + 60_000 }); + await new Promise((resolve) => setTimeout(resolve, 0)); + expect(await credentials.read("p1")).toEqual(previous); + }); + + it("resolves auth: stored credential owns the provider, ambient only when nothing stored", async () => { + const credentials = new InMemoryCredentialStore(); + const models = createModels({ credentials }); + models.setProvider(testProvider({ id: "p1", auth: { apiKey: envKeyAuth("env-key"), oauth: testOAuth() } })); + const model = testModel("p1", "model-a"); + + // model and provider-id overloads resolve the same provider-scoped auth + expect((await models.getAuth(model))?.auth.apiKey).toBe("env-key"); + expect((await models.getAuth(model.provider))?.auth.apiKey).toBe("env-key"); + expect((await models.getAuth(model, { apiKey: "explicit-key" }))?.auth.apiKey).toBe("explicit-key"); + + // stored oauth credential (persisted via the single write path): beats ambient env + await credentials.modify("p1", async () => ({ + type: "oauth", + access: "oauth-token", + refresh: "r", + expires: Date.now() + 10 * 60_000, + })); + const resolution = await models.getAuth(model.provider); + expect(resolution?.auth.apiKey).toBe("oauth-token"); + expect(resolution?.source).toBe("OAuth"); + + // stored api-key credential resolves through apiKey auth, beats env + await credentials.modify("p1", async () => ({ type: "api_key", key: "stored-key" })); + const apiKeyResolution = await models.getAuth(model.provider); + expect(apiKeyResolution?.auth.apiKey).toBe("stored-key"); + expect(apiKeyResolution?.source).toBe("stored"); + }); + + it("checks provider auth without refreshing OAuth and filters available models", async () => { + const credentials = new InMemoryCredentialStore(); + let refreshes = 0; + const models = createModels({ credentials }); + models.setProvider(testProvider({ id: "ambient", auth: { apiKey: envKeyAuth("env-key") } })); + models.setProvider(testProvider({ id: "missing", auth: { apiKey: envKeyAuth(undefined) } })); + models.setProvider( + testProvider({ + id: "oauth", + auth: { + oauth: testOAuth({ + refresh: async (credential) => { + refreshes++; + return credential; + }, + }), + }, + }), + ); + await credentials.modify("oauth", async () => ({ + type: "oauth", + access: "expired", + refresh: "refresh", + expires: 0, + })); + + expect(await models.checkAuth("ambient")).toEqual({ source: "env", type: "api_key" }); + expect(await models.checkAuth("missing")).toBeUndefined(); + expect(await models.checkAuth("oauth")).toEqual({ source: "OAuth", type: "oauth" }); + expect(refreshes).toBe(0); + expect((await models.getAvailable()).map((model) => model.provider)).toEqual(["ambient", "oauth"]); + expect((await models.getAvailable("ambient")).map((model) => model.provider)).toEqual(["ambient"]); + }); + + it("runs provider login and logout through the credential store", async () => { + const credentials = new InMemoryCredentialStore(); + const apiKey = envKeyAuth(undefined); + apiKey.login = async () => ({ type: "api_key", key: "logged-in" }); + const models = createModels({ credentials }); + models.setProvider(testProvider({ id: "p1", auth: { apiKey } })); + + const credential = await models.login("p1", "api_key", { + prompt: async () => "unused", + notify: () => {}, + }); + expect(credential).toEqual({ type: "api_key", key: "logged-in" }); + expect(await credentials.read("p1")).toEqual(credential); + + await models.logout("p1"); + expect(await credentials.read("p1")).toBeUndefined(); + }); + + it("a stored credential without a matching handler blocks ambient fallback", async () => { + const credentials = new InMemoryCredentialStore(); + const models = createModels({ credentials }); + // provider has only apiKey auth, but an oauth credential is stored (stale config) + models.setProvider(testProvider({ id: "p1", auth: { apiKey: envKeyAuth("env-key") } })); + await credentials.modify("p1", async () => ({ type: "oauth", access: "a", refresh: "r", expires: 0 })); + + expect(await models.getAuth("p1")).toBeUndefined(); + }); + + it("refreshes expired oauth credentials and persists the rotated credential", async () => { + const credentials = new InMemoryCredentialStore(); + const oauth = testOAuth({ + refresh: async (credential) => ({ ...credential, access: "new-token", expires: Date.now() + 60 * 60_000 }), + }); + const models = createModels({ credentials }); + models.setProvider(testProvider({ id: "p1", auth: { oauth } })); + await credentials.modify("p1", async () => ({ + type: "oauth", + access: "old-token", + refresh: "r", + expires: 0, + })); + + const resolution = await models.getAuth("p1"); + expect(resolution?.auth.apiKey).toBe("new-token"); + expect(((await credentials.read("p1")) as { access: string }).access).toBe("new-token"); + }); + + it("refreshes oauth credentials with less than five minutes remaining", async () => { + const credentials = new InMemoryCredentialStore(); + const refresh = vi.fn(async (credential) => ({ + ...credential, + access: "new-token", + expires: Date.now() + 60 * 60_000, + })); + const models = createModels({ credentials }); + models.setProvider(testProvider({ id: "p1", auth: { oauth: testOAuth({ refresh }) } })); + await credentials.modify("p1", async () => ({ + type: "oauth", + access: "old-token", + refresh: "r", + expires: Date.now() + 60_000, + })); + + expect((await models.getAuth("p1"))?.auth.apiKey).toBe("new-token"); + expect(refresh).toHaveBeenCalledOnce(); + }); + + it("honors a caller's longer OAuth minimum validity", async () => { + const credentials = new InMemoryCredentialStore(); + const refresh = vi.fn(async (credential) => ({ + ...credential, + access: "new-token", + expires: Date.now() + 60 * 60_000, + })); + const models = createModels({ credentials }); + models.setProvider(testProvider({ id: "p1", auth: { oauth: testOAuth({ refresh }) } })); + await credentials.modify("p1", async () => ({ + type: "oauth", + access: "old-token", + refresh: "r", + expires: Date.now() + 10 * 60_000, + })); + + expect((await models.getAuth("p1", { minOAuthValidityMs: 30 * 60_000 }))?.auth.apiKey).toBe("new-token"); + expect(refresh).toHaveBeenCalledOnce(); + }); + + it("rejects with code oauth when refresh fails, preserving the stored credential", async () => { + const credentials = new InMemoryCredentialStore(); + const oauth = testOAuth({ + refresh: async () => { + throw new Error("invalid_grant"); + }, + }); + const models = createModels({ credentials }); + models.setProvider(testProvider({ id: "p1", auth: { oauth } })); + await credentials.modify("p1", async () => ({ type: "oauth", access: "old", refresh: "r", expires: 0 })); + + await expect(models.getAuth("p1")).rejects.toMatchObject({ code: "oauth" }); + // credential preserved for retry / re-login + expect(((await credentials.read("p1")) as { access: string }).access).toBe("old"); + }); + + it("serializes concurrent OAuth refreshes through store.modify (no double refresh)", async () => { + const credentials = new InMemoryCredentialStore(); + await credentials.modify("p1", async () => ({ type: "oauth", access: "old", refresh: "r1", expires: 0 })); + + let refreshes = 0; + const oauth = testOAuth({ + refresh: async () => { + refreshes++; + await new Promise((resolve) => setTimeout(resolve, 10)); + return { type: "oauth", access: `new-${refreshes}`, refresh: "r2", expires: Date.now() + 60 * 60_000 }; + }, + }); + const models = createModels({ credentials }); + models.setProvider(testProvider({ id: "p1", auth: { oauth } })); + const model = testModel("p1", "model-a"); + + const [a, b] = await Promise.all([models.getAuth(model.provider), models.getAuth(model.provider)]); + expect(refreshes).toBe(1); + expect(a?.auth.apiKey).toBe("new-1"); + expect(b?.auth.apiKey).toBe("new-1"); + }); + + it("valid oauth tokens resolve without touching modify", async () => { + let modifies = 0; + const base = new InMemoryCredentialStore(); + const credentials: CredentialStore = { + read: (pid) => base.read(pid), + list: () => base.list(), + modify: (pid, fn) => { + modifies++; + return base.modify(pid, fn); + }, + delete: (pid) => base.delete(pid), + }; + await base.modify("p1", async () => ({ + type: "oauth", + access: "valid", + refresh: "r", + expires: Date.now() + 10 * 60_000, + })); + const models = createModels({ credentials }); + models.setProvider(testProvider({ id: "p1", auth: { oauth: testOAuth() } })); + + expect((await models.getAuth("p1"))?.auth.apiKey).toBe("valid"); + expect(modifies).toBe(0); + }); + + it("wraps credential store failures in ModelsError", async () => { + // read failure + const readFailing: CredentialStore = { + read: async () => { + throw new Error("disk on fire"); + }, + list: async () => [], + modify: async () => undefined, + delete: async () => {}, + }; + const models = createModels({ credentials: readFailing }); + models.setProvider(testProvider({ id: "p1", auth: { apiKey: envKeyAuth("env-key") } })); + await expect(models.getAuth("p1")).rejects.toMatchObject({ code: "auth" }); + + // modify failure during refresh + const modifyFailing: CredentialStore = { + read: async () => ({ type: "oauth", access: "old", refresh: "r", expires: 0 }), + list: async () => [{ providerId: "p1", type: "oauth" }], + modify: async () => { + throw new Error("disk on fire"); + }, + delete: async () => {}, + }; + const oauthModels = createModels({ credentials: modifyFailing }); + oauthModels.setProvider(testProvider({ id: "p1", auth: { oauth: testOAuth() } })); + await expect(oauthModels.getAuth("p1")).rejects.toMatchObject({ code: "auth" }); + }); + + it("keeps the underlying reason in wrapped oauth refresh errors", async () => { + const credentials = new InMemoryCredentialStore(); + await credentials.modify("p1", async () => ({ type: "oauth", access: "old", refresh: "r", expires: 0 })); + const models = createModels({ credentials }); + models.setProvider( + testProvider({ + id: "p1", + auth: { + oauth: testOAuth({ + refresh: async () => { + throw new Error("token refresh failed (400): invalid_grant"); + }, + }), + }, + }), + ); + + await expect(models.getAuth("p1")).rejects.toThrow( + "OAuth refresh failed for p1: token refresh failed (400): invalid_grant", + ); + }); + + it("wraps api-key auth failures in ModelsError", async () => { + const failing: ApiKeyAuth = { + name: "Failing", + resolve: async () => { + throw new Error("nope"); + }, + }; + const models = createModels(); + models.setProvider(testProvider({ id: "p1", auth: { apiKey: failing } })); + await expect(models.getAuth("p1")).rejects.toMatchObject({ code: "auth" }); + }); + + it("uses explicit request api key and env during provider auth resolution", async () => { + const calls: ProviderCall[] = []; + const apiKey: ApiKeyAuth = { + name: "Scoped", + resolve: async ({ credential, ctx }) => { + const account = credential?.env?.ACCOUNT_ID ?? (await ctx.env("ACCOUNT_ID")); + if (!credential?.key || !account) return undefined; + return { + auth: { apiKey: credential.key, baseUrl: `https://example.test/${account}` }, + env: { ACCOUNT_ID: account }, + }; + }, + }; + const models = createModels(); + models.setProvider(testProvider({ id: "p1", auth: { apiKey }, calls })); + const model = testModel("p1", "model-a"); + + await models.completeSimple(model, context, { apiKey: "explicit-key", env: { ACCOUNT_ID: "acct" } }); + + expect(calls[0].model.baseUrl).toBe("https://example.test/acct"); + expect(calls[0].options?.apiKey).toBe("explicit-key"); + expect(calls[0].options?.env).toEqual({ ACCOUNT_ID: "acct" }); + }); + + it("merges resolved auth into stream options; explicit options win per field", async () => { + const calls: ProviderCall[] = []; + const apiKey: ApiKeyAuth = { + name: "Test", + resolve: async () => ({ + auth: { + apiKey: "resolved-key", + headers: { Authorization: "Bearer resolved-key", "x-a": "auth", "x-b": "auth" }, + baseUrl: "https://auth.test/v1", + }, + }), + }; + const models = createModels(); + models.setProvider(testProvider({ id: "p1", auth: { apiKey }, calls })); + const model = testModel("p1", "model-a"); + + const result = await models.completeSimple(model, context, { + apiKey: "explicit-key", + headers: { authorization: "Explicit token", "x-b": "explicit" }, + }); + expect(result.stopReason).toBe("stop"); + expect(calls).toHaveLength(1); + expect(calls[0].options?.apiKey).toBe("explicit-key"); + expect(calls[0].options?.headers).toEqual({ authorization: "Explicit token", "x-a": "auth", "x-b": "explicit" }); + expect(calls[0].model.baseUrl).toBe("https://auth.test/v1"); + + // without explicit options, resolved auth applies + const result2 = await models.completeSimple(model, context); + expect(result2.stopReason).toBe("stop"); + expect(calls[1].options?.apiKey).toBe("resolved-key"); + }); + + it("adds model headers only for model auth and transforms assembled headers once", async () => { + const calls: ProviderCall[] = []; + const models = createModels(); + models.setProvider(testProvider({ id: "p1", auth: { apiKey: envKeyAuth("key") }, calls })); + const model = testModel("p1", "model-a"); + model.headers = { "x-model": "model", "x-shared": "model" }; + + expect((await models.getAuth("p1"))?.auth.headers).toBeUndefined(); + expect((await models.getAuth(model))?.auth.headers).toEqual({ "x-model": "model", "x-shared": "model" }); + + let transforms = 0; + await models.completeSimple(model, context, { + headers: { "x-explicit": "explicit", "X-Shared": "explicit" }, + transformHeaders: async (headers) => { + transforms++; + expect(headers).toEqual({ "x-model": "model", "x-explicit": "explicit", "X-Shared": "explicit" }); + return { ...headers, "x-transformed": "yes" }; + }, + }); + + expect(transforms).toBe(1); + expect(calls[0].options?.headers).toEqual({ + "x-model": "model", + "x-explicit": "explicit", + "X-Shared": "explicit", + "x-transformed": "yes", + }); + expect(calls[0].options).not.toHaveProperty("transformHeaders"); + }); + + it("produces an error stream for unknown providers instead of throwing", async () => { + const models = createModels(); + const result = await models.completeSimple(testModel("ghost", "model-a"), context); + expect(result.stopReason).toBe("error"); + expect(result.errorMessage).toContain("Unknown provider: ghost"); + }); + + it("streams through the provider", async () => { + const models = createModels(); + models.setProvider(testProvider({ id: "p1" })); + const model = testModel("p1", "model-a"); + + const events: string[] = []; + const stream = models.streamSimple(model, context); + for await (const event of stream) { + events.push(event.type); + } + expect(events).toEqual(["start", "done"]); + const message = await stream.result(); + expect(message.stopReason).toBe("stop"); + }); +}); diff --git a/packages/ai/test/node-http-proxy.test.ts b/packages/ai/test/node-http-proxy.test.ts new file mode 100644 index 0000000000000000000000000000000000000000..5ed530e2fa240c33691e1f0ed867ec076ba4534e --- /dev/null +++ b/packages/ai/test/node-http-proxy.test.ts @@ -0,0 +1,95 @@ +import { afterEach, describe, expect, it } from "vitest"; +import { resolveHttpProxyUrlForTarget, UNSUPPORTED_PROXY_PROTOCOL_MESSAGE } from "../src/utils/node-http-proxy.ts"; + +const PROXY_ENV_KEYS = [ + "HTTP_PROXY", + "HTTPS_PROXY", + "NO_PROXY", + "ALL_PROXY", + "http_proxy", + "https_proxy", + "no_proxy", + "all_proxy", + "npm_config_http_proxy", + "npm_config_https_proxy", + "npm_config_proxy", + "npm_config_no_proxy", +] as const; + +const originalEnv = new Map(); +for (const key of PROXY_ENV_KEYS) { + originalEnv.set(key, process.env[key]); +} + +function resetProxyEnv(): void { + for (const key of PROXY_ENV_KEYS) { + delete process.env[key]; + } +} + +afterEach(() => { + resetProxyEnv(); + for (const [key, value] of originalEnv) { + if (value !== undefined) { + process.env[key] = value; + } + } +}); + +describe("node HTTP proxy resolution", () => { + it("respects NO_PROXY exclusions", () => { + resetProxyEnv(); + process.env.HTTPS_PROXY = "http://proxy.example:8080"; + process.env.NO_PROXY = "bedrock-runtime.us-east-1.amazonaws.com"; + + expect(resolveHttpProxyUrlForTarget("https://bedrock-runtime.us-east-1.amazonaws.com")).toBeUndefined(); + }); + + it("resolves HTTP and HTTPS proxy URLs", () => { + resetProxyEnv(); + process.env.HTTPS_PROXY = "http://proxy.example:8080"; + + expect(resolveHttpProxyUrlForTarget("https://bedrock-runtime.us-east-1.amazonaws.com")?.toString()).toBe( + "http://proxy.example:8080/", + ); + }); + + it("prefers scoped proxy env aliases before process env aliases", () => { + resetProxyEnv(); + process.env.https_proxy = "http://process-proxy.example:8080"; + + expect( + resolveHttpProxyUrlForTarget("https://bedrock-runtime.us-east-1.amazonaws.com", { + HTTPS_PROXY: "http://scoped-proxy.example:8080", + })?.toString(), + ).toBe("http://scoped-proxy.example:8080/"); + }); + + it("rejects SOCKS and PAC proxy URLs explicitly", () => { + resetProxyEnv(); + process.env.HTTPS_PROXY = "socks5://proxy.example:1080"; + + expect(() => resolveHttpProxyUrlForTarget("https://bedrock-runtime.us-east-1.amazonaws.com")).toThrow( + UNSUPPORTED_PROXY_PROTOCOL_MESSAGE, + ); + }); + + it("handles subdomain wildcards, IPv6, and ports in NO_PROXY", () => { + resetProxyEnv(); + process.env.HTTPS_PROXY = "http://proxy.example:8080"; + process.env.NO_PROXY = "example.com, .wildcard.org, *.star.net, ::1, [2001:db8::1], 127.0.0.1:8080"; + + expect(resolveHttpProxyUrlForTarget("https://example.com")).toBeUndefined(); + expect(resolveHttpProxyUrlForTarget("https://api.example.com")).toBeUndefined(); + expect(resolveHttpProxyUrlForTarget("https://wildcard.org")).toBeUndefined(); + expect(resolveHttpProxyUrlForTarget("https://api.wildcard.org")).toBeUndefined(); + expect(resolveHttpProxyUrlForTarget("https://star.net")).toBeUndefined(); + expect(resolveHttpProxyUrlForTarget("https://api.star.net")).toBeUndefined(); + expect(resolveHttpProxyUrlForTarget("https://notexample.com")?.toString()).toBe("http://proxy.example:8080/"); + + expect(resolveHttpProxyUrlForTarget("https://[::1]:80")).toBeUndefined(); + expect(resolveHttpProxyUrlForTarget("https://[2001:db8::1]")).toBeUndefined(); + expect(resolveHttpProxyUrlForTarget("https://127.0.0.1:8080")).toBeUndefined(); + expect(resolveHttpProxyUrlForTarget("https://127.0.0.1:3000")?.toString()).toBe("http://proxy.example:8080/"); + }); +}); diff --git a/packages/ai/test/oauth-device-code.test.ts b/packages/ai/test/oauth-device-code.test.ts new file mode 100644 index 0000000000000000000000000000000000000000..2bda9f5e60b64525f31f5f87aef72bb2ae300956 --- /dev/null +++ b/packages/ai/test/oauth-device-code.test.ts @@ -0,0 +1,145 @@ +import { afterEach, describe, expect, it, vi } from "vitest"; +import { pollOAuthDeviceCodeFlow } from "../src/auth/oauth/device-code.ts"; + +const neverAbortedSignal = new AbortController().signal; + +describe("OAuth device-code polling", () => { + afterEach(() => { + vi.useRealTimers(); + }); + + it("polls immediately and returns the completed value", async () => { + vi.useFakeTimers(); + vi.setSystemTime(new Date("2026-03-09T00:00:00Z")); + + const pollTimes: number[] = []; + const poll = vi.fn(async () => { + pollTimes.push(Date.now()); + return pollTimes.length === 1 + ? { status: "pending" as const } + : { status: "complete" as const, value: "token" }; + }); + + const resultPromise = pollOAuthDeviceCodeFlow({ + intervalSeconds: 2, + expiresInSeconds: 30, + poll, + signal: neverAbortedSignal, + }); + + await vi.advanceTimersByTimeAsync(0); + expect(pollTimes).toEqual([new Date("2026-03-09T00:00:00Z").getTime()]); + + await vi.advanceTimersByTimeAsync(1999); + expect(pollTimes).toEqual([new Date("2026-03-09T00:00:00Z").getTime()]); + + await vi.advanceTimersByTimeAsync(1); + await expect(resultPromise).resolves.toBe("token"); + expect(pollTimes).toEqual([ + new Date("2026-03-09T00:00:00Z").getTime(), + new Date("2026-03-09T00:00:02Z").getTime(), + ]); + }); + + it("can wait before the first poll", async () => { + vi.useFakeTimers(); + vi.setSystemTime(new Date("2026-03-09T00:00:00Z")); + + const pollTimes: number[] = []; + const resultPromise = pollOAuthDeviceCodeFlow({ + intervalSeconds: 2, + expiresInSeconds: 30, + waitBeforeFirstPoll: true, + poll: async () => { + pollTimes.push(Date.now()); + return { status: "complete" as const, value: "token" }; + }, + signal: neverAbortedSignal, + }); + + await vi.advanceTimersByTimeAsync(1999); + expect(pollTimes).toEqual([]); + + await vi.advanceTimersByTimeAsync(1); + await expect(resultPromise).resolves.toBe("token"); + expect(pollTimes).toEqual([new Date("2026-03-09T00:00:02Z").getTime()]); + }); + + it("increases the interval by 5 seconds after slow_down without a server interval", async () => { + vi.useFakeTimers(); + vi.setSystemTime(new Date("2026-03-09T00:00:00Z")); + const startTime = Date.now(); + + const pollTimes: number[] = []; + const results = [{ status: "slow_down" as const }, { status: "complete" as const, value: "token" }]; + const resultPromise = pollOAuthDeviceCodeFlow({ + intervalSeconds: 2, + expiresInSeconds: 900, + poll: async () => { + pollTimes.push(Date.now()); + const result = results.shift(); + if (!result) throw new Error("Unexpected extra poll"); + return result; + }, + signal: neverAbortedSignal, + }); + + await vi.advanceTimersByTimeAsync(0); + expect(pollTimes).toEqual([startTime]); + + await vi.advanceTimersByTimeAsync(6999); + expect(pollTimes).toEqual([startTime]); + + await vi.advanceTimersByTimeAsync(1); + await expect(resultPromise).resolves.toBe("token"); + expect(pollTimes).toEqual([startTime, startTime + 7000]); + }); + + it("honors a server-provided slow_down interval", async () => { + vi.useFakeTimers(); + vi.setSystemTime(new Date("2026-03-09T00:00:00Z")); + const startTime = Date.now(); + + const pollTimes: number[] = []; + const results = [ + { status: "slow_down" as const, intervalSeconds: 30 }, + { status: "complete" as const, value: "token" }, + ]; + const resultPromise = pollOAuthDeviceCodeFlow({ + intervalSeconds: 2, + expiresInSeconds: 900, + poll: async () => { + pollTimes.push(Date.now()); + const result = results.shift(); + if (!result) throw new Error("Unexpected extra poll"); + return result; + }, + signal: neverAbortedSignal, + }); + + await vi.advanceTimersByTimeAsync(0); + expect(pollTimes).toEqual([startTime]); + + await vi.advanceTimersByTimeAsync(29999); + expect(pollTimes).toEqual([startTime]); + + await vi.advanceTimersByTimeAsync(1); + await expect(resultPromise).resolves.toBe("token"); + expect(pollTimes).toEqual([startTime, startTime + 30000]); + }); + + it("cancels an in-flight wait", async () => { + vi.useFakeTimers(); + const controller = new AbortController(); + + const resultPromise = pollOAuthDeviceCodeFlow({ + intervalSeconds: 5, + expiresInSeconds: 30, + poll: async () => ({ status: "pending" }), + signal: controller.signal, + }); + + controller.abort(); + await expect(resultPromise).rejects.toThrow("Login cancelled"); + }); +}); diff --git a/packages/ai/test/openai-codex-oauth.test.ts b/packages/ai/test/openai-codex-oauth.test.ts new file mode 100644 index 0000000000000000000000000000000000000000..f45d4aeea5e804fba2abf926f141c1519d3429d3 --- /dev/null +++ b/packages/ai/test/openai-codex-oauth.test.ts @@ -0,0 +1,486 @@ +import { afterEach, describe, expect, it, vi } from "vitest"; +import { openaiCodexOAuth } from "../src/auth/oauth/openai-codex.ts"; + +const neverAbortedSignal = new AbortController().signal; + +function jsonResponse(body: unknown, status: number = 200): Response { + return new Response(JSON.stringify(body), { + status, + headers: { "Content-Type": "application/json" }, + }); +} + +function getUrl(input: unknown): string { + if (typeof input === "string") return input; + if (input instanceof URL) return input.toString(); + if (input instanceof Request) return input.url; + throw new Error(`Unsupported fetch input: ${String(input)}`); +} + +function createAccessToken(accountId: string): string { + const header = Buffer.from(JSON.stringify({ alg: "none" })).toString("base64"); + const payload = Buffer.from( + JSON.stringify({ + "https://api.openai.com/auth": { + chatgpt_account_id: accountId, + }, + }), + ).toString("base64"); + return `${header}.${payload}.signature`; +} + +function deviceAuthPendingResponse(): Response { + return jsonResponse( + { + error: { + message: "Device authorization is pending. Please try again.", + type: "invalid_request_error", + param: null, + code: "deviceauth_authorization_pending", + }, + }, + 403, + ); +} + +function loginOpenAICodexDeviceCodeForTest(options: { + onDeviceCode(info: { + userCode: string; + verificationUri: string; + intervalSeconds?: number; + expiresInSeconds?: number; + }): void; + signal?: AbortSignal; +}) { + return openaiCodexOAuth.login({ + signal: options.signal ?? neverAbortedSignal, + prompt: async (prompt) => { + if (prompt.type !== "select") throw new Error(`Unexpected prompt: ${prompt.type}`); + return "device_code"; + }, + notify: (event) => { + if (event.type === "device_code") { + const { type: _, ...info } = event; + options.onDeviceCode(info); + } + }, + }); +} + +describe("OpenAI Codex OAuth", () => { + afterEach(() => { + vi.restoreAllMocks(); + vi.unstubAllGlobals(); + vi.useRealTimers(); + }); + + it("logs in with the OpenAI Codex device code flow", async () => { + vi.useFakeTimers(); + const startTime = new Date("2026-05-20T00:00:00Z"); + vi.setSystemTime(startTime); + + const accessToken = createAccessToken("account-123"); + const deviceInfos: Array<{ + userCode: string; + verificationUri: string; + instructions?: string; + intervalSeconds?: number; + expiresInSeconds?: number; + }> = []; + const pollTimes: number[] = []; + const pollResponses = [ + deviceAuthPendingResponse(), + jsonResponse({ + authorization_code: "oauth-code", + code_challenge: "device-code-challenge", + code_verifier: "device-code-verifier", + }), + ]; + + const fetchMock = vi.fn(async (input: unknown, init?: RequestInit): Promise => { + const url = getUrl(input); + + if (url === "https://auth.openai.com/api/accounts/deviceauth/usercode") { + expect(init?.method).toBe("POST"); + expect(init?.headers).toMatchObject({ "Content-Type": "application/json" }); + expect(JSON.parse(String(init?.body))).toEqual({ client_id: "app_EMoamEEZ73f0CkXaXp7hrann" }); + return jsonResponse({ + device_auth_id: "device-auth-id", + user_code: "ABCD-1234", + interval: "5", + }); + } + + if (url === "https://auth.openai.com/api/accounts/deviceauth/token") { + pollTimes.push(Date.now()); + expect(init?.method).toBe("POST"); + expect(init?.headers).toMatchObject({ "Content-Type": "application/json" }); + expect(JSON.parse(String(init?.body))).toEqual({ + device_auth_id: "device-auth-id", + user_code: "ABCD-1234", + }); + const response = pollResponses.shift(); + if (!response) { + throw new Error("Unexpected extra device auth poll"); + } + return response; + } + + if (url === "https://auth.openai.com/oauth/token") { + expect(init?.method).toBe("POST"); + expect(init?.headers).toMatchObject({ "Content-Type": "application/x-www-form-urlencoded" }); + const params = new URLSearchParams(String(init?.body)); + expect(params.get("grant_type")).toBe("authorization_code"); + expect(params.get("client_id")).toBe("app_EMoamEEZ73f0CkXaXp7hrann"); + expect(params.get("code")).toBe("oauth-code"); + expect(params.get("redirect_uri")).toBe("https://auth.openai.com/deviceauth/callback"); + expect(params.get("code_verifier")).toBe("device-code-verifier"); + return jsonResponse({ + access_token: accessToken, + refresh_token: "refresh-token", + expires_in: 3600, + }); + } + + throw new Error(`Unexpected fetch URL: ${url}`); + }); + + vi.stubGlobal("fetch", fetchMock); + + const credentialsPromise = loginOpenAICodexDeviceCodeForTest({ + onDeviceCode: (info) => deviceInfos.push(info), + }); + + for (let i = 0; i < 5 && pollTimes.length === 0; i++) { + await vi.advanceTimersByTimeAsync(0); + } + expect(deviceInfos).toEqual([ + { + userCode: "ABCD-1234", + verificationUri: "https://auth.openai.com/codex/device", + intervalSeconds: 5, + expiresInSeconds: 900, + }, + ]); + expect(pollTimes).toEqual([startTime.getTime()]); + + await vi.advanceTimersByTimeAsync(4999); + expect(pollTimes).toEqual([startTime.getTime()]); + + await vi.advanceTimersByTimeAsync(1); + await expect(credentialsPromise).resolves.toMatchObject({ + access: accessToken, + refresh: "refresh-token", + expires: startTime.getTime() + 5000 + 3600 * 1000, + accountId: "account-123", + }); + expect(pollTimes).toEqual([startTime.getTime(), startTime.getTime() + 5000]); + }); + + it("offers browser login first and uses the selected OpenAI Codex device code flow", async () => { + const accessToken = createAccessToken("account-456"); + const selectPrompts: Array<{ + message: string; + options: readonly { id: string; label: string }[]; + }> = []; + const deviceInfos: Array<{ + userCode: string; + verificationUri: string; + intervalSeconds?: number; + expiresInSeconds?: number; + }> = []; + + vi.stubGlobal( + "fetch", + vi.fn(async (input: unknown, init?: RequestInit): Promise => { + const url = getUrl(input); + if (url === "https://auth.openai.com/api/accounts/deviceauth/usercode") { + expect(JSON.parse(String(init?.body))).toEqual({ client_id: "app_EMoamEEZ73f0CkXaXp7hrann" }); + return jsonResponse({ + device_auth_id: "device-auth-id", + user_code: "WXYZ-7890", + interval: "5", + }); + } + if (url === "https://auth.openai.com/api/accounts/deviceauth/token") { + return jsonResponse({ + authorization_code: "oauth-code", + code_challenge: "device-code-challenge", + code_verifier: "device-code-verifier", + }); + } + if (url === "https://auth.openai.com/oauth/token") { + return jsonResponse({ + access_token: accessToken, + refresh_token: "refresh-token", + expires_in: 3600, + }); + } + throw new Error(`Unexpected fetch URL: ${url}`); + }), + ); + + await expect( + openaiCodexOAuth.login({ + signal: neverAbortedSignal, + prompt: async (prompt) => { + if (prompt.type !== "select") throw new Error("Text prompt should not be used"); + selectPrompts.push(prompt); + return "device_code"; + }, + notify: (event) => { + if (event.type === "auth_url") throw new Error("Browser login should not start"); + if (event.type === "device_code") { + const { type: _, ...info } = event; + deviceInfos.push(info); + } + }, + }), + ).resolves.toMatchObject({ + type: "oauth", + access: accessToken, + refresh: "refresh-token", + accountId: "account-456", + }); + + expect(selectPrompts).toEqual([ + { + type: "select", + message: "Select OpenAI Codex login method:", + options: [ + { id: "browser", label: "Browser login (default)" }, + { id: "device_code", label: "Device code login (headless)" }, + ], + }, + ]); + expect(deviceInfos).toEqual([ + { + userCode: "WXYZ-7890", + verificationUri: "https://auth.openai.com/codex/device", + intervalSeconds: 5, + expiresInSeconds: 900, + }, + ]); + }); + + it("cancels when OpenAI Codex login method selection is cancelled", async () => { + await expect( + openaiCodexOAuth.login({ + signal: neverAbortedSignal, + prompt: async () => { + throw new Error("Login cancelled"); + }, + notify: () => {}, + }), + ).rejects.toThrow("Login cancelled"); + }); + + it("cancels the OpenAI Codex device code flow while waiting", async () => { + vi.useFakeTimers(); + const controller = new AbortController(); + const pollTimes: number[] = []; + + vi.stubGlobal( + "fetch", + vi.fn(async (input: unknown, init?: RequestInit): Promise => { + const url = getUrl(input); + if (url === "https://auth.openai.com/api/accounts/deviceauth/usercode") { + expect(JSON.parse(String(init?.body))).toEqual({ client_id: "app_EMoamEEZ73f0CkXaXp7hrann" }); + return jsonResponse({ + device_auth_id: "device-auth-id", + user_code: "ABCD-1234", + interval: "5", + }); + } + if (url === "https://auth.openai.com/api/accounts/deviceauth/token") { + pollTimes.push(Date.now()); + return deviceAuthPendingResponse(); + } + throw new Error(`Unexpected fetch URL: ${url}`); + }), + ); + + const credentialsPromise = loginOpenAICodexDeviceCodeForTest({ + onDeviceCode: () => {}, + signal: controller.signal, + }); + const rejectionPromise = credentialsPromise.then( + () => new Error("Expected login to fail"), + (error: unknown) => error, + ); + + for (let i = 0; i < 5 && pollTimes.length === 0; i++) { + await vi.advanceTimersByTimeAsync(0); + } + expect(pollTimes).toHaveLength(1); + + controller.abort(); + const rejection = await rejectionPromise; + expect(rejection).toBeInstanceOf(Error); + expect((rejection as Error).message).toBe("Login cancelled"); + }); + + it("times out the OpenAI Codex device code flow after 15 minutes", async () => { + vi.useFakeTimers(); + const pollTimes: number[] = []; + + vi.stubGlobal( + "fetch", + vi.fn(async (input: unknown, init?: RequestInit): Promise => { + const url = getUrl(input); + if (url === "https://auth.openai.com/api/accounts/deviceauth/usercode") { + expect(JSON.parse(String(init?.body))).toEqual({ client_id: "app_EMoamEEZ73f0CkXaXp7hrann" }); + return jsonResponse({ + device_auth_id: "device-auth-id", + user_code: "ABCD-1234", + interval: "60", + }); + } + if (url === "https://auth.openai.com/api/accounts/deviceauth/token") { + pollTimes.push(Date.now()); + return deviceAuthPendingResponse(); + } + throw new Error(`Unexpected fetch URL: ${url}`); + }), + ); + + const credentialsPromise = loginOpenAICodexDeviceCodeForTest({ + onDeviceCode: () => {}, + }); + const rejectionPromise = credentialsPromise.then( + () => new Error("Expected login to fail"), + (error: unknown) => error, + ); + + for (let i = 0; i < 5 && pollTimes.length === 0; i++) { + await vi.advanceTimersByTimeAsync(0); + } + expect(pollTimes).toHaveLength(1); + + await vi.advanceTimersByTimeAsync(15 * 60 * 1000); + const rejection = await rejectionPromise; + expect(rejection).toBeInstanceOf(Error); + expect((rejection as Error).message).toBe("Device flow timed out"); + }); + + it("treats OpenAI Codex device auth 403 and 404 responses as pending", async () => { + vi.useFakeTimers(); + const accessToken = createAccessToken("account-403-404"); + const pollTimes: number[] = []; + const pollResponses = [ + jsonResponse({ error: "access_denied", error_description: "denied" }, 403), + new Response("not ready", { status: 404, headers: { "Content-Type": "text/plain" } }), + jsonResponse({ + authorization_code: "oauth-code", + code_challenge: "device-code-challenge", + code_verifier: "device-code-verifier", + }), + ]; + + vi.stubGlobal( + "fetch", + vi.fn(async (input: unknown): Promise => { + const url = getUrl(input); + if (url === "https://auth.openai.com/api/accounts/deviceauth/usercode") { + return jsonResponse({ + device_auth_id: "device-auth-id", + user_code: "ABCD-1234", + interval: "1", + }); + } + if (url === "https://auth.openai.com/api/accounts/deviceauth/token") { + pollTimes.push(Date.now()); + const response = pollResponses.shift(); + if (!response) { + throw new Error("Unexpected extra device auth poll"); + } + return response; + } + if (url === "https://auth.openai.com/oauth/token") { + return jsonResponse({ + access_token: accessToken, + refresh_token: "refresh-token", + expires_in: 3600, + }); + } + throw new Error(`Unexpected fetch URL: ${url}`); + }), + ); + + const credentialsPromise = loginOpenAICodexDeviceCodeForTest({ + onDeviceCode: () => {}, + }); + + for (let i = 0; i < 5 && pollTimes.length === 0; i++) { + await vi.advanceTimersByTimeAsync(0); + } + await vi.advanceTimersByTimeAsync(1000); + await vi.advanceTimersByTimeAsync(1000); + + await expect(credentialsPromise).resolves.toMatchObject({ + access: accessToken, + refresh: "refresh-token", + accountId: "account-403-404", + }); + expect(pollTimes).toHaveLength(3); + }); + + it("includes the response body in OpenAI Codex device auth poll failures", async () => { + vi.stubGlobal( + "fetch", + vi.fn(async (input: unknown): Promise => { + const url = getUrl(input); + if (url === "https://auth.openai.com/api/accounts/deviceauth/usercode") { + return jsonResponse({ + device_auth_id: "device-auth-id", + user_code: "ABCD-1234", + interval: "5", + }); + } + if (url === "https://auth.openai.com/api/accounts/deviceauth/token") { + return jsonResponse({ error: "server_error", error_description: "try again later" }, 500); + } + throw new Error(`Unexpected fetch URL: ${url}`); + }), + ); + + await expect( + loginOpenAICodexDeviceCodeForTest({ + onDeviceCode: () => {}, + }), + ).rejects.toThrow( + 'OpenAI Codex device auth failed with status 500: {"error":"server_error","error_description":"try again later"}', + ); + }); + + it("does not write token refresh failures to stderr", async () => { + const consoleError = vi.spyOn(console, "error").mockImplementation(() => {}); + vi.stubGlobal( + "fetch", + vi.fn(async (): Promise => { + return new Response( + JSON.stringify({ + error: { + message: "Could not validate your token. Please try signing in again.", + type: "invalid_request_error", + }, + }), + { status: 401, statusText: "Unauthorized", headers: { "Content-Type": "application/json" } }, + ); + }), + ); + + await expect( + openaiCodexOAuth.refresh( + { + type: "oauth", + access: "invalid-access-token", + refresh: "invalid-refresh-token", + expires: 0, + }, + neverAbortedSignal, + ), + ).rejects.toThrow(/OpenAI Codex token refresh failed \(401\).*Could not validate your token/); + expect(consoleError).not.toHaveBeenCalled(); + }); +}); diff --git a/packages/ai/test/openai-codex-stream.test.ts b/packages/ai/test/openai-codex-stream.test.ts new file mode 100644 index 0000000000000000000000000000000000000000..67294882552bc245530fd12e179f6c2e1b87bcba --- /dev/null +++ b/packages/ai/test/openai-codex-stream.test.ts @@ -0,0 +1,2654 @@ +import { mkdtempSync } from "node:fs"; +import { arch, platform, release, tmpdir } from "node:os"; +import { join } from "node:path"; +import { zstdDecompressSync } from "node:zlib"; +import { Type } from "typebox"; +import { afterEach, describe, expect, it, vi } from "vitest"; +import { + closeOpenAICodexWebSocketSessions, + getOpenAICodexWebSocketDebugStats, + resetOpenAICodexWebSocketDebugStats, + stream as streamOpenAICodexResponses, + streamSimple as streamSimpleOpenAICodexResponses, +} from "../src/api/openai-codex-responses.ts"; +import type { Context, Model } from "../src/types.ts"; + +const originalAgentDir = process.env.PI_CODING_AGENT_DIR; + +afterEach(() => { + vi.unstubAllGlobals(); + if (originalAgentDir === undefined) { + delete process.env.PI_CODING_AGENT_DIR; + } else { + process.env.PI_CODING_AGENT_DIR = originalAgentDir; + } + closeOpenAICodexWebSocketSessions(); + resetOpenAICodexWebSocketDebugStats(); + vi.useRealTimers(); + vi.restoreAllMocks(); +}); + +function mockToken(accountId = "acc_test"): string { + const payload = Buffer.from( + JSON.stringify({ "https://api.openai.com/auth": { chatgpt_account_id: accountId } }), + "utf8", + ).toString("base64"); + return `aaa.${payload}.bbb`; +} + +function decodeCodexRequestBody(body: RequestInit["body"] | undefined): Record | null { + if (typeof body === "string") { + return JSON.parse(body) as Record; + } + if (body instanceof Uint8Array) { + return JSON.parse(Buffer.from(zstdDecompressSync(body)).toString("utf8")) as Record; + } + return null; +} + +function buildSSEPayload({ + status, + includeDone = false, + endTurn, +}: { + status: "completed" | "incomplete"; + includeDone?: boolean; + endTurn?: boolean; +}): string { + const terminalType = status === "incomplete" ? "response.incomplete" : "response.completed"; + const events = [ + `data: ${JSON.stringify({ + type: "response.output_item.added", + item: { type: "message", id: "msg_1", role: "assistant", status: "in_progress", content: [] }, + })}`, + `data: ${JSON.stringify({ type: "response.content_part.added", part: { type: "output_text", text: "" } })}`, + `data: ${JSON.stringify({ type: "response.output_text.delta", delta: "Hello" })}`, + `data: ${JSON.stringify({ + type: "response.output_item.done", + item: { + type: "message", + id: "msg_1", + role: "assistant", + status: "completed", + content: [{ type: "output_text", text: "Hello" }], + }, + })}`, + `data: ${JSON.stringify({ + type: terminalType, + response: { + status, + end_turn: endTurn, + incomplete_details: status === "incomplete" ? { reason: "max_output_tokens" } : null, + usage: { + input_tokens: 5, + output_tokens: 3, + total_tokens: 8, + input_tokens_details: { cached_tokens: 0 }, + }, + }, + })}`, + ]; + + if (includeDone) { + events.push("data: [DONE]"); + } + + return `${events.join("\n\n")}\n\n`; +} + +describe("openai-codex streaming", () => { + it("streams SSE responses into AssistantMessageEventStream", async () => { + const tempDir = mkdtempSync(join(tmpdir(), "pi-codex-stream-")); + process.env.PI_CODING_AGENT_DIR = tempDir; + + const payload = Buffer.from( + JSON.stringify({ "https://api.openai.com/auth": { chatgpt_account_id: "acc_test" } }), + "utf8", + ).toString("base64"); + const token = `aaa.${payload}.bbb`; + + const sse = `${[ + `data: ${JSON.stringify({ + type: "response.output_item.added", + item: { type: "message", id: "msg_1", role: "assistant", status: "in_progress", content: [] }, + })}`, + `data: ${JSON.stringify({ type: "response.content_part.added", part: { type: "output_text", text: "" } })}`, + `data: ${JSON.stringify({ type: "response.output_text.delta", delta: "Hello" })}`, + `data: ${JSON.stringify({ + type: "response.output_item.done", + item: { + type: "message", + id: "msg_1", + role: "assistant", + status: "completed", + content: [{ type: "output_text", text: "Hello" }], + }, + })}`, + `data: ${JSON.stringify({ + type: "response.completed", + response: { + status: "completed", + usage: { + input_tokens: 5, + output_tokens: 3, + total_tokens: 8, + input_tokens_details: { cached_tokens: 0 }, + }, + }, + })}`, + ].join("\n\n")}\n\n`; + + const encoder = new TextEncoder(); + const stream = new ReadableStream({ + start(controller) { + controller.enqueue(encoder.encode(sse)); + controller.close(); + }, + }); + + const fetchMock = vi.fn(async (input: string | URL, init?: RequestInit) => { + const url = typeof input === "string" ? input : input.toString(); + if (url === "https://api.github.com/repos/openai/codex/releases/latest") { + return new Response(JSON.stringify({ tag_name: "rust-v0.0.0" }), { status: 200 }); + } + if (url.startsWith("https://raw.githubusercontent.com/openai/codex/")) { + return new Response("PROMPT", { status: 200, headers: { etag: '"etag"' } }); + } + if (url === "https://chatgpt.com/backend-api/codex/responses") { + const headers = init?.headers instanceof Headers ? init.headers : undefined; + expect(headers?.get("Authorization")).toBe(`Bearer ${token}`); + expect(headers?.get("chatgpt-account-id")).toBe("acc_test"); + expect(headers?.get("OpenAI-Beta")).toBe("responses=experimental"); + expect(headers?.get("originator")).toBe("pi"); + expect(headers?.get("User-Agent")).toBe(`pi (${platform()} ${release()}; ${arch()})`); + expect(headers?.get("accept")).toBe("text/event-stream"); + expect(headers?.has("x-api-key")).toBe(false); + return new Response(stream, { + status: 200, + headers: { "content-type": "text/event-stream" }, + }); + } + return new Response("not found", { status: 404 }); + }); + + vi.stubGlobal("fetch", fetchMock); + + const model: Model<"openai-codex-responses"> = { + id: "gpt-5.1-codex", + name: "GPT-5.1 Codex", + api: "openai-codex-responses", + provider: "openai-codex", + baseUrl: "https://chatgpt.com/backend-api", + reasoning: true, + input: ["text"], + cost: { input: 0, output: 0, cacheRead: 0, cacheWrite: 0 }, + contextWindow: 400000, + maxTokens: 128000, + }; + + const context: Context = { + systemPrompt: "You are a helpful assistant.", + messages: [{ role: "user", content: "Say hello", timestamp: Date.now() }], + }; + + const streamResult = streamOpenAICodexResponses(model, context, { apiKey: token, transport: "sse" }); + let sawTextDelta = false; + let sawDone = false; + + for await (const event of streamResult) { + if (event.type === "text_delta") { + sawTextDelta = true; + } + if (event.type === "done") { + sawDone = true; + expect(event.message.content.find((c) => c.type === "text")?.text).toBe("Hello"); + } + } + + expect(sawTextDelta).toBe(true); + expect(sawDone).toBe(true); + }); + + // Regression test for https://github.com/earendil-works/pi/issues/9047 + it("processes a terminal SSE event without a trailing blank line", async () => { + const token = mockToken(); + const sse = buildSSEPayload({ status: "completed" }).trimEnd(); + const model: Model<"openai-codex-responses"> = { + id: "gpt-5.1-codex", + name: "GPT-5.1 Codex", + api: "openai-codex-responses", + provider: "openai-codex", + baseUrl: "https://chatgpt.com/backend-api", + reasoning: true, + input: ["text"], + cost: { input: 0, output: 0, cacheRead: 0, cacheWrite: 0 }, + contextWindow: 400000, + maxTokens: 128000, + }; + const context: Context = { + systemPrompt: "You are a helpful assistant.", + messages: [{ role: "user", content: "Say hello", timestamp: Date.now() }], + }; + const resultStream = streamOpenAICodexResponses(model, context, { + apiKey: token, + transport: "sse", + fetch: async () => new Response(sse, { status: 200, headers: { "content-type": "text/event-stream" } }), + }); + const result = await resultStream.result(); + + expect(result.stopReason).toBe("stop"); + expect(result.content.find((content) => content.type === "text")?.text).toBe("Hello"); + }); + + it("completes after response.completed even when the SSE body stays open", async () => { + const tempDir = mkdtempSync(join(tmpdir(), "pi-codex-stream-")); + process.env.PI_CODING_AGENT_DIR = tempDir; + const token = mockToken(); + const encoder = new TextEncoder(); + const sse = buildSSEPayload({ status: "completed", includeDone: true, endTurn: false }); + + const stream = new ReadableStream({ + start(controller) { + controller.enqueue(encoder.encode(sse)); + }, + }); + + const fetchMock = vi.fn(async (input: string | URL) => { + const url = typeof input === "string" ? input : input.toString(); + if (url === "https://api.github.com/repos/openai/codex/releases/latest") { + return new Response(JSON.stringify({ tag_name: "rust-v0.0.0" }), { status: 200 }); + } + if (url.startsWith("https://raw.githubusercontent.com/openai/codex/")) { + return new Response("PROMPT", { status: 200, headers: { etag: '"etag"' } }); + } + if (url === "https://chatgpt.com/backend-api/codex/responses") { + return new Response(stream, { + status: 200, + headers: { "content-type": "text/event-stream" }, + }); + } + return new Response("not found", { status: 404 }); + }); + vi.stubGlobal("fetch", fetchMock); + + const model: Model<"openai-codex-responses"> = { + id: "gpt-5.1-codex", + name: "GPT-5.1 Codex", + api: "openai-codex-responses", + provider: "openai-codex", + baseUrl: "https://chatgpt.com/backend-api", + reasoning: true, + input: ["text"], + cost: { input: 0, output: 0, cacheRead: 0, cacheWrite: 0 }, + contextWindow: 400000, + maxTokens: 128000, + }; + + const context: Context = { + systemPrompt: "You are a helpful assistant.", + messages: [{ role: "user", content: "Say hello", timestamp: Date.now() }], + }; + + const result = await Promise.race([ + streamOpenAICodexResponses(model, context, { apiKey: token, transport: "sse" }).result(), + new Promise((_, reject) => { + setTimeout(() => reject(new Error("Timed out waiting for completed SSE stream")), 1000); + }), + ]); + + expect(result.content.find((c) => c.type === "text")?.text).toBe("Hello"); + expect(result.stopReason).toBe("stop"); + expect(result.endTurn).toBe(false); + }); + + it("maps response.incomplete to stopReason length even when the SSE body stays open", async () => { + const tempDir = mkdtempSync(join(tmpdir(), "pi-codex-stream-")); + process.env.PI_CODING_AGENT_DIR = tempDir; + const token = mockToken(); + const encoder = new TextEncoder(); + const sse = buildSSEPayload({ status: "incomplete" }); + + const stream = new ReadableStream({ + start(controller) { + controller.enqueue(encoder.encode(sse)); + }, + }); + + const fetchMock = vi.fn(async (input: string | URL) => { + const url = typeof input === "string" ? input : input.toString(); + if (url === "https://api.github.com/repos/openai/codex/releases/latest") { + return new Response(JSON.stringify({ tag_name: "rust-v0.0.0" }), { status: 200 }); + } + if (url.startsWith("https://raw.githubusercontent.com/openai/codex/")) { + return new Response("PROMPT", { status: 200, headers: { etag: '"etag"' } }); + } + if (url === "https://chatgpt.com/backend-api/codex/responses") { + return new Response(stream, { + status: 200, + headers: { "content-type": "text/event-stream" }, + }); + } + return new Response("not found", { status: 404 }); + }); + vi.stubGlobal("fetch", fetchMock); + + const model: Model<"openai-codex-responses"> = { + id: "gpt-5.1-codex", + name: "GPT-5.1 Codex", + api: "openai-codex-responses", + provider: "openai-codex", + baseUrl: "https://chatgpt.com/backend-api", + reasoning: true, + input: ["text"], + cost: { input: 0, output: 0, cacheRead: 0, cacheWrite: 0 }, + contextWindow: 400000, + maxTokens: 128000, + }; + + const context: Context = { + systemPrompt: "You are a helpful assistant.", + messages: [{ role: "user", content: "Say hello", timestamp: Date.now() }], + }; + + const result = await Promise.race([ + streamOpenAICodexResponses(model, context, { apiKey: token, transport: "sse" }).result(), + new Promise((_, reject) => { + setTimeout(() => reject(new Error("Timed out waiting for incomplete SSE stream")), 1000); + }), + ]); + + expect(result.content.find((c) => c.type === "text")?.text).toBe("Hello"); + expect(result.stopReason).toBe("length"); + }); + + it("aborts SSE fetch after the configured HTTP timeout when response headers do not arrive", async () => { + const token = mockToken(); + + const fetchMock = vi.fn((input: string | URL, init?: RequestInit) => { + const url = typeof input === "string" ? input : input.toString(); + if (url !== "https://chatgpt.com/backend-api/codex/responses") { + throw new Error(`Unexpected URL: ${url}`); + } + + const signal = init?.signal; + if (!signal) { + throw new Error("Expected SSE fetch to receive an abort signal"); + } + + return new Promise((_, reject) => { + const onAbort = () => { + const reason = signal.reason; + reject(reason instanceof Error ? reason : new Error("SSE fetch aborted")); + }; + if (signal.aborted) { + onAbort(); + return; + } + signal.addEventListener("abort", onAbort, { once: true }); + }); + }); + vi.stubGlobal("fetch", fetchMock); + + const model: Model<"openai-codex-responses"> = { + id: "gpt-5.1-codex", + name: "GPT-5.1 Codex", + api: "openai-codex-responses", + provider: "openai-codex", + baseUrl: "https://chatgpt.com/backend-api", + reasoning: true, + input: ["text"], + cost: { input: 0, output: 0, cacheRead: 0, cacheWrite: 0 }, + contextWindow: 400000, + maxTokens: 128000, + }; + const context: Context = { + systemPrompt: "You are a helpful assistant.", + messages: [{ role: "user", content: "Say hello", timestamp: Date.now() }], + }; + + const result = await streamOpenAICodexResponses(model, context, { + apiKey: token, + transport: "sse", + timeoutMs: 10, + }).result(); + + expect(fetchMock).toHaveBeenCalledTimes(1); + expect(result.stopReason).toBe("error"); + expect(result.errorMessage).toBe("Codex SSE response headers timed out after 10ms"); + }); + + it("aborts SSE body reads after response headers arrive", async () => { + const token = mockToken(); + const encoder = new TextEncoder(); + const timers: ReturnType[] = []; + let cancelled = false; + const stream = new ReadableStream({ + start(controller) { + const enqueue = (chunk: string) => { + if (!cancelled) controller.enqueue(encoder.encode(chunk)); + }; + enqueue( + `${[ + `data: ${JSON.stringify({ + type: "response.output_item.added", + item: { type: "message", id: "msg_1", role: "assistant", status: "in_progress", content: [] }, + })}`, + `data: ${JSON.stringify({ type: "response.content_part.added", part: { type: "output_text", text: "" } })}`, + `data: ${JSON.stringify({ type: "response.output_text.delta", delta: "one" })}`, + ].join("\n\n")}\n\n`, + ); + timers.push( + setTimeout(() => { + enqueue(`data: ${JSON.stringify({ type: "response.output_text.delta", delta: "two" })}\n\n`); + }, 10), + ); + timers.push( + setTimeout(() => { + if (cancelled) return; + enqueue( + `${[ + `data: ${JSON.stringify({ + type: "response.output_item.done", + item: { + type: "message", + id: "msg_1", + role: "assistant", + status: "completed", + content: [{ type: "output_text", text: "onetwo" }], + }, + })}`, + `data: ${JSON.stringify({ + type: "response.completed", + response: { + status: "completed", + usage: { + input_tokens: 5, + output_tokens: 3, + total_tokens: 8, + input_tokens_details: { cached_tokens: 0 }, + }, + }, + })}`, + ].join("\n\n")}\n\n`, + ); + controller.close(); + }, 20), + ); + }, + cancel() { + cancelled = true; + for (const timer of timers) clearTimeout(timer); + }, + }); + + vi.stubGlobal( + "fetch", + vi.fn(async () => new Response(stream, { status: 200, headers: { "content-type": "text/event-stream" } })), + ); + + const model: Model<"openai-codex-responses"> = { + id: "gpt-5.1-codex", + name: "GPT-5.1 Codex", + api: "openai-codex-responses", + provider: "openai-codex", + baseUrl: "https://chatgpt.com/backend-api", + reasoning: true, + input: ["text"], + cost: { input: 0, output: 0, cacheRead: 0, cacheWrite: 0 }, + contextWindow: 400000, + maxTokens: 128000, + }; + const context: Context = { + systemPrompt: "You are a helpful assistant.", + messages: [{ role: "user", content: "Say hello", timestamp: Date.now() }], + }; + const controller = new AbortController(); + const events: string[] = []; + + const resultStream = streamOpenAICodexResponses(model, context, { + apiKey: token, + transport: "sse", + signal: controller.signal, + }); + for await (const event of resultStream) { + events.push(event.type === "text_delta" ? `text_delta:${event.delta}` : event.type); + if (event.type === "text_delta" && event.delta === "one") { + controller.abort(); + } + } + + const result = await resultStream.result(); + expect(result.stopReason).toBe("aborted"); + expect(result.errorMessage).toBe("Request was aborted"); + expect(events).toContain("text_delta:one"); + expect(events).not.toContain("text_delta:two"); + expect(cancelled).toBe(true); + }); + + it("sets session-id/x-client-request-id headers and prompt_cache_key when sessionId is provided", async () => { + const tempDir = mkdtempSync(join(tmpdir(), "pi-codex-stream-")); + process.env.PI_CODING_AGENT_DIR = tempDir; + + const payload = Buffer.from( + JSON.stringify({ "https://api.openai.com/auth": { chatgpt_account_id: "acc_test" } }), + "utf8", + ).toString("base64"); + const token = `aaa.${payload}.bbb`; + + const sse = `${[ + `data: ${JSON.stringify({ + type: "response.output_item.added", + item: { type: "message", id: "msg_1", role: "assistant", status: "in_progress", content: [] }, + })}`, + `data: ${JSON.stringify({ type: "response.content_part.added", part: { type: "output_text", text: "" } })}`, + `data: ${JSON.stringify({ type: "response.output_text.delta", delta: "Hello" })}`, + `data: ${JSON.stringify({ + type: "response.output_item.done", + item: { + type: "message", + id: "msg_1", + role: "assistant", + status: "completed", + content: [{ type: "output_text", text: "Hello" }], + }, + })}`, + `data: ${JSON.stringify({ + type: "response.completed", + response: { + status: "completed", + usage: { + input_tokens: 5, + output_tokens: 3, + total_tokens: 8, + input_tokens_details: { cached_tokens: 0 }, + }, + }, + })}`, + ].join("\n\n")}\n\n`; + + const encoder = new TextEncoder(); + const stream = new ReadableStream({ + start(controller) { + controller.enqueue(encoder.encode(sse)); + controller.close(); + }, + }); + + const sessionId = "test-session-123"; + const fetchMock = vi.fn(async (input: string | URL, init?: RequestInit) => { + const url = typeof input === "string" ? input : input.toString(); + if (url === "https://api.github.com/repos/openai/codex/releases/latest") { + return new Response(JSON.stringify({ tag_name: "rust-v0.0.0" }), { status: 200 }); + } + if (url.startsWith("https://raw.githubusercontent.com/openai/codex/")) { + return new Response("PROMPT", { status: 200, headers: { etag: '"etag"' } }); + } + if (url === "https://chatgpt.com/backend-api/codex/responses") { + const headers = init?.headers instanceof Headers ? init.headers : undefined; + // Verify sessionId is set in headers + expect(headers?.get("session-id")).toBe(sessionId); + expect(headers?.has("session_id")).toBe(false); + expect(headers?.get("x-client-request-id")).toBe(sessionId); + + // Verify sessionId is set in request body as prompt_cache_key + const body = decodeCodexRequestBody(init?.body); + expect(body?.prompt_cache_key).toBe(sessionId); + + return new Response(stream, { + status: 200, + headers: { "content-type": "text/event-stream" }, + }); + } + return new Response("not found", { status: 404 }); + }); + + vi.stubGlobal("fetch", fetchMock); + + const model: Model<"openai-codex-responses"> = { + id: "gpt-5.1-codex", + name: "GPT-5.1 Codex", + api: "openai-codex-responses", + provider: "openai-codex", + baseUrl: "https://chatgpt.com/backend-api", + reasoning: true, + input: ["text"], + cost: { input: 0, output: 0, cacheRead: 0, cacheWrite: 0 }, + contextWindow: 400000, + maxTokens: 128000, + }; + + const context: Context = { + systemPrompt: "You are a helpful assistant.", + messages: [{ role: "user", content: "Say hello", timestamp: Date.now() }], + }; + + const streamResult = streamOpenAICodexResponses(model, context, { apiKey: token, sessionId, transport: "sse" }); + await streamResult.result(); + }); + + it("omits SSE cache affinity when cacheRetention is none", async () => { + const token = mockToken(); + const encoder = new TextEncoder(); + let capturedHeaders: Headers | undefined; + let capturedBody: Record | null = null; + vi.stubGlobal( + "fetch", + vi.fn(async (_input: string | URL, init?: RequestInit) => { + capturedHeaders = init?.headers instanceof Headers ? init.headers : undefined; + capturedBody = decodeCodexRequestBody(init?.body); + return new Response( + new ReadableStream({ + start(controller) { + controller.enqueue(encoder.encode(buildSSEPayload({ status: "completed" }))); + controller.close(); + }, + }), + { status: 200, headers: { "content-type": "text/event-stream" } }, + ); + }), + ); + + const model: Model<"openai-codex-responses"> = { + id: "gpt-5.1-codex", + name: "GPT-5.1 Codex", + api: "openai-codex-responses", + provider: "openai-codex", + baseUrl: "https://chatgpt.com/backend-api", + reasoning: true, + input: ["text"], + cost: { input: 0, output: 0, cacheRead: 0, cacheWrite: 0 }, + contextWindow: 400000, + maxTokens: 128000, + }; + const context: Context = { + systemPrompt: "You are a helpful assistant.", + messages: [{ role: "user", content: "Say hello", timestamp: Date.now() }], + }; + + await streamOpenAICodexResponses(model, context, { + apiKey: token, + cacheRetention: "none", + sessionId: "one-off-summary", + transport: "sse", + }).result(); + + expect(capturedHeaders?.has("session-id")).toBe(false); + expect(capturedHeaders?.has("x-client-request-id")).toBe(false); + expect(capturedBody).not.toHaveProperty("prompt_cache_key"); + }); + + it("clamps prompt_cache_key to OpenAI's 64-character limit", async () => { + const token = mockToken(); + const sessionId = "x".repeat(67); + let capturedPayload: { prompt_cache_key?: string } | undefined; + const encoder = new TextEncoder(); + vi.stubGlobal( + "fetch", + vi.fn( + async () => + new Response( + new ReadableStream({ + start(controller) { + controller.enqueue(encoder.encode(buildSSEPayload({ status: "completed" }))); + controller.close(); + }, + }), + { status: 200, headers: { "content-type": "text/event-stream" } }, + ), + ), + ); + + const model: Model<"openai-codex-responses"> = { + id: "gpt-5.1-codex", + name: "GPT-5.1 Codex", + api: "openai-codex-responses", + provider: "openai-codex", + baseUrl: "https://chatgpt.com/backend-api", + reasoning: true, + input: ["text"], + cost: { input: 0, output: 0, cacheRead: 0, cacheWrite: 0 }, + contextWindow: 400000, + maxTokens: 128000, + }; + const context: Context = { + systemPrompt: "You are a helpful assistant.", + messages: [{ role: "user", content: "Say hello", timestamp: Date.now() }], + }; + + await streamOpenAICodexResponses(model, context, { + apiKey: token, + transport: "sse", + sessionId, + onPayload: (payload) => { + capturedPayload = payload as { prompt_cache_key?: string }; + }, + }).result(); + + expect(capturedPayload?.prompt_cache_key).toBe("x".repeat(64)); + }); + + it("clamps Codex session-id header to 64 characters", async () => { + const token = mockToken(); + const sessionId = "x".repeat(67); + let capturedHeaders: Headers | undefined; + const encoder = new TextEncoder(); + vi.stubGlobal( + "fetch", + vi.fn(async (_input: string | URL, init?: RequestInit) => { + capturedHeaders = init?.headers instanceof Headers ? init.headers : undefined; + return new Response( + new ReadableStream({ + start(controller) { + controller.enqueue(encoder.encode(buildSSEPayload({ status: "completed" }))); + controller.close(); + }, + }), + { status: 200, headers: { "content-type": "text/event-stream" } }, + ); + }), + ); + + const model: Model<"openai-codex-responses"> = { + id: "gpt-5.1-codex", + name: "GPT-5.1 Codex", + api: "openai-codex-responses", + provider: "openai-codex", + baseUrl: "https://chatgpt.com/backend-api", + reasoning: true, + input: ["text"], + cost: { input: 0, output: 0, cacheRead: 0, cacheWrite: 0 }, + contextWindow: 400000, + maxTokens: 128000, + }; + const context: Context = { + systemPrompt: "You are a helpful assistant.", + messages: [{ role: "user", content: "Say hello", timestamp: Date.now() }], + }; + + await streamOpenAICodexResponses(model, context, { + apiKey: token, + transport: "sse", + sessionId, + }).result(); + + expect(capturedHeaders?.get("session-id")).toBe("x".repeat(64)); + expect(capturedHeaders?.get("x-client-request-id")).toBe("x".repeat(64)); + }); + + it("preserves gpt-5.5 xhigh reasoning effort from simple options", async () => { + const tempDir = mkdtempSync(join(tmpdir(), "pi-codex-stream-")); + process.env.PI_CODING_AGENT_DIR = tempDir; + const token = mockToken(); + const sse = buildSSEPayload({ status: "completed" }); + const encoder = new TextEncoder(); + const stream = new ReadableStream({ + start(controller) { + controller.enqueue(encoder.encode(sse)); + controller.close(); + }, + }); + let requestedReasoning: unknown; + + const fetchMock = vi.fn(async (input: string | URL, init?: RequestInit) => { + const url = typeof input === "string" ? input : input.toString(); + if (url === "https://api.github.com/repos/openai/codex/releases/latest") { + return new Response(JSON.stringify({ tag_name: "rust-v0.0.0" }), { status: 200 }); + } + if (url.startsWith("https://raw.githubusercontent.com/openai/codex/")) { + return new Response("PROMPT", { status: 200, headers: { etag: '"etag"' } }); + } + if (url === "https://chatgpt.com/backend-api/codex/responses") { + const body = decodeCodexRequestBody(init?.body); + requestedReasoning = body?.reasoning; + return new Response(stream, { + status: 200, + headers: { "content-type": "text/event-stream" }, + }); + } + return new Response("not found", { status: 404 }); + }); + vi.stubGlobal("fetch", fetchMock); + + const model: Model<"openai-codex-responses"> = { + id: "gpt-5.5", + name: "GPT-5.5", + api: "openai-codex-responses", + provider: "openai-codex", + baseUrl: "https://chatgpt.com/backend-api", + reasoning: true, + thinkingLevelMap: { xhigh: "xhigh" }, + input: ["text"], + cost: { input: 0, output: 0, cacheRead: 0, cacheWrite: 0 }, + contextWindow: 400000, + maxTokens: 128000, + }; + const context: Context = { + systemPrompt: "You are a helpful assistant.", + messages: [{ role: "user", content: "Say hello", timestamp: Date.now() }], + }; + + await streamSimpleOpenAICodexResponses(model, context, { + apiKey: token, + reasoning: "xhigh", + transport: "sse", + }).result(); + + expect(requestedReasoning).toEqual({ effort: "xhigh", summary: "auto" }); + }); + + it("forwards required tool choice", async () => { + const token = mockToken(); + const encoder = new TextEncoder(); + const sse = buildSSEPayload({ status: "completed" }); + let requestedToolChoice: unknown; + + vi.stubGlobal( + "fetch", + vi.fn(async (_input: string | URL, init?: RequestInit) => { + requestedToolChoice = decodeCodexRequestBody(init?.body)?.tool_choice; + return new Response( + new ReadableStream({ + start(controller) { + controller.enqueue(encoder.encode(sse)); + controller.close(); + }, + }), + { status: 200, headers: { "content-type": "text/event-stream" } }, + ); + }), + ); + + const model: Model<"openai-codex-responses"> = { + id: "gpt-5.5", + name: "GPT-5.5", + api: "openai-codex-responses", + provider: "openai-codex", + baseUrl: "https://chatgpt.com/backend-api", + reasoning: true, + input: ["text"], + cost: { input: 0, output: 0, cacheRead: 0, cacheWrite: 0 }, + contextWindow: 400000, + maxTokens: 128000, + }; + + await streamOpenAICodexResponses( + model, + { + messages: [ + { role: "user", content: "Do not call ping. Respond with text instead.", timestamp: Date.now() }, + ], + tools: [ + { + name: "ping", + description: "Ping", + parameters: Type.Object({ value: Type.String() }), + }, + ], + }, + { apiKey: token, transport: "sse", toolChoice: "required" }, + ).result(); + + expect(requestedToolChoice).toBe("required"); + }); + + it("sets Codex strict mode explicitly and honors constrained sampling", async () => { + const token = mockToken(); + const encoder = new TextEncoder(); + const sse = buildSSEPayload({ status: "completed" }); + let requestedTools: Array<{ type?: string; name?: string; strict?: boolean | null }> | undefined; + + vi.stubGlobal( + "fetch", + vi.fn( + async () => + new Response( + new ReadableStream({ + start(controller) { + controller.enqueue(encoder.encode(sse)); + controller.close(); + }, + }), + { status: 200, headers: { "content-type": "text/event-stream" } }, + ), + ), + ); + + const model: Model<"openai-codex-responses"> = { + id: "gpt-5.5", + name: "GPT-5.5", + api: "openai-codex-responses", + provider: "openai-codex", + baseUrl: "https://chatgpt.com/backend-api", + reasoning: true, + input: ["text"], + cost: { input: 0, output: 0, cacheRead: 0, cacheWrite: 0 }, + contextWindow: 400000, + maxTokens: 128000, + }; + + await streamOpenAICodexResponses( + model, + { + messages: [{ role: "user", content: "Use a tool", timestamp: Date.now() }], + tools: [ + { + name: "optional", + description: "Optional constrained sampling", + parameters: Type.Object({ value: Type.String() }), + constrainedSampling: false, + }, + { + name: "strict", + description: "Strict constrained sampling", + parameters: Type.Object({ value: Type.String() }, { additionalProperties: false }), + constrainedSampling: { type: "json_schema", strict: "prefer" }, + }, + ], + }, + { + apiKey: token, + transport: "sse", + onPayload: (payload) => { + requestedTools = (payload as { tools?: typeof requestedTools }).tools; + }, + }, + ).result(); + + expect(requestedTools).toMatchObject([ + { type: "function", name: "optional", strict: null }, + { type: "function", name: "strict", strict: true }, + ]); + }); + + it.each(["gpt-5.3-codex", "gpt-5.4", "gpt-5.5"])("clamps %s minimal reasoning effort to low", async (modelId) => { + const tempDir = mkdtempSync(join(tmpdir(), "pi-codex-stream-")); + process.env.PI_CODING_AGENT_DIR = tempDir; + + const payload = Buffer.from( + JSON.stringify({ "https://api.openai.com/auth": { chatgpt_account_id: "acc_test" } }), + "utf8", + ).toString("base64"); + const token = `aaa.${payload}.bbb`; + + const sse = `${[ + `data: ${JSON.stringify({ + type: "response.output_item.added", + item: { type: "message", id: "msg_1", role: "assistant", status: "in_progress", content: [] }, + })}`, + `data: ${JSON.stringify({ type: "response.content_part.added", part: { type: "output_text", text: "" } })}`, + `data: ${JSON.stringify({ type: "response.output_text.delta", delta: "Hello" })}`, + `data: ${JSON.stringify({ + type: "response.output_item.done", + item: { + type: "message", + id: "msg_1", + role: "assistant", + status: "completed", + content: [{ type: "output_text", text: "Hello" }], + }, + })}`, + `data: ${JSON.stringify({ + type: "response.completed", + response: { + status: "completed", + usage: { + input_tokens: 5, + output_tokens: 3, + total_tokens: 8, + input_tokens_details: { cached_tokens: 0 }, + }, + }, + })}`, + ].join("\n\n")}\n\n`; + + let requestedReasoning: unknown; + const encoder = new TextEncoder(); + const stream = new ReadableStream({ + start(controller) { + controller.enqueue(encoder.encode(sse)); + controller.close(); + }, + }); + + const fetchMock = vi.fn(async (input: string | URL, init?: RequestInit) => { + const url = typeof input === "string" ? input : input.toString(); + if (url === "https://api.github.com/repos/openai/codex/releases/latest") { + return new Response(JSON.stringify({ tag_name: "rust-v0.0.0" }), { status: 200 }); + } + if (url.startsWith("https://raw.githubusercontent.com/openai/codex/")) { + return new Response("PROMPT", { status: 200, headers: { etag: '"etag"' } }); + } + if (url === "https://chatgpt.com/backend-api/codex/responses") { + const body = decodeCodexRequestBody(init?.body); + requestedReasoning = body?.reasoning; + + return new Response(stream, { + status: 200, + headers: { "content-type": "text/event-stream" }, + }); + } + return new Response("not found", { status: 404 }); + }); + + vi.stubGlobal("fetch", fetchMock); + + const model: Model<"openai-codex-responses"> = { + id: modelId, + name: modelId, + api: "openai-codex-responses", + provider: "openai-codex", + baseUrl: "https://chatgpt.com/backend-api", + reasoning: true, + thinkingLevelMap: { minimal: "low" }, + input: ["text"], + cost: { input: 0, output: 0, cacheRead: 0, cacheWrite: 0 }, + contextWindow: 400000, + maxTokens: 128000, + }; + + const context: Context = { + systemPrompt: "You are a helpful assistant.", + messages: [{ role: "user", content: "Say hello", timestamp: Date.now() }], + }; + + const streamResult = streamOpenAICodexResponses(model, context, { + apiKey: token, + reasoningEffort: "minimal", + transport: "sse", + }); + await streamResult.result(); + expect(requestedReasoning).toEqual({ effort: "low", summary: "auto" }); + }); + + it.each([ + ["gpt-5.1-codex", "flex", 0.5], + ["gpt-5.1-codex", "priority", 2], + ["gpt-5.5", "flex", 0.5], + ["gpt-5.5", "priority", 2.5], + ] as const)( + "uses the client-sent %s service tier for %s when Codex echoes default", + async (modelId, serviceTier, multiplier) => { + const tempDir = mkdtempSync(join(tmpdir(), "pi-codex-stream-")); + process.env.PI_CODING_AGENT_DIR = tempDir; + const token = mockToken(); + const sse = `${[ + `data: ${JSON.stringify({ + type: "response.output_item.added", + item: { type: "message", id: "msg_1", role: "assistant", status: "in_progress", content: [] }, + })}`, + `data: ${JSON.stringify({ type: "response.content_part.added", part: { type: "output_text", text: "" } })}`, + `data: ${JSON.stringify({ type: "response.output_text.delta", delta: "Hello" })}`, + `data: ${JSON.stringify({ + type: "response.output_item.done", + item: { + type: "message", + id: "msg_1", + role: "assistant", + status: "completed", + content: [{ type: "output_text", text: "Hello" }], + }, + })}`, + `data: ${JSON.stringify({ + type: "response.completed", + response: { + status: "completed", + service_tier: "default", + usage: { + input_tokens: 1000000, + output_tokens: 1000000, + total_tokens: 2000000, + input_tokens_details: { cached_tokens: 0 }, + }, + }, + })}`, + ].join("\n\n")}\n\n`; + + const encoder = new TextEncoder(); + const stream = new ReadableStream({ + start(controller) { + controller.enqueue(encoder.encode(sse)); + controller.close(); + }, + }); + + const fetchMock = vi.fn(async (input: string | URL) => { + const url = typeof input === "string" ? input : input.toString(); + if (url === "https://api.github.com/repos/openai/codex/releases/latest") { + return new Response(JSON.stringify({ tag_name: "rust-v0.0.0" }), { status: 200 }); + } + if (url.startsWith("https://raw.githubusercontent.com/openai/codex/")) { + return new Response("PROMPT", { status: 200, headers: { etag: '"etag"' } }); + } + if (url === "https://chatgpt.com/backend-api/codex/responses") { + return new Response(stream, { + status: 200, + headers: { "content-type": "text/event-stream" }, + }); + } + return new Response("not found", { status: 404 }); + }); + vi.stubGlobal("fetch", fetchMock); + + const model: Model<"openai-codex-responses"> = { + id: modelId, + name: modelId === "gpt-5.5" ? "GPT-5.5" : "GPT-5.1 Codex", + api: "openai-codex-responses", + provider: "openai-codex", + baseUrl: "https://chatgpt.com/backend-api", + reasoning: true, + input: ["text"], + cost: { input: 1, output: 2, cacheRead: 0, cacheWrite: 0 }, + contextWindow: 400000, + maxTokens: 128000, + }; + + const context: Context = { + systemPrompt: "You are a helpful assistant.", + messages: [{ role: "user", content: "Say hello", timestamp: Date.now() }], + }; + + const result = await streamOpenAICodexResponses(model, context, { + apiKey: token, + serviceTier, + transport: "sse", + }).result(); + + expect(result.usage.cost.input).toBe(1 * multiplier); + expect(result.usage.cost.output).toBe(2 * multiplier); + expect(result.usage.cost.total).toBe(3 * multiplier); + }, + ); + + it("does not set session-id/x-client-request-id headers when sessionId is not provided", async () => { + const tempDir = mkdtempSync(join(tmpdir(), "pi-codex-stream-")); + process.env.PI_CODING_AGENT_DIR = tempDir; + + const payload = Buffer.from( + JSON.stringify({ "https://api.openai.com/auth": { chatgpt_account_id: "acc_test" } }), + "utf8", + ).toString("base64"); + const token = `aaa.${payload}.bbb`; + + const sse = `${[ + `data: ${JSON.stringify({ + type: "response.output_item.added", + item: { type: "message", id: "msg_1", role: "assistant", status: "in_progress", content: [] }, + })}`, + `data: ${JSON.stringify({ type: "response.content_part.added", part: { type: "output_text", text: "" } })}`, + `data: ${JSON.stringify({ type: "response.output_text.delta", delta: "Hello" })}`, + `data: ${JSON.stringify({ + type: "response.output_item.done", + item: { + type: "message", + id: "msg_1", + role: "assistant", + status: "completed", + content: [{ type: "output_text", text: "Hello" }], + }, + })}`, + `data: ${JSON.stringify({ + type: "response.completed", + response: { + status: "completed", + usage: { + input_tokens: 5, + output_tokens: 3, + total_tokens: 8, + input_tokens_details: { cached_tokens: 0 }, + }, + }, + })}`, + ].join("\n\n")}\n\n`; + + const encoder = new TextEncoder(); + const stream = new ReadableStream({ + start(controller) { + controller.enqueue(encoder.encode(sse)); + controller.close(); + }, + }); + + const fetchMock = vi.fn(async (input: string | URL, init?: RequestInit) => { + const url = typeof input === "string" ? input : input.toString(); + if (url === "https://api.github.com/repos/openai/codex/releases/latest") { + return new Response(JSON.stringify({ tag_name: "rust-v0.0.0" }), { status: 200 }); + } + if (url.startsWith("https://raw.githubusercontent.com/openai/codex/")) { + return new Response("PROMPT", { status: 200, headers: { etag: '"etag"' } }); + } + if (url === "https://chatgpt.com/backend-api/codex/responses") { + const headers = init?.headers instanceof Headers ? init.headers : undefined; + // Verify headers are not set when sessionId is not provided + expect(headers?.has("session-id")).toBe(false); + expect(headers?.has("session_id")).toBe(false); + expect(headers?.has("x-client-request-id")).toBe(false); + + return new Response(stream, { + status: 200, + headers: { "content-type": "text/event-stream" }, + }); + } + return new Response("not found", { status: 404 }); + }); + + vi.stubGlobal("fetch", fetchMock); + + const model: Model<"openai-codex-responses"> = { + id: "gpt-5.1-codex", + name: "GPT-5.1 Codex", + api: "openai-codex-responses", + provider: "openai-codex", + baseUrl: "https://chatgpt.com/backend-api", + reasoning: true, + input: ["text"], + cost: { input: 0, output: 0, cacheRead: 0, cacheWrite: 0 }, + contextWindow: 400000, + maxTokens: 128000, + }; + + const context: Context = { + systemPrompt: "You are a helpful assistant.", + messages: [{ role: "user", content: "Say hello", timestamp: Date.now() }], + }; + + // No sessionId provided + const streamResult = streamOpenAICodexResponses(model, context, { apiKey: token, transport: "sse" }); + await streamResult.result(); + }); + it("forwards auto transport from streamSimple options and uses cached websocket context", async () => { + const token = mockToken(); + const sentBodies: unknown[] = []; + let capturedWebSocketHeaders: Record | undefined; + + const fetchMock = vi.fn(async () => new Response("unexpected fetch", { status: 500 })); + vi.stubGlobal("fetch", fetchMock); + + class MockWebSocket { + private listeners = new Map void>>(); + + constructor(_url: string, protocols?: string | string[] | { headers?: Record }) { + if (protocols && typeof protocols === "object" && !Array.isArray(protocols)) { + capturedWebSocketHeaders = protocols.headers; + } + queueMicrotask(() => this.dispatch("open", {})); + } + + addEventListener(type: string, listener: (event: unknown) => void): void { + let listeners = this.listeners.get(type); + if (!listeners) { + listeners = new Set(); + this.listeners.set(type, listeners); + } + listeners.add(listener); + } + + removeEventListener(type: string, listener: (event: unknown) => void): void { + this.listeners.get(type)?.delete(listener); + } + + send(data: string): void { + sentBodies.push(JSON.parse(data)); + const events = [ + { + type: "response.output_item.added", + item: { type: "message", id: "msg_1", role: "assistant", status: "in_progress", content: [] }, + }, + { type: "response.content_part.added", part: { type: "output_text", text: "" } }, + { type: "response.output_text.delta", delta: "Hello" }, + { + type: "response.output_item.done", + item: { + type: "message", + id: "msg_1", + role: "assistant", + status: "completed", + content: [{ type: "output_text", text: "Hello" }], + }, + }, + { + type: "response.completed", + response: { + status: "completed", + end_turn: false, + usage: { + input_tokens: 5, + output_tokens: 3, + total_tokens: 8, + input_tokens_details: { cached_tokens: 0 }, + }, + }, + }, + ]; + queueMicrotask(() => { + for (const event of events) { + this.dispatch("message", { data: JSON.stringify(event) }); + } + }); + } + + close(): void {} + + private dispatch(type: string, event: unknown): void { + for (const listener of this.listeners.get(type) ?? []) { + listener(event); + } + } + } + + vi.stubGlobal("WebSocket", MockWebSocket); + + const model: Model<"openai-codex-responses"> = { + id: "gpt-5.1-codex", + name: "GPT-5.1 Codex", + api: "openai-codex-responses", + provider: "openai-codex", + baseUrl: "https://chatgpt.com/backend-api", + reasoning: true, + input: ["text"], + cost: { input: 0, output: 0, cacheRead: 0, cacheWrite: 0 }, + contextWindow: 400000, + maxTokens: 128000, + }; + const context: Context = { + systemPrompt: "You are a helpful assistant.", + messages: [{ role: "user", content: "Say hello", timestamp: 1 }], + }; + + const result = await streamSimpleOpenAICodexResponses(model, context, { + apiKey: token, + sessionId: "session-auto", + transport: "auto", + }).result(); + + expect(result.endTurn).toBe(false); + expect(sentBodies).toHaveLength(1); + expect(capturedWebSocketHeaders?.["session-id"]).toBe("session-auto"); + expect(capturedWebSocketHeaders?.session_id).toBeUndefined(); + expect(capturedWebSocketHeaders?.["x-client-request-id"]).toBe("session-auto"); + expect(global.fetch).not.toHaveBeenCalled(); + expect(getOpenAICodexWebSocketDebugStats("session-auto")).toMatchObject({ + cachedContextRequests: 1, + fullContextRequests: 1, + }); + }); + + it("scopes cached websockets to the authenticated account", async () => { + // Regression for #7284: rotating accounts must not reuse a socket authenticated by another account. + const connectedHeaders: Record[] = []; + let responseId = 0; + + class MockWebSocket { + static OPEN = 1; + readyState = MockWebSocket.OPEN; + private listeners = new Map void>>(); + + constructor(_url: string, protocols?: string | string[] | { headers?: Record }) { + const headers = + protocols && typeof protocols === "object" && !Array.isArray(protocols) ? protocols.headers : undefined; + connectedHeaders.push(headers ?? {}); + queueMicrotask(() => this.dispatch("open", {})); + } + + addEventListener(type: string, listener: (event: unknown) => void): void { + let listeners = this.listeners.get(type); + if (!listeners) { + listeners = new Set(); + this.listeners.set(type, listeners); + } + listeners.add(listener); + } + + removeEventListener(type: string, listener: (event: unknown) => void): void { + this.listeners.get(type)?.delete(listener); + } + + send(): void { + queueMicrotask(() => { + this.dispatch("message", { + data: JSON.stringify({ + type: "response.completed", + response: { + id: `resp_${++responseId}`, + status: "completed", + usage: { input_tokens: 1, output_tokens: 1, total_tokens: 2 }, + }, + }), + }); + }); + } + + close(): void { + this.readyState = 3; + } + + private dispatch(type: string, event: unknown): void { + for (const listener of this.listeners.get(type) ?? []) listener(event); + } + } + + vi.stubGlobal("WebSocket", MockWebSocket); + vi.stubGlobal( + "fetch", + vi.fn(async () => new Response("unexpected fetch", { status: 500 })), + ); + + const model: Model<"openai-codex-responses"> = { + id: "gpt-5.1-codex", + name: "GPT-5.1 Codex", + api: "openai-codex-responses", + provider: "openai-codex", + baseUrl: "https://chatgpt.com/backend-api", + reasoning: true, + input: ["text"], + cost: { input: 0, output: 0, cacheRead: 0, cacheWrite: 0 }, + contextWindow: 400000, + maxTokens: 128000, + }; + const context: Context = { systemPrompt: "", messages: [] }; + const options = { sessionId: "shared-session", transport: "websocket-cached" as const }; + + await streamOpenAICodexResponses(model, context, { ...options, apiKey: mockToken("account-a") }).result(); + await streamOpenAICodexResponses(model, context, { ...options, apiKey: mockToken("account-b") }).result(); + await streamOpenAICodexResponses(model, context, { ...options, apiKey: mockToken("account-a") }).result(); + + expect(connectedHeaders.map((headers) => headers["chatgpt-account-id"])).toEqual(["account-a", "account-b"]); + expect(connectedHeaders.map((headers) => headers.authorization)).toEqual([ + `Bearer ${mockToken("account-a")}`, + `Bearer ${mockToken("account-b")}`, + ]); + expect(global.fetch).not.toHaveBeenCalled(); + expect(getOpenAICodexWebSocketDebugStats("shared-session")).toMatchObject({ + connectionsCreated: 2, + connectionsReused: 1, + }); + }); + + it("closes one-shot websockets when cacheRetention is none", async () => { + const token = mockToken(); + const sentBodies: Array<{ prompt_cache_key?: string }> = []; + let connections = 0; + let closedConnections = 0; + + class MockWebSocket { + private listeners = new Map void>>(); + + constructor() { + connections++; + queueMicrotask(() => this.dispatch("open", {})); + } + + addEventListener(type: string, listener: (event: unknown) => void): void { + let listeners = this.listeners.get(type); + if (!listeners) { + listeners = new Set(); + this.listeners.set(type, listeners); + } + listeners.add(listener); + } + + removeEventListener(type: string, listener: (event: unknown) => void): void { + this.listeners.get(type)?.delete(listener); + } + + send(data: string): void { + sentBodies.push(JSON.parse(data) as { prompt_cache_key?: string }); + queueMicrotask(() => { + this.dispatch("message", { + data: JSON.stringify({ + type: "response.completed", + response: { + id: `resp_${connections}`, + status: "completed", + usage: { input_tokens: 5, output_tokens: 3, total_tokens: 8 }, + }, + }), + }); + }); + } + + close(): void { + closedConnections++; + } + + private dispatch(type: string, event: unknown): void { + for (const listener of this.listeners.get(type) ?? []) { + listener(event); + } + } + } + + vi.stubGlobal("WebSocket", MockWebSocket); + vi.stubGlobal( + "fetch", + vi.fn(async () => new Response("unexpected fetch", { status: 500 })), + ); + + const model: Model<"openai-codex-responses"> = { + id: "gpt-5.1-codex", + name: "GPT-5.1 Codex", + api: "openai-codex-responses", + provider: "openai-codex", + baseUrl: "https://chatgpt.com/backend-api", + reasoning: true, + input: ["text"], + cost: { input: 0, output: 0, cacheRead: 0, cacheWrite: 0 }, + contextWindow: 400000, + maxTokens: 128000, + }; + const context: Context = { + systemPrompt: "You are a helpful assistant.", + messages: [{ role: "user", content: "Say hello", timestamp: 1 }], + }; + const options = { + apiKey: token, + cacheRetention: "none" as const, + sessionId: "one-off-summary", + transport: "auto" as const, + }; + + await streamOpenAICodexResponses(model, context, options).result(); + await streamOpenAICodexResponses(model, context, options).result(); + + expect(connections).toBe(2); + expect(closedConnections).toBe(2); + expect(sentBodies).toHaveLength(2); + expect(sentBodies.every((body) => body.prompt_cache_key === undefined)).toBe(true); + expect(getOpenAICodexWebSocketDebugStats("one-off-summary")).toBeUndefined(); + expect(global.fetch).not.toHaveBeenCalled(); + }); + + it("falls back to SSE when websocket connect does not open before the connect timeout", async () => { + vi.useFakeTimers(); + const token = mockToken(); + const encoder = new TextEncoder(); + const sse = buildSSEPayload({ status: "completed" }); + + const fetchMock = vi.fn(async (input: string | URL) => { + const url = typeof input === "string" ? input : input.toString(); + if (url !== "https://chatgpt.com/backend-api/codex/responses") { + throw new Error(`Unexpected URL: ${url}`); + } + + return new Response( + new ReadableStream({ + start(controller) { + controller.enqueue(encoder.encode(sse)); + controller.close(); + }, + }), + { status: 200, headers: { "content-type": "text/event-stream" } }, + ); + }); + vi.stubGlobal("fetch", fetchMock); + + class MockWebSocket { + private listeners = new Map void>>(); + + addEventListener(type: string, listener: (event: unknown) => void): void { + let listeners = this.listeners.get(type); + if (!listeners) { + listeners = new Set(); + this.listeners.set(type, listeners); + } + listeners.add(listener); + } + + removeEventListener(type: string, listener: (event: unknown) => void): void { + this.listeners.get(type)?.delete(listener); + } + + send(): void { + throw new Error("send should not be called before websocket open"); + } + + close(): void {} + } + + vi.stubGlobal("WebSocket", MockWebSocket); + + const model: Model<"openai-codex-responses"> = { + id: "gpt-5.1-codex", + name: "GPT-5.1 Codex", + api: "openai-codex-responses", + provider: "openai-codex", + baseUrl: "https://chatgpt.com/backend-api", + reasoning: true, + input: ["text"], + cost: { input: 0, output: 0, cacheRead: 0, cacheWrite: 0 }, + contextWindow: 400000, + maxTokens: 128000, + }; + const context: Context = { + systemPrompt: "You are a helpful assistant.", + messages: [{ role: "user", content: "Say hello", timestamp: 1 }], + }; + + const resultPromise = streamOpenAICodexResponses(model, context, { + apiKey: token, + sessionId: "ws-connect-timeout", + transport: "auto", + timeoutMs: 300_000, + websocketConnectTimeoutMs: 50, + }).result(); + + await vi.advanceTimersByTimeAsync(50); + + const result = await resultPromise; + expect(result.content.find((content) => content.type === "text")?.text).toBe("Hello"); + expect(fetchMock).toHaveBeenCalledTimes(1); + expect(getOpenAICodexWebSocketDebugStats("ws-connect-timeout")).toMatchObject({ + websocketFailures: 1, + sseFallbacks: 1, + websocketFallbackActive: true, + lastWebSocketError: "WebSocket connect timeout after 50ms", + }); + }); + + it("reconnects once when the websocket connection limit is reached before output starts", async () => { + const token = mockToken(); + let connections = 0; + + const fetchMock = vi.fn(); + vi.stubGlobal("fetch", fetchMock); + + class MockWebSocket extends EventTarget { + private readonly limitReached = connections++ === 0; + + constructor() { + super(); + queueMicrotask(() => this.dispatchEvent(new Event("open"))); + } + + send(): void { + const event = this.limitReached + ? { type: "error", error: { code: "websocket_connection_limit_reached" } } + : { + type: "response.completed", + response: { + id: "resp_1", + status: "completed", + usage: { input_tokens: 5, output_tokens: 3, total_tokens: 8 }, + }, + }; + queueMicrotask(() => { + this.dispatchEvent(Object.assign(new Event("message"), { data: JSON.stringify(event) })); + }); + } + + close(): void {} + } + + vi.stubGlobal("WebSocket", MockWebSocket); + + const model: Model<"openai-codex-responses"> = { + id: "gpt-5.1-codex", + name: "GPT-5.1 Codex", + api: "openai-codex-responses", + provider: "openai-codex", + baseUrl: "https://chatgpt.com/backend-api", + reasoning: true, + input: ["text"], + cost: { input: 0, output: 0, cacheRead: 0, cacheWrite: 0 }, + contextWindow: 400000, + maxTokens: 128000, + }; + + const result = await streamOpenAICodexResponses( + model, + { systemPrompt: "", messages: [] }, + { + apiKey: token, + }, + ).result(); + + expect(result.stopReason).toBe("stop"); + expect(connections).toBe(2); + expect(fetchMock).not.toHaveBeenCalled(); + }); + + it("falls back to SSE when a websocket is idle before the first event", async () => { + vi.useFakeTimers(); + const token = mockToken(); + const sentBodies: unknown[] = []; + const encoder = new TextEncoder(); + const sse = buildSSEPayload({ status: "completed" }); + + const fetchMock = vi.fn(async (input: string | URL) => { + const url = typeof input === "string" ? input : input.toString(); + if (url !== "https://chatgpt.com/backend-api/codex/responses") { + throw new Error(`Unexpected URL: ${url}`); + } + + return new Response( + new ReadableStream({ + start(controller) { + controller.enqueue(encoder.encode(sse)); + controller.close(); + }, + }), + { status: 200, headers: { "content-type": "text/event-stream" } }, + ); + }); + vi.stubGlobal("fetch", fetchMock); + + class MockWebSocket { + static OPEN = 1; + readyState = MockWebSocket.OPEN; + private listeners = new Map void>>(); + + constructor(_url: string, _protocols?: string | string[] | { headers?: Record }) { + queueMicrotask(() => this.dispatch("open", {})); + } + + addEventListener(type: string, listener: (event: unknown) => void): void { + let listeners = this.listeners.get(type); + if (!listeners) { + listeners = new Set(); + this.listeners.set(type, listeners); + } + listeners.add(listener); + } + + removeEventListener(type: string, listener: (event: unknown) => void): void { + this.listeners.get(type)?.delete(listener); + } + + send(data: string): void { + sentBodies.push(JSON.parse(data)); + } + + close(): void { + this.readyState = 3; + } + + private dispatch(type: string, event: unknown): void { + for (const listener of this.listeners.get(type) ?? []) { + listener(event); + } + } + } + + vi.stubGlobal("WebSocket", MockWebSocket); + + const model: Model<"openai-codex-responses"> = { + id: "gpt-5.1-codex", + name: "GPT-5.1 Codex", + api: "openai-codex-responses", + provider: "openai-codex", + baseUrl: "https://chatgpt.com/backend-api", + reasoning: true, + input: ["text"], + cost: { input: 0, output: 0, cacheRead: 0, cacheWrite: 0 }, + contextWindow: 400000, + maxTokens: 128000, + }; + const context: Context = { + systemPrompt: "You are a helpful assistant.", + messages: [{ role: "user", content: "Say hello", timestamp: 1 }], + }; + + const resultPromise = streamOpenAICodexResponses(model, context, { + apiKey: token, + sessionId: "ws-idle-before-start", + transport: "auto", + timeoutMs: 50, + }).result(); + + await vi.advanceTimersByTimeAsync(0); + expect(sentBodies).toHaveLength(1); + await vi.advanceTimersByTimeAsync(50); + + const result = await resultPromise; + expect(result.content.find((content) => content.type === "text")?.text).toBe("Hello"); + expect(fetchMock).toHaveBeenCalledTimes(1); + expect(getOpenAICodexWebSocketDebugStats("ws-idle-before-start")).toMatchObject({ + websocketFailures: 1, + sseFallbacks: 1, + websocketFallbackActive: true, + }); + }); + + it("errors when a websocket is idle after the stream started", async () => { + vi.useFakeTimers(); + const token = mockToken(); + + const fetchMock = vi.fn(async () => new Response("unexpected fetch", { status: 500 })); + vi.stubGlobal("fetch", fetchMock); + + class MockWebSocket { + static OPEN = 1; + readyState = MockWebSocket.OPEN; + private listeners = new Map void>>(); + + constructor(_url: string, _protocols?: string | string[] | { headers?: Record }) { + queueMicrotask(() => this.dispatch("open", {})); + } + + addEventListener(type: string, listener: (event: unknown) => void): void { + let listeners = this.listeners.get(type); + if (!listeners) { + listeners = new Set(); + this.listeners.set(type, listeners); + } + listeners.add(listener); + } + + removeEventListener(type: string, listener: (event: unknown) => void): void { + this.listeners.get(type)?.delete(listener); + } + + send(): void { + queueMicrotask(() => { + this.dispatch("message", { + data: JSON.stringify({ + type: "response.output_item.added", + item: { type: "message", id: "msg_1", role: "assistant", status: "in_progress", content: [] }, + }), + }); + }); + } + + close(): void { + this.readyState = 3; + } + + private dispatch(type: string, event: unknown): void { + for (const listener of this.listeners.get(type) ?? []) { + listener(event); + } + } + } + + vi.stubGlobal("WebSocket", MockWebSocket); + + const model: Model<"openai-codex-responses"> = { + id: "gpt-5.1-codex", + name: "GPT-5.1 Codex", + api: "openai-codex-responses", + provider: "openai-codex", + baseUrl: "https://chatgpt.com/backend-api", + reasoning: true, + input: ["text"], + cost: { input: 0, output: 0, cacheRead: 0, cacheWrite: 0 }, + contextWindow: 400000, + maxTokens: 128000, + }; + const context: Context = { + systemPrompt: "You are a helpful assistant.", + messages: [{ role: "user", content: "Say hello", timestamp: 1 }], + }; + + const resultPromise = streamOpenAICodexResponses(model, context, { + apiKey: token, + transport: "auto", + timeoutMs: 50, + }).result(); + + await vi.advanceTimersByTimeAsync(0); + await vi.advanceTimersByTimeAsync(50); + + const result = await resultPromise; + expect(result.stopReason).toBe("error"); + expect(result.errorMessage).toBe("WebSocket idle timeout after 50ms"); + expect(fetchMock).not.toHaveBeenCalled(); + }); + + it("opens a fresh cached websocket before the backend connection age limit", async () => { + vi.useFakeTimers(); + const startedAt = new Date("2026-07-03T00:00:00Z"); + vi.setSystemTime(startedAt); + const token = mockToken(); + const sentConnectionIds: number[] = []; + let connections = 0; + + class MockWebSocket { + static OPEN = 1; + static CLOSED = 3; + readyState = MockWebSocket.OPEN; + private readonly connectionId = ++connections; + private listeners = new Map void>>(); + + constructor(_url: string, _protocols?: string | string[] | { headers?: Record }) { + queueMicrotask(() => this.dispatch("open", {})); + } + + addEventListener(type: string, listener: (event: unknown) => void): void { + let listeners = this.listeners.get(type); + if (!listeners) { + listeners = new Set(); + this.listeners.set(type, listeners); + } + listeners.add(listener); + } + + removeEventListener(type: string, listener: (event: unknown) => void): void { + this.listeners.get(type)?.delete(listener); + } + + send(): void { + sentConnectionIds.push(this.connectionId); + const responseId = `resp_${this.connectionId}`; + queueMicrotask(() => { + this.dispatch("message", { + data: JSON.stringify({ + type: "response.completed", + response: { + id: responseId, + status: "completed", + usage: { input_tokens: 5, output_tokens: 3, total_tokens: 8 }, + }, + }), + }); + }); + } + + close(): void { + this.readyState = MockWebSocket.CLOSED; + } + + private dispatch(type: string, event: unknown): void { + for (const listener of this.listeners.get(type) ?? []) { + listener(event); + } + } + } + + vi.stubGlobal("WebSocket", MockWebSocket); + + const model: Model<"openai-codex-responses"> = { + id: "gpt-5.1-codex", + name: "GPT-5.1 Codex", + api: "openai-codex-responses", + provider: "openai-codex", + baseUrl: "https://chatgpt.com/backend-api", + reasoning: true, + input: ["text"], + cost: { input: 0, output: 0, cacheRead: 0, cacheWrite: 0 }, + contextWindow: 400000, + maxTokens: 128000, + }; + const sessionId = "aged-ws-session"; + const firstContext: Context = { + systemPrompt: "You are a helpful assistant.", + messages: [{ role: "user", content: "Say hello", timestamp: 1 }], + }; + + const first = await streamOpenAICodexResponses(model, firstContext, { + apiKey: token, + sessionId, + transport: "websocket-cached", + }).result(); + vi.setSystemTime(new Date(startedAt.getTime() + 56 * 60 * 1000)); + const secondContext: Context = { + systemPrompt: "You are a helpful assistant.", + messages: [...firstContext.messages, first, { role: "user", content: "Now finish", timestamp: 2 }], + }; + + await streamOpenAICodexResponses(model, secondContext, { + apiKey: token, + sessionId, + transport: "websocket-cached", + }).result(); + + expect(connections).toBe(2); + expect(sentConnectionIds).toEqual([1, 2]); + expect(getOpenAICodexWebSocketDebugStats(sessionId)).toMatchObject({ + connectionsCreated: 2, + connectionsReused: 0, + }); + }); + + it("sends only response input deltas in websocket-cached mode", async () => { + const token = mockToken(); + const sentBodies: unknown[] = []; + + class MockWebSocket { + static OPEN = 1; + readyState = MockWebSocket.OPEN; + private listeners = new Map void>>(); + + constructor(_url: string, _protocols?: string | string[] | { headers?: Record }) { + queueMicrotask(() => this.dispatch("open", {})); + } + + addEventListener(type: string, listener: (event: unknown) => void): void { + let listeners = this.listeners.get(type); + if (!listeners) { + listeners = new Set(); + this.listeners.set(type, listeners); + } + listeners.add(listener); + } + + removeEventListener(type: string, listener: (event: unknown) => void): void { + this.listeners.get(type)?.delete(listener); + } + + send(data: string): void { + sentBodies.push(JSON.parse(data)); + const responseId = `resp_${sentBodies.length}`; + const outputEvents = + sentBodies.length === 1 + ? [ + { + type: "response.output_item.added", + item: { + type: "custom_tool_call", + id: "ctc_1", + call_id: "call_1", + name: "sample_tool", + input: "", + }, + }, + { type: "response.custom_tool_call_input.delta", item_id: "ctc_1", delta: "abc" }, + { type: "response.custom_tool_call_input.done", item_id: "ctc_1", input: "abc" }, + { + type: "response.output_item.done", + item: { + type: "custom_tool_call", + id: "ctc_1", + call_id: "call_1", + name: "sample_tool", + input: "abc", + }, + }, + ] + : []; + const events = [ + { type: "response.created", response: { id: responseId } }, + ...outputEvents, + { + type: "response.completed", + response: { + id: responseId, + status: "completed", + usage: { + input_tokens: 5, + output_tokens: 3, + total_tokens: 8, + input_tokens_details: { cached_tokens: 0 }, + }, + }, + }, + ]; + queueMicrotask(() => { + for (const event of events) { + this.dispatch("message", { data: JSON.stringify(event) }); + } + }); + } + + close(): void { + this.readyState = 3; + } + + private dispatch(type: string, event: unknown): void { + for (const listener of this.listeners.get(type) ?? []) { + listener(event); + } + } + } + + vi.stubGlobal("WebSocket", MockWebSocket); + + const model: Model<"openai-codex-responses"> = { + id: "gpt-5.1-codex", + name: "GPT-5.1 Codex", + api: "openai-codex-responses", + provider: "openai-codex", + baseUrl: "https://chatgpt.com/backend-api", + reasoning: true, + input: ["text"], + cost: { input: 0, output: 0, cacheRead: 0, cacheWrite: 0 }, + contextWindow: 400000, + maxTokens: 128000, + compat: { supportsOpenAIGrammarTools: true }, + }; + const firstContext: Context = { + systemPrompt: "You are a helpful assistant.", + messages: [{ role: "user", content: "Use the tool", timestamp: 1 }], + tools: [ + { + name: "sample_tool", + description: "Sample tool", + parameters: Type.Object({ payload: Type.String() }), + constrainedSampling: { type: "grammar", variants: { openai_lark: "start: /[a-z]+/" } }, + }, + ], + }; + + const first = await streamOpenAICodexResponses(model, firstContext, { + apiKey: token, + sessionId: "session-1", + transport: "websocket-cached", + }).result(); + + const secondContext: Context = { + ...firstContext, + messages: [ + ...firstContext.messages, + first, + { + role: "toolResult", + toolCallId: "call_1|ctc_1", + toolName: "sample_tool", + content: [{ type: "text", text: "real result" }], + isError: false, + timestamp: 2, + }, + { role: "user", content: "Now finish", timestamp: 3 }, + ], + }; + await streamOpenAICodexResponses(model, secondContext, { + apiKey: token, + sessionId: "session-1", + transport: "websocket-cached", + }).result(); + + expect(sentBodies).toHaveLength(2); + const firstBody = sentBodies[0] as { input: unknown[]; previous_response_id?: string; store?: boolean }; + const secondBody = sentBodies[1] as { input: unknown[]; previous_response_id?: string; store?: boolean }; + expect(firstBody.store).toBe(false); + expect(firstBody.previous_response_id).toBeUndefined(); + expect(firstBody.input).toEqual([{ role: "user", content: [{ type: "input_text", text: "Use the tool" }] }]); + expect(secondBody.store).toBe(false); + expect(secondBody.previous_response_id).toBe("resp_1"); + expect(secondBody.input).toEqual([ + { type: "custom_tool_call_output", call_id: "call_1", output: "real result" }, + { role: "user", content: [{ type: "input_text", text: "Now finish" }] }, + ]); + expect(getOpenAICodexWebSocketDebugStats("session-1")).toMatchObject({ + requests: 2, + connectionsCreated: 1, + connectionsReused: 1, + cachedContextRequests: 2, + storeTrueRequests: 0, + fullContextRequests: 1, + deltaRequests: 1, + lastDeltaInputItems: 2, + lastPreviousResponseId: "resp_1", + }); + }); + + it.each(["websocket", "sse"] as const)( + "recovers a missing cached websocket continuation via %s", + async (recoveryTransport) => { + const token = mockToken(); + const sessionId = `missing-continuation-${recoveryTransport}`; + const encoder = new TextEncoder(); + const fetchMock = vi.fn( + async () => + new Response( + new ReadableStream({ + start(controller) { + controller.enqueue(encoder.encode(buildSSEPayload({ status: "completed" }))); + controller.close(); + }, + }), + { status: 200, headers: { "content-type": "text/event-stream" } }, + ), + ); + vi.stubGlobal("fetch", fetchMock); + const sentBodies: Array<{ + connectionId: number; + input: unknown[]; + previous_response_id?: string; + }> = []; + let connections = 0; + + class MockWebSocket { + static OPEN = 1; + static CLOSED = 3; + readyState = MockWebSocket.OPEN; + private readonly connectionId = ++connections; + private listeners = new Map void>>(); + + constructor(_url: string, _protocols?: string | string[] | { headers?: Record }) { + queueMicrotask(() => this.dispatch("open", {})); + } + + addEventListener(type: string, listener: (event: unknown) => void): void { + let listeners = this.listeners.get(type); + if (!listeners) { + listeners = new Set(); + this.listeners.set(type, listeners); + } + listeners.add(listener); + } + + removeEventListener(type: string, listener: (event: unknown) => void): void { + this.listeners.get(type)?.delete(listener); + } + + send(data: string): void { + const body = JSON.parse(data) as { input: unknown[]; previous_response_id?: string }; + sentBodies.push({ ...body, connectionId: this.connectionId }); + if (sentBodies.length === 2) { + this.dispatchEvents([ + { + type: "codex.rate_limits", + plan_type: "plus", + rate_limits: { + allowed: true, + limit_reached: false, + primary: { + used_percent: 7, + window_minutes: 10080, + reset_after_seconds: 556112, + reset_at: 1785269351, + }, + secondary: null, + }, + code_review_rate_limits: null, + additional_rate_limits: null, + credits: { has_credits: false, unlimited: false, balance: "0" }, + promo: null, + }, + { + type: "error", + status: 400, + error: { + code: "previous_response_not_found", + message: "Previous response with id 'resp_1' not found.", + param: "previous_response_id", + }, + }, + ]); + return; + } + if (sentBodies.length === 3 && recoveryTransport === "sse") { + queueMicrotask(() => this.dispatch("error", { message: "retry websocket failed" })); + return; + } + + const response = + sentBodies.length === 1 + ? { responseId: "resp_1", messageId: "msg_1", text: "Hello" } + : { responseId: "resp_2", messageId: "msg_2", text: "Recovered" }; + this.dispatchEvents([ + { type: "response.created", response: { id: response.responseId } }, + { + type: "response.output_item.added", + output_index: 0, + item: { + type: "message", + id: response.messageId, + role: "assistant", + status: "in_progress", + content: [], + }, + }, + { + type: "response.output_item.done", + output_index: 0, + item: { + type: "message", + id: response.messageId, + role: "assistant", + status: "completed", + content: [{ type: "output_text", text: response.text }], + }, + }, + { + type: "response.completed", + response: { + id: response.responseId, + status: "completed", + usage: { input_tokens: 5, output_tokens: 3, total_tokens: 8 }, + }, + }, + ]); + } + + close(): void { + this.readyState = MockWebSocket.CLOSED; + } + + private dispatchEvents(events: unknown[]): void { + queueMicrotask(() => { + for (const event of events) { + this.dispatch("message", { data: JSON.stringify(event) }); + } + }); + } + + private dispatch(type: string, event: unknown): void { + for (const listener of this.listeners.get(type) ?? []) { + listener(event); + } + } + } + + vi.stubGlobal("WebSocket", MockWebSocket); + + const model: Model<"openai-codex-responses"> = { + id: "gpt-5.1-codex", + name: "GPT-5.1 Codex", + api: "openai-codex-responses", + provider: "openai-codex", + baseUrl: "https://chatgpt.com/backend-api", + reasoning: true, + input: ["text"], + cost: { input: 0, output: 0, cacheRead: 0, cacheWrite: 0 }, + contextWindow: 400000, + maxTokens: 128000, + }; + const firstContext: Context = { + systemPrompt: "You are a helpful assistant.", + messages: [{ role: "user", content: "Say hello", timestamp: 1 }], + }; + + const first = await streamOpenAICodexResponses(model, firstContext, { + apiKey: token, + sessionId, + transport: "websocket-cached", + }).result(); + const secondContext: Context = { + systemPrompt: "You are a helpful assistant.", + messages: [...firstContext.messages, first, { role: "user", content: "Now finish", timestamp: 2 }], + }; + const eventTypes: string[] = []; + const secondStream = streamOpenAICodexResponses(model, secondContext, { + apiKey: token, + sessionId, + transport: "websocket-cached", + }); + for await (const event of secondStream) { + eventTypes.push(event.type); + } + const second = await secondStream.result(); + + expect(second.stopReason).toBe("stop"); + expect(second.content.find((content) => content.type === "text")?.text).toBe( + recoveryTransport === "sse" ? "Hello" : "Recovered", + ); + expect(eventTypes.filter((type) => type === "start")).toHaveLength(1); + expect(eventTypes).not.toContain("error"); + expect(connections).toBe(2); + expect(sentBodies).toHaveLength(3); + expect(sentBodies.map((body) => body.connectionId)).toEqual([1, 1, 2]); + expect(sentBodies[1].previous_response_id).toBe("resp_1"); + expect(sentBodies[1].input).toEqual([{ role: "user", content: [{ type: "input_text", text: "Now finish" }] }]); + expect(sentBodies[2].previous_response_id).toBeUndefined(); + expect(sentBodies[2].input).toHaveLength(3); + expect(sentBodies[2].input.at(-1)).toEqual({ + role: "user", + content: [{ type: "input_text", text: "Now finish" }], + }); + expect(fetchMock).toHaveBeenCalledTimes(recoveryTransport === "sse" ? 1 : 0); + expect(getOpenAICodexWebSocketDebugStats(sessionId)).toMatchObject({ + requests: 3, + connectionsCreated: 2, + connectionsReused: 1, + fullContextRequests: 2, + deltaRequests: 1, + websocketFailures: recoveryTransport === "sse" ? 1 : 0, + sseFallbacks: recoveryTransport === "sse" ? 1 : 0, + }); + }, + ); + + it.each([ + ["retry-after-ms", () => ({ "content-type": "application/json", "retry-after-ms": "1500" }), 1500], + ["retry-after seconds", () => ({ "content-type": "application/json", "retry-after": "60" }), 60_000], + [ + "retry-after HTTP date", + () => ({ "content-type": "application/json", "retry-after": new Date(Date.now() + 45_000).toUTCString() }), + 45_000, + ], + ] as const)("uses %s for SSE retries", async (_name, makeHeaders, expectedDelay) => { + vi.useFakeTimers(); + vi.setSystemTime(new Date("2026-05-13T00:00:00Z")); + const setTimeoutSpy = vi.spyOn(globalThis, "setTimeout"); + const token = mockToken(); + const encoder = new TextEncoder(); + const sse = buildSSEPayload({ status: "completed" }); + let codexRequests = 0; + + const fetchMock = vi.fn(async (input: string | URL) => { + const url = typeof input === "string" ? input : input.toString(); + if (url !== "https://chatgpt.com/backend-api/codex/responses") { + throw new Error(`Unexpected URL: ${url}`); + } + + codexRequests++; + if (codexRequests === 1) { + return new Response(JSON.stringify({ error: { code: "rate_limit_exceeded", message: "rate limited" } }), { + status: 429, + headers: makeHeaders(), + }); + } + + return new Response( + new ReadableStream({ + start(controller) { + controller.enqueue(encoder.encode(sse)); + controller.close(); + }, + }), + { status: 200, headers: { "content-type": "text/event-stream" } }, + ); + }); + vi.stubGlobal("fetch", fetchMock); + + const model: Model<"openai-codex-responses"> = { + id: "gpt-5.1-codex", + name: "GPT-5.1 Codex", + api: "openai-codex-responses", + provider: "openai-codex", + baseUrl: "https://chatgpt.com/backend-api", + reasoning: true, + input: ["text"], + cost: { input: 0, output: 0, cacheRead: 0, cacheWrite: 0 }, + contextWindow: 400000, + maxTokens: 128000, + }; + const context: Context = { + systemPrompt: "You are a helpful assistant.", + messages: [{ role: "user", content: "Say hello", timestamp: Date.now() }], + }; + + const resultPromise = streamOpenAICodexResponses(model, context, { + apiKey: token, + transport: "sse", + maxRetries: 1, + }).result(); + await vi.advanceTimersByTimeAsync(0); + expect(setTimeoutSpy).toHaveBeenCalledWith(expect.any(Function), expectedDelay); + + await vi.advanceTimersToNextTimerAsync(); + const result = await resultPromise; + expect(result.content.find((content) => content.type === "text")?.text).toBe("Hello"); + expect(codexRequests).toBe(2); + }); + + it.each([429, 503])("fails immediately when a %i retry delay exceeds the limit", async (status) => { + const token = mockToken(); + const fetchMock = vi.fn( + async () => + new Response(JSON.stringify({ error: { code: "temporarily_unavailable", message: "retry later" } }), { + status, + headers: { "content-type": "application/json", "retry-after": "2" }, + }), + ); + vi.stubGlobal("fetch", fetchMock); + + const model: Model<"openai-codex-responses"> = { + id: "gpt-5.1-codex", + name: "GPT-5.1 Codex", + api: "openai-codex-responses", + provider: "openai-codex", + baseUrl: "https://chatgpt.com/backend-api", + reasoning: true, + input: ["text"], + cost: { input: 0, output: 0, cacheRead: 0, cacheWrite: 0 }, + contextWindow: 400000, + maxTokens: 128000, + }; + const context: Context = { + systemPrompt: "You are a helpful assistant.", + messages: [{ role: "user", content: "Say hello", timestamp: Date.now() }], + }; + + const result = await streamOpenAICodexResponses(model, context, { + apiKey: token, + transport: "sse", + maxRetries: 3, + maxRetryDelayMs: 1000, + }).result(); + + expect(result.stopReason).toBe("error"); + expect(result.errorMessage).toBe("Server requested 2s retry delay (max: 1s)"); + expect(fetchMock).toHaveBeenCalledTimes(1); + }); + + it("zstd-compresses SSE request bodies", async () => { + const token = mockToken(); + const encoder = new TextEncoder(); + const sse = buildSSEPayload({ status: "completed" }); + + let capturedEncoding: string | null = null; + let capturedBody: Uint8Array | string | undefined; + + const fetchMock = vi.fn(async (input: string | URL, init?: RequestInit) => { + const url = typeof input === "string" ? input : input.toString(); + if (url !== "https://chatgpt.com/backend-api/codex/responses") { + throw new Error(`Unexpected URL: ${url}`); + } + const headers = init?.headers instanceof Headers ? init.headers : undefined; + capturedEncoding = headers?.get("content-encoding") ?? null; + capturedBody = init?.body as Uint8Array | string | undefined; + return new Response( + new ReadableStream({ + start(controller) { + controller.enqueue(encoder.encode(sse)); + controller.close(); + }, + }), + { status: 200, headers: { "content-type": "text/event-stream" } }, + ); + }); + vi.stubGlobal("fetch", fetchMock); + + const model: Model<"openai-codex-responses"> = { + id: "gpt-5.1-codex", + name: "GPT-5.1 Codex", + api: "openai-codex-responses", + provider: "openai-codex", + baseUrl: "https://chatgpt.com/backend-api", + reasoning: true, + input: ["text"], + cost: { input: 0, output: 0, cacheRead: 0, cacheWrite: 0 }, + contextWindow: 400000, + maxTokens: 128000, + }; + + const largeText = "compress me ".repeat(400); + await streamOpenAICodexResponses( + model, + { + systemPrompt: "You are a helpful assistant.", + messages: [{ role: "user", content: largeText, timestamp: 1 }], + }, + { apiKey: token, transport: "sse" }, + ).result(); + + expect(capturedEncoding).toBe("zstd"); + expect(capturedBody).toBeInstanceOf(Uint8Array); + const decoded = JSON.parse(Buffer.from(zstdDecompressSync(capturedBody as Uint8Array)).toString("utf8")) as { + input: Array<{ content: Array<{ text: string }> }>; + }; + expect(decoded.input[0].content[0].text).toBe(largeText); + + capturedEncoding = null; + capturedBody = undefined; + await streamOpenAICodexResponses( + model, + { + systemPrompt: "You are a helpful assistant.", + messages: [{ role: "user", content: "hi", timestamp: 1 }], + }, + { apiKey: token, transport: "sse" }, + ).result(); + + expect(capturedEncoding).toBe("zstd"); + expect(capturedBody).toBeInstanceOf(Uint8Array); + }); + + it("uses exponential backoff across repeated SSE retries without retry headers", async () => { + vi.useFakeTimers(); + vi.setSystemTime(new Date("2026-05-13T00:00:00Z")); + const setTimeoutSpy = vi.spyOn(globalThis, "setTimeout"); + const token = mockToken(); + const encoder = new TextEncoder(); + const sse = buildSSEPayload({ status: "completed" }); + let codexRequests = 0; + + const fetchMock = vi.fn(async (input: string | URL) => { + const url = typeof input === "string" ? input : input.toString(); + if (url !== "https://chatgpt.com/backend-api/codex/responses") { + throw new Error(`Unexpected URL: ${url}`); + } + + codexRequests++; + if (codexRequests <= 3) { + return new Response(JSON.stringify({ error: { code: "rate_limit_exceeded", message: "rate limited" } }), { + status: 429, + headers: { "content-type": "application/json" }, + }); + } + + return new Response( + new ReadableStream({ + start(controller) { + controller.enqueue(encoder.encode(sse)); + controller.close(); + }, + }), + { status: 200, headers: { "content-type": "text/event-stream" } }, + ); + }); + vi.stubGlobal("fetch", fetchMock); + + const model: Model<"openai-codex-responses"> = { + id: "gpt-5.1-codex", + name: "GPT-5.1 Codex", + api: "openai-codex-responses", + provider: "openai-codex", + baseUrl: "https://chatgpt.com/backend-api", + reasoning: true, + input: ["text"], + cost: { input: 0, output: 0, cacheRead: 0, cacheWrite: 0 }, + contextWindow: 400000, + maxTokens: 128000, + }; + const context: Context = { + systemPrompt: "You are a helpful assistant.", + messages: [{ role: "user", content: "Say hello", timestamp: Date.now() }], + }; + + const retryTimeoutDelays = () => + setTimeoutSpy.mock.calls + .map((call) => call[1]) + .filter((delay): delay is number => delay === 1000 || delay === 2000 || delay === 4000); + + const resultPromise = streamOpenAICodexResponses(model, context, { + apiKey: token, + transport: "sse", + maxRetries: 3, + }).result(); + await vi.advanceTimersByTimeAsync(0); + expect(retryTimeoutDelays()).toEqual([1000]); + + await vi.advanceTimersToNextTimerAsync(); + expect(retryTimeoutDelays()).toEqual([1000, 2000]); + + await vi.advanceTimersToNextTimerAsync(); + expect(retryTimeoutDelays()).toEqual([1000, 2000, 4000]); + + await vi.advanceTimersToNextTimerAsync(); + const result = await resultPromise; + expect(result.content.find((content) => content.type === "text")?.text).toBe("Hello"); + expect(codexRequests).toBe(4); + }); +}); diff --git a/packages/ai/test/openai-completions-cache-control-format.test.ts b/packages/ai/test/openai-completions-cache-control-format.test.ts new file mode 100644 index 0000000000000000000000000000000000000000..4a26f055c01e1d895a0a99d1e6b1ecbc42eff985 --- /dev/null +++ b/packages/ai/test/openai-completions-cache-control-format.test.ts @@ -0,0 +1,231 @@ +import { Type } from "typebox"; +import { beforeEach, describe, expect, it, vi } from "vitest"; +import { stream as streamOpenAICompletions } from "../src/api/openai-completions.ts"; +import { getModel } from "../src/compat.ts"; +import type { Message, Model } from "../src/types.ts"; + +interface CacheControl { + type: "ephemeral"; + ttl?: string; +} + +interface TextPart { + type: "text"; + text: string; + cache_control?: CacheControl; +} + +interface ToolWithCacheControl { + type: string; + cache_control?: CacheControl; +} + +interface CapturedParams { + messages: Array<{ + role: string; + content: string | TextPart[] | null; + }>; + tools?: ToolWithCacheControl[]; +} + +const mockState = vi.hoisted(() => ({ + lastParams: undefined as CapturedParams | undefined, +})); + +vi.mock("openai", () => { + class FakeOpenAI { + chat = { + completions: { + create: (params: CapturedParams) => { + mockState.lastParams = params; + const stream = { + async *[Symbol.asyncIterator]() { + yield { + id: "chatcmpl-test", + choices: [{ delta: {}, finish_reason: "stop" }], + usage: { + prompt_tokens: 1, + completion_tokens: 1, + prompt_tokens_details: { cached_tokens: 0 }, + completion_tokens_details: { reasoning_tokens: 0 }, + }, + }; + }, + }; + const promise = Promise.resolve(stream) as Promise & { + withResponse: () => Promise<{ + data: typeof stream; + response: { status: number; headers: Headers }; + }>; + }; + promise.withResponse = async () => ({ + data: stream, + response: { status: 200, headers: new Headers() }, + }); + return promise; + }, + }, + }; + } + + return { default: FakeOpenAI }; +}); + +async function capturePayload( + model: Model<"openai-completions">, + options?: { cacheRetention?: "none" | "short" | "long" }, + messages?: Message[], +): Promise { + const timestamp = Date.now(); + + await streamOpenAICompletions( + model, + { + systemPrompt: "System prompt", + messages: messages ?? [{ role: "user", content: "Hello", timestamp }], + tools: [ + { + name: "read", + description: "Read a file", + parameters: Type.Object({ + path: Type.String(), + }), + }, + ], + }, + { apiKey: "test-key", ...options }, + ).result(); + + if (!mockState.lastParams) { + throw new Error("Expected payload to be captured"); + } + + return mockState.lastParams; +} + +function getInstructionMessage(params: CapturedParams) { + return params.messages.find((message) => message.role === "system" || message.role === "developer"); +} + +function expectAnthropicCacheMarkers(params: CapturedParams): void { + const instructionMessage = getInstructionMessage(params); + expect(instructionMessage).toBeDefined(); + expect(Array.isArray(instructionMessage?.content)).toBe(true); + expect((instructionMessage?.content as TextPart[])[0]?.cache_control).toEqual({ type: "ephemeral" }); + + expect(params.tools).toHaveLength(1); + expect(params.tools?.[0]?.cache_control).toEqual({ type: "ephemeral" }); + + const lastMessage = params.messages[params.messages.length - 1]; + expect(lastMessage.role).toBe("user"); + expect(Array.isArray(lastMessage.content)).toBe(true); + expect((lastMessage.content as TextPart[])[0]?.cache_control).toEqual({ type: "ephemeral" }); +} + +describe("openai-completions cacheControlFormat", () => { + beforeEach(() => { + mockState.lastParams = undefined; + }); + + it("applies Anthropic-style cache markers when model compat enables them", async () => { + const model: Model<"openai-completions"> = { + id: "custom-qwen", + name: "Custom Qwen", + api: "openai-completions", + provider: "openrouter", + baseUrl: "https://example.com/v1", + reasoning: true, + input: ["text"], + cost: { + input: 0, + output: 0, + cacheRead: 0, + cacheWrite: 0, + }, + contextWindow: 128000, + maxTokens: 32000, + compat: { + cacheControlFormat: "anthropic", + }, + }; + + const params = await capturePayload(model); + expectAnthropicCacheMarkers(params); + }); + + it("preserves Anthropic-style cache markers for OpenRouter Anthropic batch aliases", async () => { + const model = getModel("openrouter", "anthropic/claude-fable-5.1:batch"); + const params = await capturePayload(model); + expectAnthropicCacheMarkers(params); + }); + + it("moves the conversation cache marker to a tool result", async () => { + const model = getModel("openrouter", "anthropic/claude-fable-5.1:batch"); + const timestamp = Date.now(); + const params = await capturePayload(model, undefined, [ + { role: "user", content: "Read the file", timestamp }, + { + role: "assistant", + content: [{ type: "toolCall", id: "call_1", name: "read", arguments: { path: "README.md" } }], + api: "openai-completions", + provider: "openrouter", + model: model.id, + usage: { + input: 0, + output: 0, + cacheRead: 0, + cacheWrite: 0, + totalTokens: 0, + cost: { input: 0, output: 0, cacheRead: 0, cacheWrite: 0, total: 0 }, + }, + stopReason: "toolUse", + timestamp, + }, + { + role: "toolResult", + toolCallId: "call_1", + toolName: "read", + content: [{ type: "text", text: "file contents" }], + isError: false, + timestamp, + }, + ]); + + const userMessage = params.messages.find((message) => message.role === "user"); + expect(userMessage?.content).toBe("Read the file"); + + const toolMessage = params.messages[params.messages.length - 1]; + expect(toolMessage.role).toBe("tool"); + expect(Array.isArray(toolMessage.content)).toBe(true); + expect((toolMessage.content as TextPart[])[0]?.cache_control).toEqual({ type: "ephemeral" }); + }); + + it("omits Anthropic-style cache markers when cacheRetention is none", async () => { + const model: Model<"openai-completions"> = { + id: "custom-qwen", + name: "Custom Qwen", + api: "openai-completions", + provider: "openrouter", + baseUrl: "https://example.com/v1", + reasoning: true, + input: ["text"], + cost: { + input: 0, + output: 0, + cacheRead: 0, + cacheWrite: 0, + }, + contextWindow: 128000, + maxTokens: 32000, + compat: { + cacheControlFormat: "anthropic", + }, + }; + const params = await capturePayload(model, { cacheRetention: "none" }); + const instructionMessage = getInstructionMessage(params); + + expect(Array.isArray(instructionMessage?.content)).toBe(false); + expect(params.tools?.[0]?.cache_control).toBeUndefined(); + expect(typeof params.messages[params.messages.length - 1]?.content).toBe("string"); + }); +}); diff --git a/packages/ai/test/openai-completions-empty-tools.test.ts b/packages/ai/test/openai-completions-empty-tools.test.ts new file mode 100644 index 0000000000000000000000000000000000000000..92235e2d8b4b688e28ef46d3d36f85e0822355ba --- /dev/null +++ b/packages/ai/test/openai-completions-empty-tools.test.ts @@ -0,0 +1,304 @@ +import { beforeEach, describe, expect, it, vi } from "vitest"; +import { getModel, streamSimple } from "../src/compat.ts"; + +// Empty tools arrays must NOT be serialized as `tools: []` — some OpenAI-compatible +// backends (e.g. DashScope / Aliyun Qwen via compatible-mode) reject the request with +// `"[] is too short - 'tools'"` (HTTP 400) when `--no-tools` produces an empty array. +// Regression for https://github.com/earendil-works/pi/issues/3649 + +const mockState = vi.hoisted(() => ({ + lastParams: undefined as unknown, + lastClientOptions: undefined as unknown, +})); + +vi.mock("openai", () => { + class FakeOpenAI { + constructor(options: unknown) { + mockState.lastClientOptions = options; + } + + chat = { + completions: { + create: (params: unknown) => { + mockState.lastParams = params; + const stream = { + async *[Symbol.asyncIterator]() { + yield { + choices: [{ delta: {}, finish_reason: "stop" }], + usage: { + prompt_tokens: 1, + completion_tokens: 1, + prompt_tokens_details: { cached_tokens: 0 }, + completion_tokens_details: { reasoning_tokens: 0 }, + }, + }; + }, + }; + const promise = Promise.resolve(stream) as Promise & { + withResponse: () => Promise<{ + data: typeof stream; + response: { status: number; headers: Headers }; + }>; + }; + promise.withResponse = async () => ({ + data: stream, + response: { status: 200, headers: new Headers() }, + }); + return promise; + }, + }, + }; + } + + return { default: FakeOpenAI }; +}); + +describe("openai-completions empty tools handling", () => { + beforeEach(() => { + mockState.lastParams = undefined; + mockState.lastClientOptions = undefined; + }); + + it("omits tools field when context.tools is an empty array", async () => { + const { compat: _compat, ...baseModel } = getModel("openai", "gpt-4o-mini")!; + const model = { ...baseModel, api: "openai-completions" } as const; + + await streamSimple( + model, + { + messages: [{ role: "user", content: "hi", timestamp: Date.now() }], + tools: [], + }, + { apiKey: "test" }, + ).result(); + + const params = mockState.lastParams as { tools?: unknown }; + expect("tools" in (params as object)).toBe(false); + }); + + it("omits tools field when context.tools is undefined", async () => { + const { compat: _compat, ...baseModel } = getModel("openai", "gpt-4o-mini")!; + const model = { ...baseModel, api: "openai-completions" } as const; + + await streamSimple( + model, + { + messages: [{ role: "user", content: "hi", timestamp: Date.now() }], + }, + { apiKey: "test" }, + ).result(); + + const params = mockState.lastParams as { tools?: unknown }; + expect("tools" in (params as object)).toBe(false); + }); + + it("sends default maxTokens", async () => { + const { compat: _compat, ...baseModel } = getModel("openai", "gpt-4o-mini")!; + const model = { ...baseModel, api: "openai-completions" } as const; + + await streamSimple( + model, + { + messages: [{ role: "user", content: "hi", timestamp: Date.now() }], + }, + { apiKey: "test" }, + ).result(); + + const params = mockState.lastParams as { max_tokens?: number; max_completion_tokens?: number }; + expect(params.max_tokens).toBeUndefined(); + expect(params.max_completion_tokens).toBe(model.maxTokens); + }); + + it("sends explicit maxTokens", async () => { + const { compat: _compat, ...baseModel } = getModel("openai", "gpt-4o-mini")!; + const model = { ...baseModel, api: "openai-completions" } as const; + + await streamSimple( + model, + { + messages: [{ role: "user", content: "hi", timestamp: Date.now() }], + }, + { apiKey: "test", maxTokens: 1234 }, + ).result(); + + const params = mockState.lastParams as { max_tokens?: number; max_completion_tokens?: number }; + expect(params.max_tokens).toBeUndefined(); + expect(params.max_completion_tokens).toBe(1234); + }); + + it("clamps default maxTokens to remaining context", async () => { + const { compat: _compat, ...baseModel } = getModel("openai", "gpt-4o-mini")!; + const model = { ...baseModel, api: "openai-completions", contextWindow: 10000, maxTokens: 8000 } as const; + + await streamSimple( + model, + { + messages: [{ role: "user", content: "x".repeat(8000), timestamp: Date.now() }], + }, + { apiKey: "test" }, + ).result(); + + const params = mockState.lastParams as { max_tokens?: number; max_completion_tokens?: number }; + expect(params.max_tokens).toBeUndefined(); + expect(params.max_completion_tokens).toBe(3904); + }); + + it("clamps explicit maxTokens to remaining context", async () => { + const { compat: _compat, ...baseModel } = getModel("openai", "gpt-4o-mini")!; + const model = { ...baseModel, api: "openai-completions", contextWindow: 10000, maxTokens: 8000 } as const; + + await streamSimple( + model, + { + messages: [{ role: "user", content: "x".repeat(8000), timestamp: Date.now() }], + }, + { apiKey: "test", maxTokens: 7000 }, + ).result(); + + const params = mockState.lastParams as { max_tokens?: number; max_completion_tokens?: number }; + expect(params.max_tokens).toBeUndefined(); + expect(params.max_completion_tokens).toBe(3904); + }); + + it("uses conservative OpenAI-compatible fields for Cloudflare AI Gateway /compat models", async () => { + process.env.CLOUDFLARE_API_KEY = "cf-token"; + process.env.CLOUDFLARE_ACCOUNT_ID = "account-id"; + process.env.CLOUDFLARE_GATEWAY_ID = "gateway-id"; + const model = getModel("cloudflare-ai-gateway", "workers-ai/@cf/moonshotai/kimi-k2.6")!; + + await streamSimple( + model, + { + systemPrompt: "You are helpful.", + messages: [{ role: "user", content: "hi", timestamp: Date.now() }], + }, + { maxTokens: 1234, reasoning: "high" }, + ).result(); + + const params = mockState.lastParams as { + messages: Array<{ role: string }>; + max_tokens?: number; + max_completion_tokens?: number; + reasoning_effort?: string; + store?: boolean; + }; + expect(params.messages[0].role).toBe("system"); + expect(params.max_tokens).toBe(1234); + expect(params.max_completion_tokens).toBeUndefined(); + expect(params.reasoning_effort).toBeUndefined(); + expect(params.store).toBeUndefined(); + + const clientOptions = mockState.lastClientOptions as { + baseURL?: string; + defaultHeaders?: Record; + }; + expect(clientOptions.baseURL).toBe("https://gateway.ai.cloudflare.com/v1/account-id/gateway-id/compat"); + expect(clientOptions.defaultHeaders?.Authorization).toBeNull(); + expect(clientOptions.defaultHeaders?.["cf-aig-authorization"]).toBe("Bearer cf-token"); + }); + + it("resolves Cloudflare AI Gateway base URL through provider auth", async () => { + process.env.CLOUDFLARE_API_KEY = "cf-token"; + process.env.CLOUDFLARE_ACCOUNT_ID = "account-id"; + process.env.CLOUDFLARE_GATEWAY_ID = "gateway-id"; + const model = getModel("cloudflare-ai-gateway", "workers-ai/@cf/moonshotai/kimi-k2.6")!; + + await streamSimple(model, { + messages: [{ role: "user", content: "hi", timestamp: Date.now() }], + }).result(); + + const clientOptions = mockState.lastClientOptions as { baseURL?: string }; + expect(clientOptions.baseURL).toBe("https://gateway.ai.cloudflare.com/v1/account-id/gateway-id/compat"); + }); + + it("preserves inline upstream Authorization for Cloudflare AI Gateway BYOK requests", async () => { + process.env.CLOUDFLARE_API_KEY = "cf-token"; + process.env.CLOUDFLARE_ACCOUNT_ID = "account-id"; + process.env.CLOUDFLARE_GATEWAY_ID = "gateway-id"; + const model = getModel("cloudflare-ai-gateway", "gpt-5.1")!; + + await streamSimple( + model, + { + messages: [{ role: "user", content: "hi", timestamp: Date.now() }], + }, + { headers: { Authorization: "Bearer upstream-token" } }, + ).result(); + + const clientOptions = mockState.lastClientOptions as { defaultHeaders?: Record }; + expect(clientOptions.defaultHeaders?.Authorization).toBe("Bearer upstream-token"); + expect(clientOptions.defaultHeaders?.["cf-aig-authorization"]).toBe("Bearer cf-token"); + }); + + it("sends session affinity headers for Workers AI through Cloudflare AI Gateway", async () => { + process.env.CLOUDFLARE_API_KEY = "cf-token"; + process.env.CLOUDFLARE_ACCOUNT_ID = "account-id"; + process.env.CLOUDFLARE_GATEWAY_ID = "gateway-id"; + const workersModel = getModel("cloudflare-ai-gateway", "workers-ai/@cf/moonshotai/kimi-k2.6")!; + + await streamSimple( + workersModel, + { + messages: [{ role: "user", content: "hi", timestamp: Date.now() }], + }, + { sessionId: "session-1" }, + ).result(); + + const clientOptions = mockState.lastClientOptions as { defaultHeaders?: Record }; + expect(clientOptions.defaultHeaders?.session_id).toBe("session-1"); + expect(clientOptions.defaultHeaders?.["x-client-request-id"]).toBe("session-1"); + expect(clientOptions.defaultHeaders?.["x-session-affinity"]).toBe("session-1"); + }); + + it("still emits tools: [] for Anthropic/LiteLLM proxy when conversation has tool history", async () => { + const { compat: _compat, ...baseModel } = getModel("openai", "gpt-4o-mini")!; + const model = { ...baseModel, api: "openai-completions" } as const; + + await streamSimple( + model, + { + messages: [ + { role: "user", content: "use the tool", timestamp: Date.now() }, + { + role: "assistant", + content: [ + { + type: "toolCall", + id: "t1", + name: "noop", + arguments: {}, + }, + ], + stopReason: "toolUse", + usage: { + input: 0, + output: 0, + cacheRead: 0, + cacheWrite: 0, + totalTokens: 0, + cost: { input: 0, output: 0, cacheRead: 0, cacheWrite: 0, total: 0 }, + }, + api: "openai-completions", + provider: "openai", + model: "gpt-4o-mini", + timestamp: Date.now(), + }, + { + role: "toolResult", + toolCallId: "t1", + toolName: "noop", + content: [{ type: "text", text: "done" }], + isError: false, + timestamp: Date.now(), + }, + ], + tools: [], + }, + { apiKey: "test" }, + ).result(); + + const params = mockState.lastParams as { tools?: unknown[] }; + expect(Array.isArray(params.tools)).toBe(true); + expect(params.tools).toEqual([]); + }); +}); diff --git a/packages/ai/test/openai-completions-reasoning-details.test.ts b/packages/ai/test/openai-completions-reasoning-details.test.ts new file mode 100644 index 0000000000000000000000000000000000000000..a76c3b1dd8e2e0560a282f073be0d7a9cffec838 --- /dev/null +++ b/packages/ai/test/openai-completions-reasoning-details.test.ts @@ -0,0 +1,251 @@ +import { Type } from "typebox"; +import { beforeEach, describe, expect, it, vi } from "vitest"; +import { stream as streamOpenAICompletions } from "../src/api/openai-completions.ts"; +import type { AssistantMessage, Model, Tool } from "../src/types.ts"; + +const mockState = vi.hoisted(() => ({ + chunkSets: [] as unknown[][], + payloads: [] as unknown[], +})); + +vi.mock("openai", () => { + class FakeOpenAI { + chat = { + completions: { + create: (payload: unknown) => { + mockState.payloads.push(payload); + const chunks = mockState.chunkSets.shift() ?? []; + const stream = { + async *[Symbol.asyncIterator]() { + for (const chunk of chunks) { + yield chunk; + } + }, + }; + const result = Promise.resolve(stream) as Promise & { + withResponse: () => Promise<{ data: typeof stream; response: { status: number; headers: Headers } }>; + }; + result.withResponse = async () => ({ + data: stream, + response: { status: 200, headers: new Headers() }, + }); + return result; + }, + }, + }; + } + return { default: FakeOpenAI }; +}); + +const reasoningDetail = { type: "reasoning.encrypted", id: "call_1", data: "encrypted-signature" }; +const signedReasoningTextDetail = { + type: "reasoning.text", + text: "I should call the read tool.", + signature: "sha256:signed-text", + id: "reasoning-text-1", + format: "anthropic-claude-v1", + index: 0, +}; +const reasoningSummaryDetail = { + type: "reasoning.summary", + summary: "Decided to inspect the requested file.", + id: "reasoning-summary-1", + format: "anthropic-claude-v1", + index: 1, +}; +const readTool: Tool = { + name: "read", + description: "Read a file", + parameters: Type.Object({ path: Type.String() }), +}; + +function model(): Model<"openai-completions"> { + return { + id: "google/gemini-test", + name: "Gemini Test", + api: "openai-completions", + provider: "openrouter", + baseUrl: "https://openrouter.ai/api/v1", + reasoning: true, + input: ["text"], + cost: { input: 0, output: 0, cacheRead: 0, cacheWrite: 0 }, + contextWindow: 100_000, + maxTokens: 4096, + }; +} + +function chunk(delta: Record, finishReason: string | null = null): unknown { + return { + id: "chatcmpl-test", + model: "google/gemini-test", + choices: [{ index: 0, delta, finish_reason: finishReason }], + }; +} + +function toolCallChunk(): unknown { + return chunk({ + tool_calls: [ + { + index: 0, + id: "call_1", + type: "function", + function: { name: "read", arguments: '{"path":"README.md"}' }, + }, + ], + }); +} + +async function runOpenAICompletionsStream(messages: AssistantMessage[] = []): Promise { + return await streamOpenAICompletions(model(), { messages, tools: [readTool] }, { apiKey: "test" }).result(); +} + +function getAssistantPayload(payload: unknown): { reasoning?: unknown; reasoning_details?: unknown } | undefined { + const messages = ( + payload as { messages?: Array<{ role?: string; reasoning?: unknown; reasoning_details?: unknown }> } + ).messages; + return messages?.find((message) => message.role === "assistant"); +} + +describe("openai-completions reasoning_details streaming", () => { + beforeEach(() => { + mockState.chunkSets = []; + mockState.payloads = []; + }); + + it("preserves reasoning_details in the thinking signature", async () => { + mockState.chunkSets = [ + [chunk({ reasoning_details: [reasoningDetail] }), toolCallChunk(), chunk({}, "tool_calls")], + [chunk({ content: "ok" }), chunk({}, "stop")], + ]; + + const assistantMessage = await runOpenAICompletionsStream(); + const thinking = assistantMessage.content.find((block) => block.type === "thinking"); + expect(thinking).toEqual({ + type: "thinking", + thinking: "", + thinkingSignature: JSON.stringify([reasoningDetail]), + }); + const toolCall = assistantMessage.content.find((block) => block.type === "toolCall"); + expect(toolCall).toEqual({ + type: "toolCall", + id: "call_1", + name: "read", + arguments: { path: "README.md" }, + }); + + await runOpenAICompletionsStream([assistantMessage]); + + expect(getAssistantPayload(mockState.payloads[1])?.reasoning_details).toEqual([reasoningDetail]); + }); + + it("falls back to encrypted tool-call signatures for older stored assistant messages", async () => { + mockState.chunkSets = [ + [chunk({ reasoning_details: [reasoningDetail] }), toolCallChunk(), chunk({}, "tool_calls")], + [chunk({ content: "ok" }), chunk({}, "stop")], + ]; + + const assistantMessage = await runOpenAICompletionsStream(); + assistantMessage.content = assistantMessage.content.filter((block) => block.type !== "thinking"); + const toolCall = assistantMessage.content.find((block) => block.type === "toolCall"); + if (!toolCall || toolCall.type !== "toolCall") throw new Error("Expected tool call"); + toolCall.thoughtSignature = JSON.stringify(reasoningDetail); + + await runOpenAICompletionsStream([assistantMessage]); + + expect(getAssistantPayload(mockState.payloads[1])?.reasoning_details).toEqual([reasoningDetail]); + }); + + it("preserves signed text and summary reasoning_details in their original sequence", async () => { + mockState.chunkSets = [ + [ + chunk({ reasoning: signedReasoningTextDetail.text, reasoning_details: [signedReasoningTextDetail] }), + chunk({ reasoning_details: [reasoningDetail, reasoningSummaryDetail] }), + toolCallChunk(), + chunk({}, "tool_calls"), + ], + [chunk({ content: "ok" }), chunk({}, "stop")], + ]; + + const assistantMessage = await runOpenAICompletionsStream(); + const expectedReasoningDetails = [signedReasoningTextDetail, reasoningDetail, reasoningSummaryDetail]; + const thinking = assistantMessage.content.find((block) => block.type === "thinking"); + expect(thinking).toEqual({ + type: "thinking", + thinking: signedReasoningTextDetail.text, + thinkingSignature: JSON.stringify(expectedReasoningDetails), + }); + + await runOpenAICompletionsStream([assistantMessage]); + + const payload = getAssistantPayload(mockState.payloads[1]); + expect(payload?.reasoning_details).toEqual(expectedReasoningDetails); + expect(payload?.reasoning).toBeUndefined(); + }); + + it("merges consecutive text and summary reasoning_details deltas before replay", async () => { + const textDelta = { type: "reasoning.text", text: "The", index: 0 }; + const textDeltaWithSignature = { + type: "reasoning.text", + text: " user wants the time.", + signature: "sha256:text-signature", + format: "openai-responses-v1", + index: 0, + }; + const summaryDelta = { type: "reasoning.summary", summary: "Looked", index: 0 }; + const summaryDeltaWithFormat = { + type: "reasoning.summary", + summary: " up time.", + format: "openai-responses-v1", + index: 0, + }; + const laterSummaryDelta = { + type: "reasoning.summary", + summary: "After encrypted block.", + format: "openai-responses-v1", + index: 0, + }; + const expectedReasoningDetails = [ + { + type: "reasoning.text", + text: "The user wants the time.", + index: 0, + signature: "sha256:text-signature", + format: "openai-responses-v1", + }, + { + type: "reasoning.summary", + summary: "Looked up time.", + index: 0, + format: "openai-responses-v1", + }, + reasoningDetail, + laterSummaryDelta, + ]; + + mockState.chunkSets = [ + [ + chunk({ reasoning_details: [textDelta] }), + chunk({ reasoning_details: [textDeltaWithSignature] }), + chunk({ reasoning_details: [summaryDelta] }), + chunk({ reasoning_details: [summaryDeltaWithFormat] }), + chunk({ reasoning_details: [reasoningDetail] }), + chunk({ reasoning_details: [laterSummaryDelta] }), + toolCallChunk(), + chunk({}, "tool_calls"), + ], + [chunk({ content: "ok" }), chunk({}, "stop")], + ]; + + const assistantMessage = await runOpenAICompletionsStream(); + const thinking = assistantMessage.content.find((block) => block.type === "thinking"); + expect(thinking).toEqual({ + type: "thinking", + thinking: "", + thinkingSignature: JSON.stringify(expectedReasoningDetails), + }); + + await runOpenAICompletionsStream([assistantMessage]); + + expect(getAssistantPayload(mockState.payloads[1])?.reasoning_details).toEqual(expectedReasoningDetails); + }); +}); diff --git a/packages/ai/test/openai-completions-response-model.test.ts b/packages/ai/test/openai-completions-response-model.test.ts new file mode 100644 index 0000000000000000000000000000000000000000..5139bd54d2e7d035f315c6de9c90aa987abcd0c2 --- /dev/null +++ b/packages/ai/test/openai-completions-response-model.test.ts @@ -0,0 +1,140 @@ +import { beforeEach, describe, expect, it, vi } from "vitest"; +import { complete } from "../src/compat.ts"; +import type { Model } from "../src/types.ts"; + +// Router/virtual ids (e.g. OpenRouter `auto`) keep `model` pinned to the +// requested id and surface the routed concrete id on `responseModel`. + +const mockState = vi.hoisted(() => ({ + chunks: [] as unknown[], +})); + +vi.mock("openai", () => { + class FakeOpenAI { + chat = { + completions: { + create: () => { + const chunks = mockState.chunks; + const stream = { + async *[Symbol.asyncIterator]() { + for (const chunk of chunks) yield chunk; + }, + }; + const promise = Promise.resolve(stream) as Promise & { + withResponse: () => Promise<{ + data: typeof stream; + response: { status: number; headers: Headers }; + }>; + }; + promise.withResponse = async () => ({ + data: stream, + response: { status: 200, headers: new Headers() }, + }); + return promise; + }, + }, + }; + } + return { default: FakeOpenAI }; +}); + +function openRouterAuto(): Model<"openai-completions"> { + return { + id: "openrouter/auto", + name: "OpenRouter Auto", + api: "openai-completions", + provider: "openrouter", + baseUrl: "https://openrouter.ai/api/v1", + reasoning: false, + input: ["text"], + cost: { input: 0, output: 0, cacheRead: 0, cacheWrite: 0 }, + contextWindow: 200_000, + maxTokens: 8192, + }; +} + +describe("openai-completions responseModel", () => { + beforeEach(() => { + mockState.chunks = []; + }); + + it("surfaces routed chunk.model on responseModel without changing model", async () => { + mockState.chunks = [ + { id: "chatcmpl-1", model: "anthropic/claude-opus-4.8", choices: [{ index: 0, delta: { content: "hi" } }] }, + { + id: "chatcmpl-1", + model: "anthropic/claude-opus-4.8", + choices: [{ index: 0, delta: {}, finish_reason: "stop" }], + usage: { + prompt_tokens: 10, + completion_tokens: 5, + prompt_tokens_details: { cached_tokens: 0 }, + completion_tokens_details: { reasoning_tokens: 0 }, + }, + }, + ]; + + const message = await complete( + openRouterAuto(), + { messages: [{ role: "user", content: "hi", timestamp: Date.now() }] }, + { apiKey: "test" }, + ); + + expect(message.model).toBe("openrouter/auto"); + expect(message.responseModel).toBe("anthropic/claude-opus-4.8"); + expect(message.provider).toBe("openrouter"); + expect(message.stopReason).toBe("stop"); + }); + + it("leaves responseModel undefined when chunks echo the requested id", async () => { + mockState.chunks = [ + { id: "chatcmpl-2", model: "openrouter/auto", choices: [{ index: 0, delta: { content: "hi" } }] }, + { + id: "chatcmpl-2", + model: "openrouter/auto", + choices: [{ index: 0, delta: {}, finish_reason: "stop" }], + usage: { + prompt_tokens: 1, + completion_tokens: 1, + prompt_tokens_details: { cached_tokens: 0 }, + completion_tokens_details: { reasoning_tokens: 0 }, + }, + }, + ]; + + const message = await complete( + openRouterAuto(), + { messages: [{ role: "user", content: "hi", timestamp: Date.now() }] }, + { apiKey: "test" }, + ); + + expect(message.model).toBe("openrouter/auto"); + expect(message.responseModel).toBeUndefined(); + }); + + it("ignores empty or missing chunk.model", async () => { + mockState.chunks = [ + { id: "chatcmpl-3", choices: [{ index: 0, delta: { content: "hi" } }] }, + { id: "chatcmpl-3", model: "", choices: [{ index: 0, delta: { content: "!" } }] }, + { + id: "chatcmpl-3", + choices: [{ index: 0, delta: {}, finish_reason: "stop" }], + usage: { + prompt_tokens: 1, + completion_tokens: 2, + prompt_tokens_details: { cached_tokens: 0 }, + completion_tokens_details: { reasoning_tokens: 0 }, + }, + }, + ]; + + const message = await complete( + openRouterAuto(), + { messages: [{ role: "user", content: "hi", timestamp: Date.now() }] }, + { apiKey: "test" }, + ); + + expect(message.model).toBe("openrouter/auto"); + expect(message.responseModel).toBeUndefined(); + }); +}); diff --git a/packages/ai/test/openai-completions-thinking-as-text.test.ts b/packages/ai/test/openai-completions-thinking-as-text.test.ts new file mode 100644 index 0000000000000000000000000000000000000000..eabaed1c5a86bdda0d956ddbc693fd6c6bd39074 --- /dev/null +++ b/packages/ai/test/openai-completions-thinking-as-text.test.ts @@ -0,0 +1,227 @@ +import { once } from "node:events"; +import http from "node:http"; +import type { AddressInfo } from "node:net"; +import { afterEach, describe, expect, it } from "vitest"; +import { convertMessages, stream as streamOpenAICompletions } from "../src/api/openai-completions.ts"; +import type { + AssistantMessage, + AssistantMessageEvent, + Context, + Model, + OpenAICompletionsCompat, + Usage, +} from "../src/types.ts"; + +const emptyUsage: Usage = { + input: 0, + output: 0, + cacheRead: 0, + cacheWrite: 0, + totalTokens: 0, + cost: { input: 0, output: 0, cacheRead: 0, cacheWrite: 0, total: 0 }, +}; + +const compat = { + supportsStore: true, + supportsDeveloperRole: true, + supportsReasoningEffort: true, + supportsUsageInStreaming: true, + supportsFinishReason: true, + maxTokensField: "max_completion_tokens", + requiresToolResultName: false, + requiresAssistantAfterToolResult: false, + requiresThinkingAsText: true, + requiresReasoningContentOnAssistantMessages: false, + thinkingFormat: "openai", + openRouterRouting: {}, + vercelGatewayRouting: {}, + chatTemplateKwargs: {}, + chatTemplateArgs: {}, + zaiToolStream: false, + supportsThinkingTokenBudget: false, + thinkingTokenBudgetField: undefined, + supportsStrictMode: true, + supportsOpenAIGrammarTools: false, + cacheControlFormat: undefined, + sendSessionAffinityHeaders: false, + sessionAffinityFormat: "openai", + supportsLongCacheRetention: true, +} satisfies Omit< + Required, + "cacheControlFormat" | "deferredToolsMode" | "thinkingTokenBudgetField" | "vllmPriority" +> & { + cacheControlFormat?: OpenAICompletionsCompat["cacheControlFormat"]; + deferredToolsMode?: OpenAICompletionsCompat["deferredToolsMode"]; + thinkingTokenBudgetField?: OpenAICompletionsCompat["thinkingTokenBudgetField"]; +}; + +function buildModel(baseUrl = "http://127.0.0.1:1"): Model<"openai-completions"> { + return { + id: "repro-model", + name: "Repro Model", + api: "openai-completions", + provider: "repro-provider", + baseUrl, + reasoning: true, + input: ["text"], + cost: { input: 0, output: 0, cacheRead: 0, cacheWrite: 0 }, + contextWindow: 128000, + maxTokens: 4096, + compat, + }; +} + +function buildAssistant(content: AssistantMessage["content"]): AssistantMessage { + return { + role: "assistant", + content, + api: "openai-completions", + provider: "repro-provider", + model: "repro-model", + usage: emptyUsage, + stopReason: "stop", + timestamp: 2, + }; +} + +function buildContext(assistant: AssistantMessage): Context { + return { + messages: [ + { role: "user", content: "hello", timestamp: 1 }, + assistant, + { role: "user", content: "continue", timestamp: 3 }, + ], + }; +} + +async function collectEvents(stream: AsyncIterable): Promise { + const events: AssistantMessageEvent[] = []; + for await (const event of stream) { + events.push(event); + } + return events; +} + +interface ChatCompletionsRequestBody { + model: string; + messages: Array<{ role: string; content?: unknown }>; + stream: boolean; + stream_options?: { include_usage?: boolean }; +} + +describe("openai-completions thinking-as-text replay", () => { + afterEach(() => { + delete process.env.OPENAI_API_KEY; + }); + + it("serializes same-model thinking-plus-text replay as assistant text parts", () => { + const messages = convertMessages( + buildModel(), + buildContext( + buildAssistant([ + { type: "thinking", thinking: "internal reasoning" }, + { type: "text", text: "visible answer" }, + ]), + ), + compat, + ); + + expect(messages[1]).toEqual({ + role: "assistant", + content: [ + { type: "text", text: "internal reasoning" }, + { type: "text", text: "visible answer" }, + ], + }); + }); + + it("serializes same-model thinking-only replay as assistant text parts", () => { + const messages = convertMessages( + buildModel(), + buildContext(buildAssistant([{ type: "thinking", thinking: "internal reasoning" }])), + compat, + ); + + expect(messages[1]).toEqual({ + role: "assistant", + content: [{ type: "text", text: "internal reasoning" }], + }); + }); + + it("reaches the endpoint when replay contains both thinking and text", async () => { + const requestBodies: ChatCompletionsRequestBody[] = []; + const server = http.createServer(async (req, res) => { + if (req.method !== "POST" || req.url !== "/chat/completions") { + res.writeHead(404).end(); + return; + } + + let body = ""; + for await (const chunk of req) { + body += chunk.toString(); + } + requestBodies.push(JSON.parse(body) as ChatCompletionsRequestBody); + + res.writeHead(200, { + "content-type": "text/event-stream", + "cache-control": "no-cache", + connection: "keep-alive", + }); + res.write( + `data: ${JSON.stringify({ + id: "chatcmpl-repro", + object: "chat.completion.chunk", + created: 0, + model: "repro-model", + choices: [{ index: 0, delta: { role: "assistant", content: "ok" }, finish_reason: null }], + })}\n\n`, + ); + res.write( + `data: ${JSON.stringify({ + id: "chatcmpl-repro", + object: "chat.completion.chunk", + created: 0, + model: "repro-model", + choices: [{ index: 0, delta: {}, finish_reason: "stop" }], + usage: { prompt_tokens: 1, completion_tokens: 1 }, + })}\n\n`, + ); + res.write("data: [DONE]\n\n"); + res.end(); + }); + + server.listen(0, "127.0.0.1"); + await once(server, "listening"); + + try { + const { port } = server.address() as AddressInfo; + const events = await collectEvents( + streamOpenAICompletions( + buildModel(`http://127.0.0.1:${port}`), + buildContext( + buildAssistant([ + { type: "thinking", thinking: "internal reasoning" }, + { type: "text", text: "visible answer" }, + ]), + ), + { apiKey: "test-key" }, + ), + ); + + expect(requestBodies).toHaveLength(1); + expect(requestBodies[0]?.messages[1]).toEqual({ + role: "assistant", + content: [ + { type: "text", text: "internal reasoning" }, + { type: "text", text: "visible answer" }, + ], + }); + + const terminalEvent = events.at(-1); + expect(terminalEvent?.type).toBe("done"); + } finally { + server.close(); + await once(server, "close"); + } + }); +}); diff --git a/packages/ai/test/openai-completions-tool-choice.test.ts b/packages/ai/test/openai-completions-tool-choice.test.ts new file mode 100644 index 0000000000000000000000000000000000000000..065bd4a2202b495beadec577ed5e43aab60fe039 --- /dev/null +++ b/packages/ai/test/openai-completions-tool-choice.test.ts @@ -0,0 +1,1880 @@ +import { Type } from "typebox"; +import { beforeEach, describe, expect, it, vi } from "vitest"; +import { convertMessages } from "../src/api/openai-completions.ts"; +import { getModel, stream, streamSimple } from "../src/compat.ts"; +import type { AssistantMessage, Model, SimpleStreamOptions, Tool, ToolResultMessage } from "../src/types.ts"; + +const mockState = vi.hoisted(() => ({ + lastParams: undefined as unknown, + chunks: undefined as + | Array; finish_reason: string | null; usage?: unknown }>; + usage?: { + prompt_tokens: number; + completion_tokens: number; + prompt_tokens_details: { cached_tokens: number; cache_write_tokens?: number }; + completion_tokens_details: { reasoning_tokens: number }; + }; + }> + | undefined, +})); + +vi.mock("openai", () => { + class FakeOpenAI { + chat = { + completions: { + create: (params: unknown) => { + mockState.lastParams = params; + const stream = { + async *[Symbol.asyncIterator]() { + const chunks = mockState.chunks ?? [ + { + choices: [{ delta: {}, finish_reason: "stop" }], + usage: { + prompt_tokens: 1, + completion_tokens: 1, + prompt_tokens_details: { cached_tokens: 0 }, + completion_tokens_details: { reasoning_tokens: 0 }, + }, + }, + ]; + for (const chunk of chunks) { + yield chunk; + } + }, + }; + const promise = Promise.resolve(stream) as Promise & { + withResponse: () => Promise<{ + data: typeof stream; + response: { status: number; headers: Headers }; + }>; + }; + promise.withResponse = async () => ({ + data: stream, + response: { status: 200, headers: new Headers() }, + }); + return promise; + }, + }, + }; + } + + return { default: FakeOpenAI }; +}); + +const localOpenAICompletionsModel = { + api: "openai-completions", + provider: "local-vllm", + baseUrl: "http://localhost:8000/v1", + reasoning: true, + input: ["text"], + cost: { input: 0, output: 0, cacheRead: 0, cacheWrite: 0 }, + contextWindow: 128000, + maxTokens: 8192, +} satisfies Omit, "id" | "name" | "compat">; + +type CapturedParams = { + chat_template_kwargs?: Record; + thinking?: unknown; + reasoning_effort?: string; +}; + +async function captureSimpleParams( + model: Model<"openai-completions">, + reasoning?: SimpleStreamOptions["reasoning"], +): Promise { + let payload: unknown; + + await streamSimple( + model, + { + messages: [{ role: "user", content: "Hi", timestamp: Date.now() }], + }, + { + apiKey: "test", + reasoning, + onPayload: (params: unknown) => { + payload = params; + }, + }, + ).result(); + + return (payload ?? mockState.lastParams) as CapturedParams; +} + +describe("openai-completions tool_choice", () => { + beforeEach(() => { + mockState.lastParams = undefined; + mockState.chunks = undefined; + }); + + it("forwards toolChoice from simple options to payload", async () => { + const { compat: _compat, ...baseModel } = getModel("openai", "gpt-4o-mini")!; + const model = { ...baseModel, api: "openai-completions" } as const; + const tools: Tool[] = [ + { + name: "ping", + description: "Ping tool", + parameters: Type.Object({ + ok: Type.Boolean(), + }), + }, + ]; + let payload: unknown; + + await streamSimple( + model, + { + messages: [ + { + role: "user", + content: "Call ping with ok=true", + timestamp: Date.now(), + }, + ], + tools, + }, + { + apiKey: "test", + toolChoice: "required", + onPayload: (params: unknown) => { + payload = params; + }, + } as unknown as Parameters[2], + ).result(); + + const params = (payload ?? mockState.lastParams) as { tool_choice?: string; tools?: unknown[] }; + expect(params.tool_choice).toBe("required"); + expect(Array.isArray(params.tools)).toBe(true); + expect(params.tools?.length ?? 0).toBeGreaterThan(0); + }); + + it("includes toolChoice when no tools are provided", async () => { + const { compat: _compat, ...baseModel } = getModel("openai", "gpt-4o-mini")!; + const model = { ...baseModel, api: "openai-completions" } as const; + let payload: unknown; + + await streamSimple( + model, + { + messages: [{ role: "user", content: "Summarize the conversation", timestamp: Date.now() }], + }, + { + apiKey: "test", + toolChoice: "none", + onPayload: (params: unknown) => { + payload = params; + }, + }, + ).result(); + + const params = (payload ?? mockState.lastParams) as { tool_choice?: string; tools?: unknown[] }; + expect(params.tool_choice).toBe("none"); + expect(params).not.toHaveProperty("tools"); + }); + + it("omits strict when compat disables strict mode", async () => { + const { compat: _compat, ...baseModel } = getModel("openai", "gpt-4o-mini")!; + const model = { + ...baseModel, + api: "openai-completions", + compat: { supportsStrictMode: false }, + } as const; + const tools: Tool[] = [ + { + name: "ping", + description: "Ping tool", + parameters: Type.Object({ + ok: Type.Boolean(), + }), + }, + ]; + let payload: unknown; + + await streamSimple( + model, + { + messages: [ + { + role: "user", + content: "Call ping with ok=true", + timestamp: Date.now(), + }, + ], + tools, + }, + { + apiKey: "test", + onPayload: (params: unknown) => { + payload = params; + }, + } as unknown as Parameters[2], + ).result(); + + const params = (payload ?? mockState.lastParams) as { tools?: Array<{ function?: Record }> }; + const tool = params.tools?.[0]?.function; + expect(tool).toBeTruthy(); + expect(tool?.strict).toBeUndefined(); + expect("strict" in (tool ?? {})).toBe(false); + }); + + it("maps Groq Qwen reasoning levels to default reasoning_effort", async () => { + const model = getModel("groq", "qwen/qwen3.6-27b")!; + let payload: unknown; + + await streamSimple( + model, + { + messages: [ + { + role: "user", + content: "Hi", + timestamp: Date.now(), + }, + ], + }, + { + apiKey: "test", + reasoning: "medium", + onPayload: (params: unknown) => { + payload = params; + }, + }, + ).result(); + + const params = (payload ?? mockState.lastParams) as { reasoning_effort?: string }; + expect(params.reasoning_effort).toBe("default"); + }); + + it("keeps normal reasoning_effort for groq models without compat mapping", async () => { + const model = getModel("groq", "openai/gpt-oss-20b")!; + let payload: unknown; + + await streamSimple( + model, + { + messages: [ + { + role: "user", + content: "Hi", + timestamp: Date.now(), + }, + ], + }, + { + apiKey: "test", + reasoning: "medium", + onPayload: (params: unknown) => { + payload = params; + }, + }, + ).result(); + + const params = (payload ?? mockState.lastParams) as { reasoning_effort?: string }; + expect(params.reasoning_effort).toBe("medium"); + }); + + it("enables tool_stream for supported z.ai models with tools", async () => { + const model = getModel("zai", "glm-5.2")!; + const tools: Tool[] = [ + { + name: "ping", + description: "Ping tool", + parameters: Type.Object({ + ok: Type.Boolean(), + }), + }, + ]; + let payload: unknown; + + await streamSimple( + model, + { + messages: [ + { + role: "user", + content: "Call ping with ok=true", + timestamp: Date.now(), + }, + ], + tools, + }, + { + apiKey: "test", + onPayload: (params: unknown) => { + payload = params; + }, + }, + ).result(); + + const params = (payload ?? mockState.lastParams) as { tool_stream?: boolean }; + expect(params.tool_stream).toBe(true); + }); + + it("stores z.ai tool_stream support in model compat metadata", () => { + expect(getModel("zai", "glm-4.7")?.compat?.zaiToolStream).toBe(true); + expect(getModel("zai", "glm-4.7")?.compat?.zaiToolStream).toBe(true); + expect(getModel("zai", "glm-5-turbo")?.compat?.zaiToolStream).toBe(true); + expect(getModel("zai", "glm-5.2")?.compat?.zaiToolStream).toBe(true); + }); + + it("stores z.ai effort metadata", () => { + for (const provider of ["zai", "zai-coding-cn"] as const) { + for (const modelId of ["glm-5.2", "glm-5.2-highspeed"] as const) { + const model = getModel(provider, modelId)!; + expect(model.compat?.supportsReasoningEffort).toBe(true); + expect(model.thinkingLevelMap).toEqual({ + off: "none", + minimal: null, + low: null, + medium: null, + high: "high", + xhigh: null, + max: "max", + }); + } + + const glm53 = getModel(provider, "glm-5.3")!; + expect(glm53.compat?.supportsReasoningEffort).toBe(true); + expect(glm53.thinkingLevelMap).toEqual({ + off: null, + minimal: null, + low: "low", + medium: null, + high: "high", + xhigh: null, + max: "max", + }); + } + }); + + it("maps z.ai GLM-5.2 thinking levels to reasoning_effort", async () => { + const model = getModel("zai", "glm-5.2")!; + const cases = [ + { reasoning: "low", effort: "high" }, + { reasoning: "medium", effort: "high" }, + { reasoning: "high", effort: "high" }, + { reasoning: "max", effort: "max" }, + ] as const; + + for (const testCase of cases) { + let payload: unknown; + + await streamSimple( + model, + { + messages: [ + { + role: "user", + content: "Hi", + timestamp: Date.now(), + }, + ], + }, + { + apiKey: "test", + reasoning: testCase.reasoning, + onPayload: (params: unknown) => { + payload = params; + }, + }, + ).result(); + + const params = (payload ?? mockState.lastParams) as { thinking?: unknown; reasoning_effort?: string }; + expect(params.thinking).toEqual({ type: "enabled", clear_thinking: false }); + expect(params.reasoning_effort).toBe(testCase.effort); + } + }); + + it("preserves z.ai thinking when replaying reasoning_content", async () => { + const model = getModel("zai", "glm-5.2")!; + const assistantMessage: AssistantMessage = { + role: "assistant", + api: "openai-completions", + provider: "zai", + model: "glm-5.2", + content: [ + { type: "thinking", thinking: "prior reasoning", thinkingSignature: "reasoning_content" }, + { type: "toolCall", id: "call_1", name: "read", arguments: { path: "README.md" } }, + ], + usage: { + input: 0, + output: 0, + cacheRead: 0, + cacheWrite: 0, + totalTokens: 0, + cost: { input: 0, output: 0, cacheRead: 0, cacheWrite: 0, total: 0 }, + }, + stopReason: "toolUse", + timestamp: Date.now(), + }; + const toolResult: ToolResultMessage = { + role: "toolResult", + toolCallId: "call_1", + toolName: "read", + content: [{ type: "text", text: "contents" }], + isError: false, + timestamp: Date.now(), + }; + let payload: unknown; + + await streamSimple( + model, + { + messages: [ + { role: "user", content: "Read README.md", timestamp: Date.now() }, + assistantMessage, + toolResult, + { role: "user", content: "Continue", timestamp: Date.now() }, + ], + }, + { + apiKey: "test", + reasoning: "high", + onPayload: (params: unknown) => { + payload = params; + }, + }, + ).result(); + + const params = (payload ?? mockState.lastParams) as { + messages?: Array>; + thinking?: unknown; + }; + const replayedAssistant = params.messages?.find((message) => message.role === "assistant"); + expect(replayedAssistant).toMatchObject({ reasoning_content: "prior reasoning" }); + expect(params.thinking).toEqual({ type: "enabled", clear_thinking: false }); + }); + + it("omits z.ai GLM-5.2 reasoning_effort when thinking is off", async () => { + const model = getModel("zai", "glm-5.2")!; + let payload: unknown; + + await streamSimple( + model, + { + messages: [ + { + role: "user", + content: "Hi", + timestamp: Date.now(), + }, + ], + }, + { + apiKey: "test", + onPayload: (params: unknown) => { + payload = params; + }, + }, + ).result(); + + const params = (payload ?? mockState.lastParams) as { thinking?: unknown; reasoning_effort?: string }; + expect(params.thinking).toEqual({ type: "disabled" }); + expect(params.reasoning_effort).toBeUndefined(); + }); + + it("respects explicit z.ai tool_stream compat override", async () => { + const baseModel = getModel("zai", "glm-5.2")!; + const model = { + ...baseModel, + compat: { + ...baseModel.compat, + zaiToolStream: true, + }, + } as const; + const tools: Tool[] = [ + { + name: "ping", + description: "Ping tool", + parameters: Type.Object({ + ok: Type.Boolean(), + }), + }, + ]; + let payload: unknown; + + await streamSimple( + model, + { + messages: [ + { + role: "user", + content: "Call ping with ok=true", + timestamp: Date.now(), + }, + ], + tools, + }, + { + apiKey: "test", + onPayload: (params: unknown) => { + payload = params; + }, + }, + ).result(); + + const params = (payload ?? mockState.lastParams) as { tool_stream?: boolean }; + expect(params.tool_stream).toBe(true); + }); + + it("omits tool_stream when no tools are provided", async () => { + const model = getModel("zai", "glm-5.2")!; + let payload: unknown; + + await streamSimple( + model, + { + messages: [ + { + role: "user", + content: "Hi", + timestamp: Date.now(), + }, + ], + }, + { + apiKey: "test", + onPayload: (params: unknown) => { + payload = params; + }, + }, + ).result(); + + const params = (payload ?? mockState.lastParams) as { tool_stream?: boolean }; + expect(params.tool_stream).toBeUndefined(); + }); + + it("maps non-standard provider finish_reason values to stopReason error", async () => { + mockState.chunks = [ + { + choices: [{ delta: { content: "partial" }, finish_reason: null }], + }, + { + choices: [{ delta: {}, finish_reason: "network_error" }], + usage: { + prompt_tokens: 1, + completion_tokens: 1, + prompt_tokens_details: { cached_tokens: 0 }, + completion_tokens_details: { reasoning_tokens: 0 }, + }, + }, + ]; + + const model = getModel("zai", "glm-5.2")!; + const response = await streamSimple( + model, + { + messages: [ + { + role: "user", + content: "Hi", + timestamp: Date.now(), + }, + ], + }, + { apiKey: "test" }, + ).result(); + + expect(response.stopReason).toBe("error"); + expect(response.errorMessage).toBe("Provider finish_reason: network_error"); + }); + + it("ignores null stream chunks from openai-compatible providers", async () => { + mockState.chunks = [ + null, + { + id: "chatcmpl-test", + choices: [{ delta: { content: "OK" }, finish_reason: null }], + }, + { + id: "chatcmpl-test", + choices: [{ delta: {}, finish_reason: "stop" }], + usage: { + prompt_tokens: 3, + completion_tokens: 1, + prompt_tokens_details: { cached_tokens: 0 }, + completion_tokens_details: { reasoning_tokens: 0 }, + }, + }, + ]; + + const { compat: _compat, ...baseModel } = getModel("openai", "gpt-4o-mini")!; + const model = { ...baseModel, api: "openai-completions" } as const; + const response = await streamSimple( + model, + { + messages: [ + { + role: "user", + content: "Reply with exactly OK", + timestamp: Date.now(), + }, + ], + }, + { apiKey: "test" }, + ).result(); + + expect(response.stopReason).toBe("stop"); + expect(response.errorMessage).toBeUndefined(); + expect(response.responseId).toBe("chatcmpl-test"); + expect(response.usage.totalTokens).toBe(4); + expect(response.content).toEqual([{ type: "text", text: "OK" }]); + }); + + it("errors when a stream ends after only null finish_reason chunks", async () => { + mockState.chunks = [ + { + id: "chatcmpl-truncated", + choices: [{ delta: { content: "partial answer" }, finish_reason: null }], + }, + { + id: "chatcmpl-truncated", + choices: [{ delta: { content: "partial answer" }, finish_reason: null }], + }, + ]; + + const { compat: _compat, ...baseModel } = getModel("openai", "gpt-4o-mini")!; + const model = { ...baseModel, api: "openai-completions" } as const; + const response = await streamSimple( + model, + { + messages: [ + { + role: "user", + content: "Reply with a longer sentence", + timestamp: Date.now(), + }, + ], + }, + { apiKey: "test" }, + ).result(); + + expect(response.stopReason).toBe("error"); + expect(response.errorMessage).toBe("Stream ended without finish_reason"); + }); + + it("accepts streams without finish_reason when compat disables it", async () => { + mockState.chunks = [ + { + id: "chatcmpl-no-finish-reason", + choices: [{ delta: { content: "complete answer" }, finish_reason: null }], + }, + ]; + + const { compat: _compat, ...baseModel } = getModel("openai", "gpt-4o-mini")!; + const model = { + ...baseModel, + api: "openai-completions", + compat: { supportsFinishReason: false }, + } as const; + const response = await streamSimple( + model, + { + messages: [{ role: "user", content: "Reply with a complete answer", timestamp: Date.now() }], + }, + { apiKey: "test" }, + ).result(); + + expect(response.stopReason).toBe("stop"); + expect(response.errorMessage).toBeUndefined(); + expect(response.content).toEqual([{ type: "text", text: "complete answer" }]); + }); + + it("ignores empty custom objects on function tool call deltas", async () => { + mockState.chunks = [ + { + id: "chatcmpl-empty-custom", + choices: [ + { + delta: { + tool_calls: [ + { + index: 0, + id: "call_1", + type: "function", + function: { name: "read", arguments: '{"path":"README.md"}' }, + custom: {}, + }, + ], + }, + finish_reason: "tool_calls", + }, + ], + }, + ]; + + const { compat: _compat, ...baseModel } = getModel("openai", "gpt-4o-mini")!; + const model = { ...baseModel, api: "openai-completions" } as const; + const tool: Tool = { + name: "read", + description: "Read a file", + parameters: Type.Object({ path: Type.String() }), + }; + const response = await streamSimple( + model, + { + messages: [{ role: "user", content: "Read README.md", timestamp: Date.now() }], + tools: [tool], + }, + { apiKey: "test" }, + ).result(); + + expect(response.content).toEqual([ + { + type: "toolCall", + id: "call_1", + name: "read", + arguments: { path: "README.md" }, + }, + ]); + }); + + it("coalesces tool call deltas by stable index when provider mutates ids mid-stream", async () => { + mockState.chunks = [ + { + id: "chatcmpl-kimi-bad-stream", + choices: [ + { + delta: { + tool_calls: [ + { + index: 0, + id: "functions.read:0", + type: "function", + function: { name: "read", arguments: "" }, + }, + ], + }, + finish_reason: null, + }, + ], + }, + { + id: "chatcmpl-kimi-bad-stream", + choices: [ + { + delta: { + tool_calls: [ + { + index: 0, + id: "chatcmpl-tool-a", + type: "function", + function: { name: null, arguments: '{"path":"README' }, + }, + ], + }, + finish_reason: null, + }, + ], + }, + { + id: "chatcmpl-kimi-bad-stream", + choices: [ + { + delta: { + tool_calls: [ + { + index: 0, + id: "chatcmpl-tool-b", + type: "function", + function: { name: null, arguments: '.md"}' }, + }, + ], + }, + finish_reason: "tool_calls", + }, + ], + usage: { + prompt_tokens: 10, + completion_tokens: 5, + prompt_tokens_details: { cached_tokens: 0 }, + completion_tokens_details: { reasoning_tokens: 0 }, + }, + }, + ]; + + const { compat: _compat, ...baseModel } = getModel("openai", "gpt-4o-mini")!; + const model = { ...baseModel, api: "openai-completions" } as const; + const tool: Tool = { + name: "read", + description: "Read a file", + parameters: Type.Object({ + path: Type.String(), + }), + }; + const s = streamSimple( + model, + { + messages: [ + { + role: "user", + content: "Read README.md", + timestamp: Date.now(), + }, + ], + tools: [tool], + }, + { apiKey: "test" }, + ); + + const toolCallContentIndexes: number[] = []; + for await (const event of s) { + if (event.type === "toolcall_start" || event.type === "toolcall_delta" || event.type === "toolcall_end") { + toolCallContentIndexes.push(event.contentIndex); + } + } + + const response = await s.result(); + expect(response.stopReason).toBe("toolUse"); + expect(toolCallContentIndexes).toEqual([0, 0, 0, 0, 0]); + expect(response.content).toHaveLength(1); + const toolCall = response.content[0]; + expect(toolCall.type).toBe("toolCall"); + if (toolCall.type !== "toolCall") { + throw new Error("Expected toolCall content"); + } + expect(toolCall.id).toBe("functions.read:0"); + expect(toolCall.name).toBe("read"); + expect(toolCall.arguments).toEqual({ path: "README.md" }); + expect(toolCall).not.toHaveProperty("streamIndex"); + expect(toolCall).not.toHaveProperty("partialArgs"); + }); + + it("accumulates mixed content, reasoning, and parallel tool call deltas independently", async () => { + mockState.chunks = [ + { + id: "chatcmpl-mixed-deltas", + choices: [ + { + delta: { + content: "answer 1", + reasoning_content: "think 1", + tool_calls: [ + { + index: 0, + id: "tc_read_initial", + type: "function", + function: { name: "read", arguments: '{"path":"README' }, + }, + { + index: 1, + id: "tc_grep_initial", + type: "function", + function: { name: "grep", arguments: '{"pattern":"TODO' }, + }, + { + id: "tc_list_no_index", + type: "function", + function: { name: "list", arguments: '{"path":"packages' }, + }, + { + id: "tc_write_no_index", + type: "function", + function: { name: "write", arguments: '{"path":"out' }, + }, + ], + }, + finish_reason: null, + }, + ], + }, + { + id: "chatcmpl-mixed-deltas", + choices: [ + { + delta: { + content: " answer 2", + tool_calls: [ + { + index: 1, + id: "tc_grep_changed", + type: "function", + function: { arguments: '","path":"src' }, + }, + { + id: "tc_write_no_index", + type: "function", + function: { arguments: '.txt","content":"ok"}' }, + }, + { + id: "tc_list_no_index", + type: "function", + function: { arguments: '/ai"}' }, + }, + ], + }, + finish_reason: null, + }, + ], + }, + { + id: "chatcmpl-mixed-deltas", + choices: [ + { + delta: { + content: "\n", + reasoning_content: " think 2", + tool_calls: [ + { + index: 0, + id: "tc_read_changed", + type: "function", + function: { arguments: '.md"}' }, + }, + { + index: 1, + type: "function", + function: { arguments: '"}' }, + }, + ], + }, + finish_reason: "tool_calls", + }, + ], + usage: { + prompt_tokens: 10, + completion_tokens: 8, + prompt_tokens_details: { cached_tokens: 0 }, + completion_tokens_details: { reasoning_tokens: 2 }, + }, + }, + ]; + + const { compat: _compat, ...baseModel } = getModel("openai", "gpt-4o-mini")!; + const model = { ...baseModel, api: "openai-completions" } as const; + const tools: Tool[] = [ + { + name: "read", + description: "Read a file", + parameters: Type.Object({ path: Type.String() }), + }, + { + name: "grep", + description: "Search a file", + parameters: Type.Object({ pattern: Type.String(), path: Type.String() }), + }, + { + name: "list", + description: "List a directory", + parameters: Type.Object({ path: Type.String() }), + }, + { + name: "write", + description: "Write a file", + parameters: Type.Object({ path: Type.String(), content: Type.String() }), + }, + ]; + const s = streamSimple( + model, + { + messages: [ + { + role: "user", + content: "Think, answer, and use tools.", + timestamp: Date.now(), + }, + ], + tools, + }, + { apiKey: "test" }, + ); + + const eventTypes: string[] = []; + const toolEventsByContentIndex = new Map(); + for await (const event of s) { + eventTypes.push(event.type); + if (event.type === "toolcall_start" || event.type === "toolcall_delta" || event.type === "toolcall_end") { + const events = toolEventsByContentIndex.get(event.contentIndex) ?? []; + events.push(event.type); + toolEventsByContentIndex.set(event.contentIndex, events); + } + } + + const response = await s.result(); + expect(response.stopReason).toBe("toolUse"); + expect(eventTypes.filter((type) => type === "text_start")).toHaveLength(1); + expect(eventTypes.filter((type) => type === "text_delta")).toHaveLength(3); + expect(eventTypes.filter((type) => type === "text_end")).toHaveLength(1); + expect(eventTypes.filter((type) => type === "thinking_start")).toHaveLength(1); + expect(eventTypes.filter((type) => type === "thinking_delta")).toHaveLength(2); + expect(eventTypes.filter((type) => type === "thinking_end")).toHaveLength(1); + expect(eventTypes.filter((type) => type === "toolcall_start")).toHaveLength(4); + expect(eventTypes.filter((type) => type === "toolcall_delta")).toHaveLength(9); + expect(eventTypes.filter((type) => type === "toolcall_end")).toHaveLength(4); + expect(toolEventsByContentIndex.get(2)).toEqual([ + "toolcall_start", + "toolcall_delta", + "toolcall_delta", + "toolcall_end", + ]); + expect(toolEventsByContentIndex.get(3)).toEqual([ + "toolcall_start", + "toolcall_delta", + "toolcall_delta", + "toolcall_delta", + "toolcall_end", + ]); + expect(toolEventsByContentIndex.get(4)).toEqual([ + "toolcall_start", + "toolcall_delta", + "toolcall_delta", + "toolcall_end", + ]); + expect(toolEventsByContentIndex.get(5)).toEqual([ + "toolcall_start", + "toolcall_delta", + "toolcall_delta", + "toolcall_end", + ]); + + expect(response.content).toHaveLength(6); + expect(response.content[0]).toEqual({ type: "text", text: "answer 1 answer 2\n" }); + expect(response.content[1]).toEqual({ + type: "thinking", + thinking: "think 1 think 2", + thinkingSignature: "reasoning_content", + }); + const readCall = response.content[2]; + const grepCall = response.content[3]; + const listCall = response.content[4]; + const writeCall = response.content[5]; + expect(readCall.type).toBe("toolCall"); + expect(grepCall.type).toBe("toolCall"); + expect(listCall.type).toBe("toolCall"); + expect(writeCall.type).toBe("toolCall"); + if ( + readCall.type !== "toolCall" || + grepCall.type !== "toolCall" || + listCall.type !== "toolCall" || + writeCall.type !== "toolCall" + ) { + throw new Error("Expected toolCall content"); + } + expect(readCall.id).toBe("tc_read_initial"); + expect(readCall.name).toBe("read"); + expect(readCall.arguments).toEqual({ path: "README.md" }); + expect(readCall).not.toHaveProperty("streamIndex"); + expect(readCall).not.toHaveProperty("partialArgs"); + expect(grepCall.id).toBe("tc_grep_initial"); + expect(grepCall.name).toBe("grep"); + expect(grepCall.arguments).toEqual({ pattern: "TODO", path: "src" }); + expect(grepCall).not.toHaveProperty("streamIndex"); + expect(grepCall).not.toHaveProperty("partialArgs"); + expect(listCall.id).toBe("tc_list_no_index"); + expect(listCall.name).toBe("list"); + expect(listCall.arguments).toEqual({ path: "packages/ai" }); + expect(listCall).not.toHaveProperty("streamIndex"); + expect(listCall).not.toHaveProperty("partialArgs"); + expect(writeCall.id).toBe("tc_write_no_index"); + expect(writeCall.name).toBe("write"); + expect(writeCall.arguments).toEqual({ path: "out.txt", content: "ok" }); + expect(writeCall).not.toHaveProperty("streamIndex"); + expect(writeCall).not.toHaveProperty("partialArgs"); + }); + + it("uses system messages for non-OpenAI/Anthropic OpenRouter reasoning model instructions", async () => { + const model = getModel("openrouter", "deepseek/deepseek-v4-pro")!; + let payload: unknown; + + await streamSimple( + model, + { + systemPrompt: "Follow instructions.", + messages: [{ role: "user", content: "Hi", timestamp: Date.now() }], + }, + { + apiKey: "test", + onPayload: (params: unknown) => { + payload = params; + }, + }, + ).result(); + + const params = payload as { messages?: Array<{ role?: string }> }; + expect(params.messages?.[0]?.role).toBe("system"); + }); + + it("keeps developer messages for OpenAI and Anthropic OpenRouter batch instructions", async () => { + for (const model of [ + getModel("openrouter", "openai/gpt-5.2-codex"), + getModel("openrouter", "anthropic/claude-fable-5.1:batch"), + ]) { + expect(model).toBeDefined(); + let payload: unknown; + + await streamSimple( + model!, + { + systemPrompt: "Follow instructions.", + messages: [{ role: "user", content: "Hi", timestamp: Date.now() }], + }, + { + apiKey: "test", + onPayload: (params: unknown) => { + payload = params; + }, + }, + ).result(); + + const params = payload as { messages?: Array<{ role?: string }> }; + expect(params.messages?.[0]?.role).toBe("developer"); + } + }); + + it("keeps developer messages for OpenAI reasoning model instructions", async () => { + const { compat: _compat, ...baseModel } = getModel("openai", "gpt-5.5")!; + const model = { ...baseModel, api: "openai-completions" } as const; + let payload: unknown; + + await streamSimple( + model, + { + systemPrompt: "Follow instructions.", + messages: [{ role: "user", content: "Hi", timestamp: Date.now() }], + }, + { + apiKey: "test", + onPayload: (params: unknown) => { + payload = params; + }, + }, + ).result(); + + const params = payload as { messages?: Array<{ role?: string }> }; + expect(params.messages?.[0]?.role).toBe("developer"); + }); + + it("stores OpenRouter Kimi K2.6 reasoning replay compat in built-in metadata", () => { + // `:free` variant delisted from the OpenRouter API; the generator override + // matches any `moonshotai/kimi-k2.6*` variant that is listed. + const model = getModel("openrouter", "moonshotai/kimi-k2.6")!; + expect(model.compat?.supportsDeveloperRole).toBe(false); + expect(model.compat?.requiresReasoningContentOnAssistantMessages).toBe(true); + }); + + it("stores Xiaomi MiMo reasoning replay compat in built-in metadata", () => { + const providers = ["xiaomi", "xiaomi-token-plan-cn", "xiaomi-token-plan-ams", "xiaomi-token-plan-sgp"] as const; + + for (const provider of providers) { + const model = getModel(provider, "mimo-v2.5-pro")!; + expect(model.compat?.requiresReasoningContentOnAssistantMessages).toBe(true); + expect(model.compat?.thinkingFormat).toBe("deepseek"); + expect(model.compat?.maxTokensField).toBeUndefined(); + expect(model.compat?.supportsDeveloperRole).toBeUndefined(); + } + }); + + it("stores Qwen Token Plan reasoning replay compat in built-in metadata", () => { + const providers = ["qwen-token-plan", "qwen-token-plan-cn", "qwen-token-plan-individual"] as const; + + for (const provider of providers) { + const model = getModel(provider, "qwen3.7-max")!; + expect(model.compat?.thinkingFormat).toBe("qwen"); + expect(model.compat?.requiresReasoningContentOnAssistantMessages).toBeUndefined(); + expect(model.compat?.supportsDeveloperRole).toBe(false); + expect(model.compat?.supportsStore).toBe(false); + } + }); + + it("replays Xiaomi MiMo assistant tool calls with empty reasoning_content when thinking is missing", async () => { + const model = getModel("xiaomi", "mimo-v2.5-pro")!; + const assistantMessage: AssistantMessage = { + role: "assistant", + api: "openai-completions", + provider: "xiaomi", + model: "mimo-v2.5-pro", + content: [{ type: "toolCall", id: "call_1", name: "read", arguments: { path: "README.md" } }], + usage: { + input: 0, + output: 0, + cacheRead: 0, + cacheWrite: 0, + totalTokens: 0, + cost: { input: 0, output: 0, cacheRead: 0, cacheWrite: 0, total: 0 }, + }, + stopReason: "toolUse", + timestamp: Date.now(), + }; + const toolResult: ToolResultMessage = { + role: "toolResult", + toolCallId: "call_1", + toolName: "read", + content: [{ type: "text", text: "contents" }], + isError: false, + timestamp: Date.now(), + }; + let payload: unknown; + + await streamSimple( + model, + { + messages: [ + { role: "user", content: "Read README.md", timestamp: Date.now() }, + assistantMessage, + toolResult, + ], + }, + { + apiKey: "test", + reasoning: "high", + onPayload: (params: unknown) => { + payload = params; + }, + }, + ).result(); + + const params = (payload ?? mockState.lastParams) as { + messages?: Array>; + thinking?: { type?: string }; + reasoning_effort?: string; + }; + const replayedAssistant = params.messages?.find((message) => message.role === "assistant"); + expect(replayedAssistant).toMatchObject({ role: "assistant", reasoning_content: "" }); + expect(params.thinking).toEqual({ type: "enabled" }); + expect(params.reasoning_effort).toBe("high"); + }); + + it("normalizes OpenCode Go reasoning deltas to reasoning_content for replay", async () => { + mockState.chunks = [ + { + id: "chatcmpl-opencode-go-reasoning", + choices: [{ delta: { reasoning: "think" }, finish_reason: "stop" }], + }, + ]; + + const { compat: _compat, ...baseModel } = getModel("opencode-go", "kimi-k2.6")!; + const model = { ...baseModel, api: "openai-completions" } as const; + const response = await streamSimple( + model, + { + messages: [{ role: "user", content: "Use reasoning.", timestamp: Date.now() }], + }, + { apiKey: "test" }, + ).result(); + + expect(response.content).toEqual([ + { + type: "thinking", + thinking: "think", + thinkingSignature: "reasoning_content", + }, + ]); + }); + + it("keeps non-OpenCode Go reasoning deltas on the original reasoning field", async () => { + mockState.chunks = [ + { + id: "chatcmpl-reasoning", + choices: [{ delta: { reasoning: "think" }, finish_reason: "stop" }], + }, + ]; + + const { compat: _compat, ...baseModel } = getModel("openai", "gpt-4o-mini")!; + const model = { ...baseModel, api: "openai-completions" } as const; + const response = await streamSimple( + model, + { + messages: [{ role: "user", content: "Use reasoning.", timestamp: Date.now() }], + }, + { apiKey: "test" }, + ).result(); + + expect(response.content).toEqual([ + { + type: "thinking", + thinking: "think", + thinkingSignature: "reasoning", + }, + ]); + }); + + it("replays OpenCode Go reasoning thinking blocks as reasoning_content", () => { + const { compat: _compat, ...baseModel } = getModel("opencode-go", "kimi-k2.6")!; + const model = { ...baseModel, api: "openai-completions" } as Model<"openai-completions">; + const messages = convertMessages( + model, + { + messages: [ + { + role: "assistant", + api: "openai-completions", + provider: "opencode-go", + model: "kimi-k2.6", + content: [ + { type: "thinking", thinking: "think", thinkingSignature: "reasoning" }, + { type: "toolCall", id: "call_1", name: "read", arguments: { path: "README.md" } }, + ], + usage: { + input: 0, + output: 0, + cacheRead: 0, + cacheWrite: 0, + totalTokens: 0, + cost: { input: 0, output: 0, cacheRead: 0, cacheWrite: 0, total: 0 }, + }, + stopReason: "stop", + timestamp: Date.now(), + }, + ], + }, + { + ...model.compat, + supportsStore: false, + supportsDeveloperRole: false, + supportsReasoningEffort: true, + supportsUsageInStreaming: true, + supportsFinishReason: true, + maxTokensField: "max_completion_tokens", + requiresToolResultName: false, + requiresAssistantAfterToolResult: false, + requiresThinkingAsText: false, + requiresReasoningContentOnAssistantMessages: false, + thinkingFormat: "openai", + openRouterRouting: {}, + vercelGatewayRouting: {}, + chatTemplateKwargs: {}, + chatTemplateArgs: {}, + zaiToolStream: false, + supportsStrictMode: true, + supportsOpenAIGrammarTools: false, + sendSessionAffinityHeaders: false, + sessionAffinityFormat: "openai", + supportsLongCacheRetention: true, + }, + ); + + expect(messages[0]).toMatchObject({ role: "assistant", reasoning_content: "think" }); + expect(messages[0]).not.toHaveProperty("reasoning"); + }); + + it("sends thinking disabled for OpenCode Go Kimi K2.6 when thinking is off", async () => { + const model = getModel("opencode-go", "kimi-k2.6")!; + let payload: unknown; + + await streamSimple( + model, + { + messages: [{ role: "user", content: "Hi", timestamp: Date.now() }], + }, + { + apiKey: "test", + onPayload: (params: unknown) => { + payload = params; + }, + }, + ).result(); + + const params = (payload ?? mockState.lastParams) as { thinking?: unknown; reasoning_effort?: string }; + expect(params.thinking).toEqual({ type: "disabled" }); + expect(params.reasoning_effort).toBeUndefined(); + }); + + it("sends thinking enabled for OpenCode Go Kimi K2.6 when thinking is enabled", async () => { + const model = getModel("opencode-go", "kimi-k2.6")!; + let payload: unknown; + + await streamSimple( + model, + { + messages: [{ role: "user", content: "Hi", timestamp: Date.now() }], + }, + { + apiKey: "test", + reasoning: "high", + onPayload: (params: unknown) => { + payload = params; + }, + }, + ).result(); + + const params = (payload ?? mockState.lastParams) as { thinking?: unknown; reasoning_effort?: string }; + expect(params.thinking).toEqual({ type: "enabled" }); + expect(params.reasoning_effort).toBeUndefined(); + }); + + it("omits disabled thinking for Moonshot Kimi K2.7 Code models", async () => { + const cases = [getModel("moonshotai", "kimi-k2.7-code"), getModel("moonshotai-cn", "kimi-k2.7-code")]; + + for (const model of cases) { + expect(model).toBeDefined(); + let payload: unknown; + + await streamSimple( + model!, + { + messages: [{ role: "user", content: "Hi", timestamp: Date.now() }], + }, + { + apiKey: "test", + onPayload: (params: unknown) => { + payload = params; + }, + }, + ).result(); + + const params = (payload ?? mockState.lastParams) as { thinking?: unknown; reasoning_effort?: string }; + expect(params.thinking).toBeUndefined(); + expect(params.reasoning_effort).toBeUndefined(); + } + }); + + it("keeps disabled thinking for Moonshot Kimi K2.6 when thinking is off", async () => { + const model = getModel("moonshotai-cn", "kimi-k2.6")!; + let payload: unknown; + + await streamSimple( + model, + { + messages: [{ role: "user", content: "Hi", timestamp: Date.now() }], + }, + { + apiKey: "test", + onPayload: (params: unknown) => { + payload = params; + }, + }, + ).result(); + + const params = (payload ?? mockState.lastParams) as { thinking?: unknown; reasoning_effort?: string }; + expect(params.thinking).toEqual({ type: "disabled" }); + expect(params.reasoning_effort).toBeUndefined(); + }); + + it("sends max_tokens for OpenCode completions models", async () => { + const cases = [getModel("opencode-go", "kimi-k2.6")!, getModel("opencode", "kimi-k2.6")!] as const; + + for (const model of cases) { + let payload: unknown; + expect(model.compat?.maxTokensField).toBe("max_tokens"); + + await streamSimple( + model, + { + messages: [{ role: "user", content: "Hi", timestamp: Date.now() }], + }, + { + apiKey: "test", + maxTokens: 123, + onPayload: (params: unknown) => { + payload = params; + }, + }, + ).result(); + + const params = (payload ?? mockState.lastParams) as { max_tokens?: number; max_completion_tokens?: number }; + expect(params.max_tokens).toBe(123); + expect(params.max_completion_tokens).toBeUndefined(); + } + }); + + it("sends max_tokens for built-in and custom DeepSeek API models", async () => { + const customModel = { + ...localOpenAICompletionsModel, + id: "custom-deepseek-model", + name: "Custom DeepSeek Model", + provider: "custom-deepseek", + baseUrl: "https://api.deepseek.com", + } satisfies Model<"openai-completions">; + const customUppercaseModel = { + ...customModel, + id: "custom-uppercase-deepseek-model", + name: "Custom Uppercase DeepSeek Model", + baseUrl: "https://API.DeepSeek.COM", + } satisfies Model<"openai-completions">; + const nativeModels = [getModel("deepseek", "deepseek-flash")!, getModel("deepseek", "deepseek-v4-pro")!] as const; + const cases = [...nativeModels, customModel, customUppercaseModel] as const; + + for (const model of nativeModels) { + expect(model.compat?.maxTokensField).toBe("max_tokens"); + } + + for (const model of cases) { + let payload: unknown; + + await streamSimple( + model, + { + messages: [{ role: "user", content: "Hi", timestamp: Date.now() }], + }, + { + apiKey: "test", + maxTokens: 123, + onPayload: (params: unknown) => { + payload = params; + }, + }, + ).result(); + + const params = (payload ?? mockState.lastParams) as { max_tokens?: number; max_completion_tokens?: number }; + expect(params.max_tokens).toBe(123); + expect(params.max_completion_tokens).toBeUndefined(); + } + }); + + it("sends max_tokens for Z.AI completions models", async () => { + const cases = [getModel("zai", "glm-5-turbo")!, getModel("zai", "glm-5.2")!] as const; + + for (const model of cases) { + expect(model.compat?.maxTokensField).toBe("max_tokens"); + let payload: unknown; + + await streamSimple( + model, + { + messages: [{ role: "user", content: "Hi", timestamp: Date.now() }], + }, + { + apiKey: "test", + maxTokens: 123, + onPayload: (params: unknown) => { + payload = params; + }, + }, + ).result(); + + const params = (payload ?? mockState.lastParams) as { max_tokens?: number; max_completion_tokens?: number }; + expect(params.max_tokens).toBe(123); + expect(params.max_completion_tokens).toBeUndefined(); + } + }); + + it("omits reasoning effort for OpenCode Grok Build", async () => { + const model = getModel("opencode", "grok-build-0.1")!; + let payload: unknown; + + await streamSimple( + model, + { + messages: [{ role: "user", content: "Hi", timestamp: Date.now() }], + }, + { + apiKey: "test", + reasoning: "high", + onPayload: (params: unknown) => { + payload = params; + }, + }, + ).result(); + + const params = (payload ?? mockState.lastParams) as { reasoning_effort?: string }; + expect(params.reasoning_effort).toBeUndefined(); + }); + + it("does not double-count reasoning tokens in completion usage", async () => { + mockState.chunks = [ + { + id: "chatcmpl-reasoning-usage", + choices: [{ delta: {}, finish_reason: "stop" }], + usage: { + prompt_tokens: 10, + completion_tokens: 33, + prompt_tokens_details: { cached_tokens: 0 }, + completion_tokens_details: { reasoning_tokens: 21 }, + }, + }, + ]; + + const { compat: _compat, ...baseModel } = getModel("openai", "gpt-4o-mini")!; + const model = { ...baseModel, api: "openai-completions" } as const; + const response = await streamSimple( + model, + { + messages: [ + { + role: "user", + content: "Use reasoning.", + timestamp: Date.now(), + }, + ], + }, + { apiKey: "test" }, + ).result(); + + expect(response.usage.input).toBe(10); + expect(response.usage.output).toBe(33); + expect(response.usage.totalTokens).toBe(43); + }); + + it("preserves prompt_tokens_details cache read/write fields from chunk usage", async () => { + mockState.chunks = [ + { + id: "chatcmpl-cache-write", + choices: [{ delta: { content: "OK" }, finish_reason: null }], + }, + { + id: "chatcmpl-cache-write", + choices: [{ delta: {}, finish_reason: "stop" }], + usage: { + prompt_tokens: 100, + completion_tokens: 5, + prompt_tokens_details: { cached_tokens: 50, cache_write_tokens: 30 }, + completion_tokens_details: { reasoning_tokens: 0 }, + }, + }, + ]; + + const { compat: _compat, ...baseModel } = getModel("openai", "gpt-4o-mini")!; + const model = { ...baseModel, api: "openai-completions" } as const; + const response = await streamSimple( + model, + { + messages: [ + { + role: "user", + content: "Reply with exactly OK", + timestamp: Date.now(), + }, + ], + }, + { apiKey: "test" }, + ).result(); + + // cached_tokens is documented as cache reads; cache_write_tokens is separate. + expect(response.usage.input).toBe(20); + expect(response.usage.cacheRead).toBe(50); + expect(response.usage.cacheWrite).toBe(30); + expect(response.usage.totalTokens).toBe(105); + }); + + it("preserves prompt_tokens_details cache read/write fields from choice usage fallback", async () => { + mockState.chunks = [ + { + id: "chatcmpl-cache-write-choice", + choices: [{ delta: { content: "OK" }, finish_reason: null }], + }, + { + id: "chatcmpl-cache-write-choice", + choices: [ + { + delta: {}, + finish_reason: "stop", + usage: { + prompt_tokens: 100, + completion_tokens: 5, + prompt_tokens_details: { cached_tokens: 50, cache_write_tokens: 30 }, + completion_tokens_details: { reasoning_tokens: 0 }, + }, + }, + ], + }, + ]; + + const { compat: _compat, ...baseModel } = getModel("openai", "gpt-4o-mini")!; + const model = { ...baseModel, api: "openai-completions" } as const; + const response = await streamSimple( + model, + { + messages: [ + { + role: "user", + content: "Reply with exactly OK", + timestamp: Date.now(), + }, + ], + }, + { apiKey: "test" }, + ).result(); + + // cached_tokens is documented as cache reads; cache_write_tokens is separate. + expect(response.usage.input).toBe(20); + expect(response.usage.cacheRead).toBe(50); + expect(response.usage.cacheWrite).toBe(30); + expect(response.usage.totalTokens).toBe(105); + }); + + it("uses OpenRouter reasoning object instead of reasoning_effort", async () => { + const model = getModel("openrouter", "deepseek/deepseek-r1")!; + let payload: unknown; + + await streamSimple( + model, + { + messages: [ + { + role: "user", + content: "Hi", + timestamp: Date.now(), + }, + ], + }, + { + apiKey: "test", + reasoning: "high", + onPayload: (params: unknown) => { + payload = params; + }, + }, + ).result(); + + const params = (payload ?? mockState.lastParams) as { + reasoning?: { effort?: string }; + reasoning_effort?: string; + }; + expect(params.reasoning).toEqual({ effort: "high" }); + expect(params.reasoning_effort).toBeUndefined(); + }); + + it("uses configurable chat template boolean thinking kwargs", async () => { + const model = { + ...localOpenAICompletionsModel, + id: "deepseek-ai/DeepSeek-V3.1", + name: "DeepSeek V3.1 via vLLM", + compat: { + thinkingFormat: "chat-template", + supportsReasoningEffort: false, + chatTemplateKwargs: { thinking: { $var: "thinking.enabled" } }, + }, + } satisfies Model<"openai-completions">; + + for (const testCase of [ + { reasoning: "high" as const, expected: true }, + { reasoning: undefined, expected: false }, + ]) { + const params = await captureSimpleParams(model, testCase.reasoning); + + expect(params.chat_template_kwargs).toEqual({ thinking: testCase.expected }); + expect(params.thinking).toBeUndefined(); + expect(params.reasoning_effort).toBeUndefined(); + } + }); + + it("uses qwen chat template thinking kwargs", async () => { + const model = { + ...localOpenAICompletionsModel, + id: "Qwen/Qwen3-Coder", + name: "Qwen3 Coder via vLLM", + compat: { + thinkingFormat: "qwen-chat-template", + supportsReasoningEffort: false, + }, + } satisfies Model<"openai-completions">; + + for (const testCase of [ + { reasoning: "high" as const, expected: true }, + { reasoning: undefined, expected: false }, + ]) { + const params = await captureSimpleParams(model, testCase.reasoning); + + expect(params.chat_template_kwargs).toEqual({ + enable_thinking: testCase.expected, + preserve_thinking: true, + }); + expect(params.reasoning_effort).toBeUndefined(); + } + }); + + it("uses configurable chat template effort kwargs with static kwargs", async () => { + const model = { + ...localOpenAICompletionsModel, + id: "unsloth/gpt-oss-120b-GGUF", + name: "GPT OSS via vLLM", + thinkingLevelMap: { xhigh: "max" }, + compat: { + thinkingFormat: "chat-template", + supportsReasoningEffort: false, + chatTemplateKwargs: { + preserve_thinking: true, + reasoning_effort: { $var: "thinking.effort", omitWhenOff: true }, + }, + }, + } satisfies Model<"openai-completions">; + + const params = await captureSimpleParams(model, "xhigh"); + + expect(params.chat_template_kwargs).toEqual({ preserve_thinking: true, reasoning_effort: "max" }); + expect(params.reasoning_effort).toBeUndefined(); + }); + + it("uses Ant Ling compatibility metadata", async () => { + const model = getModel("ant-ling", "Ring-2.6-1T")!; + let payload: unknown; + + expect(model.compat).toMatchObject({ + supportsStore: false, + supportsDeveloperRole: false, + supportsReasoningEffort: false, + maxTokensField: "max_tokens", + thinkingFormat: "ant-ling", + supportsLongCacheRetention: false, + }); + expect(model.compat?.supportsStrictMode).toBeUndefined(); + expect(model.compat?.requiresReasoningContentOnAssistantMessages).toBeUndefined(); + + await streamSimple( + model, + { + systemPrompt: "Follow instructions.", + messages: [{ role: "user", content: "Hi", timestamp: Date.now() }], + }, + { + apiKey: "test", + maxTokens: 123, + reasoning: "high", + cacheRetention: "long", + sessionId: "ant-ling-session", + onPayload: (params: unknown) => { + payload = params; + }, + }, + ).result(); + + const params = (payload ?? mockState.lastParams) as { + max_tokens?: number; + max_completion_tokens?: number; + messages?: Array<{ role?: string }>; + reasoning?: { effort?: string }; + reasoning_effort?: string; + store?: boolean; + prompt_cache_key?: string; + prompt_cache_retention?: string; + }; + expect(params.max_tokens).toBe(123); + expect(params.max_completion_tokens).toBeUndefined(); + expect(params.messages?.[0]?.role).toBe("system"); + expect(params.reasoning).toEqual({ effort: "high" }); + expect(params.reasoning_effort).toBeUndefined(); + expect(params.store).toBeUndefined(); + expect(params.prompt_cache_key).toBeUndefined(); + expect(params.prompt_cache_retention).toBeUndefined(); + }); + + it("omits Ant Ling reasoning for unmapped direct reasoning efforts and non-reasoning models", async () => { + const ring = getModel("ant-ling", "Ring-2.6-1T")!; + let payload: unknown; + + await stream( + ring, + { + messages: [{ role: "user", content: "Hi", timestamp: Date.now() }], + }, + { + apiKey: "test", + reasoningEffort: "medium", + onPayload: (params: unknown) => { + payload = params; + }, + }, + ).result(); + + expect((payload ?? mockState.lastParams) as { reasoning?: unknown }).not.toHaveProperty("reasoning"); + + const ling = getModel("ant-ling", "Ling-2.6-flash")!; + await streamSimple( + ling, + { + messages: [{ role: "user", content: "Hi", timestamp: Date.now() }], + }, + { + apiKey: "test", + reasoning: "high", + onPayload: (params: unknown) => { + payload = params; + }, + }, + ).result(); + + expect((payload ?? mockState.lastParams) as { reasoning?: unknown }).not.toHaveProperty("reasoning"); + }); +}); diff --git a/packages/ai/test/openai-completions-tool-result-images.test.ts b/packages/ai/test/openai-completions-tool-result-images.test.ts new file mode 100644 index 0000000000000000000000000000000000000000..02de0c6236611aff3e78ab614c85b17dba562aed --- /dev/null +++ b/packages/ai/test/openai-completions-tool-result-images.test.ts @@ -0,0 +1,161 @@ +import { describe, expect, it } from "vitest"; +import { convertMessages } from "../src/api/openai-completions.ts"; +import { getModel } from "../src/compat.ts"; +import type { + AssistantMessage, + Context, + Model, + OpenAICompletionsCompat, + ToolResultMessage, + Usage, +} from "../src/types.ts"; + +const emptyUsage: Usage = { + input: 0, + output: 0, + cacheRead: 0, + cacheWrite: 0, + totalTokens: 0, + cost: { input: 0, output: 0, cacheRead: 0, cacheWrite: 0, total: 0 }, +}; + +const compat: Omit< + Required, + "deferredToolsMode" | "thinkingTokenBudgetField" | "vllmPriority" +> & { + deferredToolsMode?: OpenAICompletionsCompat["deferredToolsMode"]; + thinkingTokenBudgetField?: OpenAICompletionsCompat["thinkingTokenBudgetField"]; +} = { + supportsStore: true, + supportsDeveloperRole: true, + supportsReasoningEffort: true, + supportsUsageInStreaming: true, + supportsFinishReason: true, + maxTokensField: "max_completion_tokens", + requiresToolResultName: false, + requiresAssistantAfterToolResult: false, + requiresThinkingAsText: false, + requiresReasoningContentOnAssistantMessages: false, + thinkingFormat: "openai", + openRouterRouting: {}, + vercelGatewayRouting: {}, + chatTemplateKwargs: {}, + chatTemplateArgs: {}, + zaiToolStream: false, + supportsThinkingTokenBudget: false, + thinkingTokenBudgetField: undefined, + supportsStrictMode: true, + supportsOpenAIGrammarTools: false, + cacheControlFormat: "anthropic", + sendSessionAffinityHeaders: false, + sessionAffinityFormat: "openai", + supportsLongCacheRetention: true, +}; + +function buildToolResult(toolCallId: string, timestamp: number): ToolResultMessage { + return { + role: "toolResult", + toolCallId, + toolName: "read", + content: [ + { type: "text", text: "Read image file [image/png]" }, + { type: "image", data: "ZmFrZQ==", mimeType: "image/png" }, + ], + isError: false, + timestamp, + }; +} + +function buildEmptyToolResult(toolCallId: string, timestamp: number): ToolResultMessage { + return { + role: "toolResult", + toolCallId, + toolName: "bash", + content: [{ type: "text", text: "" }], + isError: false, + timestamp, + }; +} + +describe("openai-completions convertMessages", () => { + it("batches tool-result images after consecutive tool results", () => { + const { compat: _compat, ...baseModel } = getModel("openai", "gpt-4o-mini"); + const model: Model<"openai-completions"> = { + ...baseModel, + api: "openai-completions", + input: ["text", "image"], + }; + + const now = Date.now(); + const assistantMessage: AssistantMessage = { + role: "assistant", + content: [ + { type: "toolCall", id: "tool-1", name: "read", arguments: { path: "img-1.png" } }, + { type: "toolCall", id: "tool-2", name: "read", arguments: { path: "img-2.png" } }, + ], + api: model.api, + provider: model.provider, + model: model.id, + usage: emptyUsage, + stopReason: "toolUse", + timestamp: now, + }; + + const context: Context = { + messages: [ + { role: "user", content: "Read the images", timestamp: now - 2 }, + assistantMessage, + buildToolResult("tool-1", now + 1), + buildToolResult("tool-2", now + 2), + ], + }; + + const messages = convertMessages(model, context, compat); + const roles = messages.map((message) => message.role); + expect(roles).toEqual(["user", "assistant", "tool", "tool", "user"]); + + const imageMessage = messages[messages.length - 1]; + expect(imageMessage.role).toBe("user"); + expect(Array.isArray(imageMessage.content)).toBe(true); + + const imageParts = (imageMessage.content as Array<{ type?: string }>).filter( + (part) => part?.type === "image_url", + ); + expect(imageParts.length).toBe(2); + }); + + it("uses '(no tool output)' placeholder for empty tool results without images", () => { + const { compat: _compat, ...baseModel } = getModel("openai", "gpt-4o-mini"); + const model: Model<"openai-completions"> = { + ...baseModel, + api: "openai-completions", + input: ["text", "image"], + }; + + const now = Date.now(); + const assistantMessage: AssistantMessage = { + role: "assistant", + content: [{ type: "toolCall", id: "tool-1", name: "bash", arguments: { command: "true" } }], + api: model.api, + provider: model.provider, + model: model.id, + usage: emptyUsage, + stopReason: "toolUse", + timestamp: now, + }; + + const context: Context = { + messages: [ + { role: "user", content: "Run the command", timestamp: now - 1 }, + assistantMessage, + buildEmptyToolResult("tool-1", now + 1), + ], + }; + + const messages = convertMessages(model, context, compat); + const toolMessage = messages.find((m) => m.role === "tool") as { role: "tool"; content: string } | undefined; + expect(toolMessage).toBeTruthy(); + expect(toolMessage?.content).toBe("(no tool output)"); + expect(toolMessage?.content).not.toContain("see attached image"); + }); +}); diff --git a/packages/ai/test/openai-responses-empty-tool-result.test.ts b/packages/ai/test/openai-responses-empty-tool-result.test.ts new file mode 100644 index 0000000000000000000000000000000000000000..0db91237746588c1d8302889d5a454b03fdebc5a --- /dev/null +++ b/packages/ai/test/openai-responses-empty-tool-result.test.ts @@ -0,0 +1,58 @@ +import { describe, expect, it } from "vitest"; +import { convertResponsesMessages } from "../src/api/openai-responses-shared.ts"; +import { getModel } from "../src/compat.ts"; +import type { AssistantMessage, Context, ToolResultMessage, Usage } from "../src/types.ts"; + +const usage: Usage = { + input: 0, + output: 0, + cacheRead: 0, + cacheWrite: 0, + totalTokens: 0, + cost: { input: 0, output: 0, cacheRead: 0, cacheWrite: 0, total: 0 }, +}; + +function buildEmptyToolResult(toolCallId: string, timestamp: number): ToolResultMessage { + return { + role: "toolResult", + toolCallId, + toolName: "bash", + content: [{ type: "text", text: "" }], + isError: false, + timestamp, + }; +} + +describe("OpenAI Responses convertResponsesMessages empty tool result", () => { + it("uses '(no tool output)' placeholder for empty tool results without images", () => { + const model = getModel("openai", "gpt-4o-mini"); + const now = Date.now(); + const assistant: AssistantMessage = { + role: "assistant", + content: [{ type: "toolCall", id: "tool-1", name: "bash", arguments: { command: "true" } }], + api: model.api, + provider: model.provider, + model: model.id, + usage, + stopReason: "toolUse", + timestamp: now, + }; + + const context: Context = { + messages: [ + { role: "user", content: "Run the command", timestamp: now - 1 }, + assistant, + buildEmptyToolResult("tool-1", now + 1), + ], + }; + + const input = convertResponsesMessages(model, context, new Set(["openai", "openai-codex", "opencode"])); + const functionCallOutput = input.find((item) => item.type === "function_call_output") as + | { type: "function_call_output"; output: string } + | undefined; + + expect(functionCallOutput).toBeTruthy(); + expect(functionCallOutput?.output).toBe("(no tool output)"); + expect(functionCallOutput?.output).not.toContain("see attached image"); + }); +}); diff --git a/packages/ai/test/openai-responses-terminal-event.test.ts b/packages/ai/test/openai-responses-terminal-event.test.ts new file mode 100644 index 0000000000000000000000000000000000000000..a6c2fd94c9db311e21108ebc354814891a400eb7 --- /dev/null +++ b/packages/ai/test/openai-responses-terminal-event.test.ts @@ -0,0 +1,356 @@ +import type { ResponseStreamEvent } from "openai/resources/responses/responses.js"; +import { describe, expect, it, vi } from "vitest"; +import { stream as streamOpenAIResponses } from "../src/api/openai-responses.ts"; +import { processResponsesStream } from "../src/api/openai-responses-shared.ts"; +import type { AssistantMessage, AssistantMessageEvent, Context, Model } from "../src/types.ts"; +import { AssistantMessageEventStream } from "../src/utils/event-stream.ts"; + +vi.mock("openai", () => { + async function* createMockResponsesStream(): AsyncIterable { + yield { + type: "response.created", + sequence_number: 0, + response: { id: "resp_wrapper_early_eof" }, + } as ResponseStreamEvent; + yield { + type: "response.output_item.added", + sequence_number: 1, + output_index: 0, + item: { type: "reasoning", id: "rs_wrapper_early_eof", summary: [] }, + } as ResponseStreamEvent; + yield { + type: "response.reasoning_text.delta", + sequence_number: 2, + output_index: 0, + content_index: 0, + item_id: "rs_wrapper_early_eof", + delta: "partial reasoning before the wrapper stream ends", + } as ResponseStreamEvent; + } + + class FakeOpenAI { + responses = { + create: () => { + const responseStream = createMockResponsesStream(); + const promise = Promise.resolve(responseStream) as Promise> & { + withResponse: () => Promise<{ + data: AsyncIterable; + response: { status: number; headers: Headers }; + }>; + }; + promise.withResponse = async () => ({ + data: responseStream, + response: { status: 200, headers: new Headers() }, + }); + return promise; + }, + }; + } + + return { default: FakeOpenAI }; +}); + +function createModel(): Model<"openai-responses"> { + return { + id: "gpt-5-mini", + name: "GPT-5 Mini", + api: "openai-responses", + provider: "openai", + baseUrl: "https://api.openai.com/v1", + reasoning: true, + input: ["text"], + cost: { input: 0, output: 0, cacheRead: 0, cacheWrite: 0 }, + contextWindow: 400000, + maxTokens: 128000, + }; +} + +function createOutput(model: Model<"openai-responses">): AssistantMessage { + return { + role: "assistant", + content: [], + api: model.api, + provider: model.provider, + model: model.id, + usage: { + input: 0, + output: 0, + cacheRead: 0, + cacheWrite: 0, + totalTokens: 0, + cost: { input: 0, output: 0, cacheRead: 0, cacheWrite: 0, total: 0 }, + }, + stopReason: "pending", + timestamp: Date.now(), + }; +} + +async function* createEarlyEofEvents(): AsyncIterable { + yield { + type: "response.created", + sequence_number: 0, + response: { id: "resp_early_eof" }, + } as ResponseStreamEvent; + yield { + type: "response.output_item.added", + sequence_number: 1, + output_index: 0, + item: { type: "reasoning", id: "rs_early_eof", summary: [] }, + } as ResponseStreamEvent; + yield { + type: "response.reasoning_text.delta", + sequence_number: 2, + output_index: 0, + content_index: 0, + item_id: "rs_early_eof", + delta: "partial reasoning before the stream ends", + } as ResponseStreamEvent; +} + +async function* createCompletedEvents(): AsyncIterable { + yield { + type: "response.completed", + sequence_number: 0, + response: { + id: "resp_completed", + status: "completed", + usage: { + input_tokens: 20, + output_tokens: 7, + total_tokens: 27, + input_tokens_details: { cached_tokens: 2, cache_write_tokens: 3 }, + }, + }, + } as unknown as ResponseStreamEvent; +} + +async function* createIncompleteEvents(reason = "max_output_tokens"): AsyncIterable { + yield { + type: "response.incomplete", + sequence_number: 0, + response: { + id: "resp_incomplete", + status: "incomplete", + incomplete_details: { reason }, + usage: { + input_tokens: 30, + output_tokens: 12, + total_tokens: 42, + input_tokens_details: { cached_tokens: 5 }, + }, + }, + } as ResponseStreamEvent; +} + +async function* createFailedEvents(): AsyncIterable { + yield { + type: "response.failed", + sequence_number: 0, + response: { + id: "resp_failed", + status: "failed", + error: { code: "server_error", message: "boom" }, + }, + } as ResponseStreamEvent; +} + +async function* createPhasedMessageEvents( + phases: readonly ["commentary" | "final_answer", "commentary" | "final_answer"], + terminalStatus: "completed" | "incomplete" = "completed", +): AsyncIterable { + yield { + type: "response.output_item.added", + sequence_number: 0, + output_index: 0, + item: { + type: "message", + id: "msg_phase", + role: "assistant", + status: "in_progress", + content: [], + phase: phases[0], + }, + } as ResponseStreamEvent; + yield { + type: "response.output_item.done", + sequence_number: 1, + output_index: 0, + item: { + type: "message", + id: "msg_phase", + role: "assistant", + status: "completed", + content: [{ type: "output_text", text: "answer", annotations: [] }], + phase: phases[1], + }, + } as ResponseStreamEvent; + if (terminalStatus === "incomplete") { + yield { + type: "response.incomplete", + sequence_number: 2, + response: { + id: "resp_phase", + status: "incomplete", + incomplete_details: { reason: "max_output_tokens" }, + }, + } as ResponseStreamEvent; + return; + } + yield { + type: "response.completed", + sequence_number: 2, + response: { id: "resp_phase", status: "completed" }, + } as ResponseStreamEvent; +} + +describe("OpenAI Responses terminal event handling", () => { + it("rejects streams that end before a terminal response event", async () => { + const model = createModel(); + const output = createOutput(model); + const stream = new AssistantMessageEventStream(); + + await expect(processResponsesStream(createEarlyEofEvents(), output, stream, model)).rejects.toThrow( + "OpenAI Responses stream ended before a terminal response event", + ); + }); + + it("emits an error final result when the wrapper stream ends before a terminal response event", async () => { + const model = createModel(); + const context: Context = { + systemPrompt: "", + messages: [{ role: "user", content: [{ type: "text", text: "hi" }], timestamp: 0 }], + tools: [], + }; + const stream = streamOpenAIResponses(model, context, { apiKey: "test" }); + const events: AssistantMessageEvent[] = []; + let initialStopReason: AssistantMessage["stopReason"] | undefined; + + for await (const event of stream) { + if (event.type === "start") initialStopReason = event.partial.stopReason; + events.push(event); + } + + const result = await stream.result(); + const lastEvent = events.at(-1); + expect(initialStopReason).toBe("pending"); + expect(lastEvent?.type).toBe("error"); + expect(result.stopReason).toBe("error"); + expect(result.errorMessage).toBe("OpenAI Responses stream ended before a terminal response event"); + }); + + it.each([ + { phases: ["commentary", "commentary"], expected: ["pending", "pending"] }, + { phases: ["final_answer", "final_answer"], expected: ["stop", "stop"] }, + { phases: ["commentary", "final_answer"], expected: ["pending", "stop"] }, + ] as const)("tracks message phases $phases", async ({ phases, expected }) => { + const model = createModel(); + const output = createOutput(model); + const stream = new AssistantMessageEventStream(); + const observedStopReasons: AssistantMessage["stopReason"][] = []; + const push = stream.push.bind(stream); + stream.push = (event) => { + if ("partial" in event) observedStopReasons.push(event.partial.stopReason); + push(event); + }; + + await processResponsesStream(createPhasedMessageEvents(phases), output, stream, model); + + expect(observedStopReasons).toEqual(expected); + expect(output.stopReason).toBe("stop"); + }); + + it("replaces a provisional final-answer stop with an incomplete terminal reason", async () => { + const model = createModel(); + const output = createOutput(model); + const stream = new AssistantMessageEventStream(); + const observedStopReasons: AssistantMessage["stopReason"][] = []; + const push = stream.push.bind(stream); + stream.push = (event) => { + if ("partial" in event) observedStopReasons.push(event.partial.stopReason); + push(event); + }; + + await processResponsesStream( + createPhasedMessageEvents(["final_answer", "final_answer"], "incomplete"), + output, + stream, + model, + ); + + expect(observedStopReasons).toEqual(["stop", "stop"]); + expect(output.stopReason).toBe("length"); + }); + + it("finalizes completed terminal events as stop", async () => { + const model = createModel(); + const output = createOutput(model); + const stream = new AssistantMessageEventStream(); + + await processResponsesStream(createCompletedEvents(), output, stream, model); + + expect(output.responseId).toBe("resp_completed"); + expect(output.stopReason).toBe("stop"); + expect(output.rawStopReason).toBe("completed"); + expect(output.usage).toMatchObject({ + input: 15, + output: 7, + cacheRead: 2, + cacheWrite: 3, + totalTokens: 27, + }); + }); + + it("finalizes incomplete terminal events as length stops", async () => { + const model = createModel(); + const output = createOutput(model); + const stream = new AssistantMessageEventStream(); + + await processResponsesStream(createIncompleteEvents(), output, stream, model); + + expect(output.responseId).toBe("resp_incomplete"); + expect(output.stopReason).toBe("length"); + expect(output.rawStopReason).toBe("incomplete.max_output_tokens"); + expect(output.usage).toMatchObject({ + input: 25, + output: 12, + cacheRead: 5, + cacheWrite: 0, + totalTokens: 42, + }); + }); + + it("finalizes content-filtered incomplete responses as non-retryable errors", async () => { + const model = createModel(); + const output = createOutput(model); + const stream = new AssistantMessageEventStream(); + + await processResponsesStream(createIncompleteEvents("content_filter"), output, stream, model); + + expect(output.stopReason).toBe("error"); + expect(output.rawStopReason).toBe("incomplete.content_filter"); + expect(output.errorMessage).toBe("Response incomplete: content_filter"); + }); + + it("preserves unknown provider incomplete reasons as non-retryable errors", async () => { + const model = createModel(); + const output = createOutput(model); + const stream = new AssistantMessageEventStream(); + + await processResponsesStream(createIncompleteEvents("max_time_limit"), output, stream, model); + + expect(output.stopReason).toBe("error"); + expect(output.rawStopReason).toBe("incomplete.max_time_limit"); + expect(output.errorMessage).toBe("Response incomplete: max_time_limit"); + }); + + it("rejects failed terminal events with the provider error", async () => { + const model = createModel(); + const output = createOutput(model); + const stream = new AssistantMessageEventStream(); + + await expect(processResponsesStream(createFailedEvents(), output, stream, model)).rejects.toThrow( + "server_error: boom", + ); + expect(output.rawStopReason).toBe("failed"); + }); +}); diff --git a/packages/ai/test/openrouter-cache-control-models.test.ts b/packages/ai/test/openrouter-cache-control-models.test.ts new file mode 100644 index 0000000000000000000000000000000000000000..f8db1794f9eff6774901391b5abe0bcd5ff33fa1 --- /dev/null +++ b/packages/ai/test/openrouter-cache-control-models.test.ts @@ -0,0 +1,18 @@ +import { describe, expect, it } from "vitest"; +import { getModel } from "../src/compat.ts"; + +const OPENROUTER_ANTHROPIC_LATEST_MODEL_IDS = [ + "~anthropic/claude-fable-latest", + "~anthropic/claude-haiku-latest", + "~anthropic/claude-opus-latest", + "~anthropic/claude-sonnet-latest", +] as const; + +describe("OpenRouter Anthropic latest alias metadata", () => { + it.each(OPENROUTER_ANTHROPIC_LATEST_MODEL_IDS)("keeps completions cache control for %s", (modelId) => { + const model = getModel("openrouter", modelId); + expect(model.api).toBe("openai-completions"); + if (model.api !== "openai-completions") throw new Error(`Unexpected API for ${modelId}`); + expect(model.compat?.cacheControlFormat).toBe("anthropic"); + }); +}); diff --git a/packages/ai/test/openrouter-oauth.test.ts b/packages/ai/test/openrouter-oauth.test.ts new file mode 100644 index 0000000000000000000000000000000000000000..47ff090f030143dbbfddaac0a530341b92301c5f --- /dev/null +++ b/packages/ai/test/openrouter-oauth.test.ts @@ -0,0 +1,322 @@ +import { afterEach, describe, expect, it, vi } from "vitest"; +import { InMemoryCredentialStore } from "../src/auth/credential-store.ts"; +import { openRouterOAuth } from "../src/auth/oauth/openrouter.ts"; +import { createImagesModels } from "../src/images-models.ts"; +import { createModels } from "../src/models.ts"; +import { openrouterProvider } from "../src/providers/openrouter.ts"; +import { openrouterImagesProvider } from "../src/providers/openrouter-images.ts"; + +const TOKEN_URL = "https://openrouter.ai/api/v1/auth/keys"; +const nativeFetch = globalThis.fetch; +const neverAbortedSignal = new AbortController().signal; + +function jsonResponse(body: unknown, status = 200): Response { + return new Response(JSON.stringify(body), { status, headers: { "content-type": "application/json" } }); +} + +function base64url(bytes: Uint8Array): string { + let binary = ""; + for (const byte of bytes) binary += String.fromCharCode(byte); + return btoa(binary).replaceAll("+", "-").replaceAll("/", "_").replaceAll("=", ""); +} + +describe.sequential("OpenRouter OAuth", () => { + afterEach(() => { + vi.unstubAllGlobals(); + vi.unstubAllEnvs(); + }); + + it("is exposed by both OpenRouter providers alongside API-key auth", () => { + for (const provider of [openrouterProvider(), openrouterImagesProvider()]) { + expect(provider.auth.apiKey).toBeDefined(); + expect(provider.auth.oauth).toBeDefined(); + expect(provider.auth.oauth?.loginLabel).toBe("Sign in with OpenRouter"); + } + }); + + it("resolves the same stored OAuth key for text and image providers", async () => { + const credentials = new InMemoryCredentialStore(); + await credentials.modify("openrouter", async () => ({ + type: "oauth", + access: "sk-or-stored", + refresh: "", + expires: Number.MAX_SAFE_INTEGER, + })); + + const textModels = createModels({ credentials }); + textModels.setProvider(openrouterProvider()); + const imageModels = createImagesModels({ credentials }); + imageModels.setProvider(openrouterImagesProvider()); + + expect((await textModels.getAuth("openrouter"))?.auth.apiKey).toBe("sk-or-stored"); + expect((await imageModels.getAuth("openrouter"))?.auth.apiKey).toBe("sk-or-stored"); + }); + + it("runs PKCE on a one-shot loopback callback and exchanges the code for a permanent API key", async () => { + let exchangeBody: Record | undefined; + const fetchMock = vi.fn(async (input: string | URL | Request, init?: RequestInit) => { + const url = input instanceof Request ? input.url : String(input); + if (url !== TOKEN_URL) return nativeFetch(input, init); + exchangeBody = JSON.parse(String(init?.body)) as Record; + return jsonResponse({ key: "sk-or-test" }); + }); + vi.stubGlobal("fetch", fetchMock); + + let authorizeUrl: URL | undefined; + let callbackResponse: Promise | undefined; + let manualSignal: AbortSignal | undefined; + const credential = await openRouterOAuth.login({ + signal: neverAbortedSignal, + prompt: (prompt) => { + manualSignal = prompt.signal; + return new Promise(() => {}); + }, + notify: (event) => { + if (event.type !== "auth_url") return; + authorizeUrl = new URL(event.url); + const callbackUrl = new URL(authorizeUrl.searchParams.get("callback_url") ?? ""); + callbackUrl.searchParams.set("code", "authorization-code"); + callbackResponse = nativeFetch(callbackUrl); + }, + }); + + expect(credential).toEqual({ + type: "oauth", + access: "sk-or-test", + refresh: "", + expires: Number.MAX_SAFE_INTEGER, + }); + expect((await callbackResponse)?.status).toBe(200); + expect(manualSignal?.aborted).toBe(true); + expect(authorizeUrl?.origin).toBe("https://openrouter.ai"); + expect(authorizeUrl?.pathname).toBe("/auth"); + expect(authorizeUrl?.searchParams.get("code_challenge_method")).toBe("S256"); + + const callbackUrl = new URL(authorizeUrl?.searchParams.get("callback_url") ?? ""); + expect(callbackUrl.hostname).toBe("127.0.0.1"); + expect(callbackUrl.pathname).toMatch(/^\/oauth\/callback\/[0-9a-f-]+$/); + + expect(exchangeBody).toMatchObject({ + code: "authorization-code", + code_challenge_method: "S256", + }); + const verifier = exchangeBody?.code_verifier; + expect(typeof verifier).toBe("string"); + const digest = await crypto.subtle.digest("SHA-256", new TextEncoder().encode(String(verifier))); + expect(authorizeUrl?.searchParams.get("code_challenge")).toBe(base64url(new Uint8Array(digest))); + expect(fetchMock).toHaveBeenCalledTimes(1); + }); + + it("reports token exchange failures through both the callback page and login", async () => { + vi.stubGlobal( + "fetch", + vi.fn(async () => jsonResponse({ error: { message: "invalid code" } }, 403)), + ); + + let callbackResponse: Promise | undefined; + const login = openRouterOAuth.login({ + signal: neverAbortedSignal, + prompt: () => new Promise(() => {}), + notify: (event) => { + if (event.type !== "auth_url") return; + const callbackUrl = new URL(new URL(event.url).searchParams.get("callback_url") ?? ""); + callbackUrl.searchParams.set("code", "bad-code"); + callbackResponse = nativeFetch(callbackUrl); + }, + }); + + await expect(login).rejects.toThrow("OpenRouter OAuth key exchange failed (HTTP 403): invalid code"); + expect((await callbackResponse)?.status).toBe(502); + }); + + it("allows only one token exchange for a callback", async () => { + let completeExchange = (_response: Response): void => { + throw new Error("Token exchange did not start"); + }; + const fetchMock = vi.fn( + async () => + new Promise((resolve) => { + completeExchange = resolve; + }), + ); + vi.stubGlobal("fetch", fetchMock); + + let callbackUrl: URL | undefined; + let firstCallback: Promise | undefined; + const login = openRouterOAuth.login({ + signal: neverAbortedSignal, + prompt: () => new Promise(() => {}), + notify: (event) => { + if (event.type !== "auth_url") return; + callbackUrl = new URL(new URL(event.url).searchParams.get("callback_url") ?? ""); + callbackUrl.searchParams.set("code", "authorization-code"); + firstCallback = nativeFetch(callbackUrl); + }, + }); + + await vi.waitFor(() => expect(fetchMock).toHaveBeenCalledTimes(1)); + if (!callbackUrl) throw new Error("OpenRouter did not provide a callback URL"); + expect((await nativeFetch(callbackUrl)).status).toBe(409); + expect(fetchMock).toHaveBeenCalledTimes(1); + completeExchange(jsonResponse({ key: "sk-or-test" })); + + await expect(login).resolves.toMatchObject({ access: "sk-or-test" }); + expect((await firstCallback)?.status).toBe(200); + }); + + it("rejects a successful response that does not contain a key", async () => { + vi.stubGlobal( + "fetch", + vi.fn(async () => jsonResponse({ user_id: "user-1" })), + ); + + let callbackResponse: Promise | undefined; + const login = openRouterOAuth.login({ + signal: neverAbortedSignal, + prompt: () => new Promise(() => {}), + notify: (event) => { + if (event.type !== "auth_url") return; + const callbackUrl = new URL(new URL(event.url).searchParams.get("callback_url") ?? ""); + callbackUrl.searchParams.set("code", "code-without-key"); + callbackResponse = nativeFetch(callbackUrl); + }, + }); + + await expect(login).rejects.toThrow('OpenRouter OAuth response carries no "key"'); + expect((await callbackResponse)?.status).toBe(502); + }); + + it("mints a key from a pasted redirect URL when the loopback callback never arrives", async () => { + let exchangeBody: Record | undefined; + const fetchMock = vi.fn(async (input: string | URL | Request, init?: RequestInit) => { + const url = input instanceof Request ? input.url : String(input); + if (url !== TOKEN_URL) return nativeFetch(input, init); + exchangeBody = JSON.parse(String(init?.body)) as Record; + return jsonResponse({ key: "sk-or-manual" }); + }); + vi.stubGlobal("fetch", fetchMock); + + let callbackUrl: string | undefined; + const credential = await openRouterOAuth.login({ + signal: neverAbortedSignal, + prompt: async (prompt) => { + if (prompt.type !== "manual_code") throw new Error(`Unexpected prompt: ${prompt.type}`); + return `${callbackUrl}?code=manual-code`; + }, + notify: (event) => { + if (event.type !== "auth_url") return; + callbackUrl = new URL(event.url).searchParams.get("callback_url") ?? undefined; + }, + }); + + expect(credential).toEqual({ + type: "oauth", + access: "sk-or-manual", + refresh: "", + expires: Number.MAX_SAFE_INTEGER, + }); + expect(exchangeBody).toMatchObject({ code: "manual-code", code_challenge_method: "S256" }); + expect(fetchMock).toHaveBeenCalledTimes(1); + }); + + it("accepts a bare authorization code from the manual prompt", async () => { + let exchangeBody: Record | undefined; + const fetchMock = vi.fn(async (input: string | URL | Request, init?: RequestInit) => { + const url = input instanceof Request ? input.url : String(input); + if (url !== TOKEN_URL) return nativeFetch(input, init); + exchangeBody = JSON.parse(String(init?.body)) as Record; + return jsonResponse({ key: "sk-or-manual" }); + }); + vi.stubGlobal("fetch", fetchMock); + + const credential = await openRouterOAuth.login({ + signal: neverAbortedSignal, + prompt: async () => " manual-code ", + notify: () => {}, + }); + + expect(credential).toMatchObject({ access: "sk-or-manual" }); + expect(exchangeBody).toMatchObject({ code: "manual-code" }); + }); + + it("fails login when the manual prompt is cancelled", async () => { + const fetchMock = vi.fn(async () => jsonResponse({ key: "sk-or-unexpected" })); + vi.stubGlobal("fetch", fetchMock); + + await expect( + openRouterOAuth.login({ + signal: neverAbortedSignal, + prompt: async () => { + throw new Error("Login cancelled"); + }, + notify: () => {}, + }), + ).rejects.toThrow("Login cancelled"); + expect(fetchMock).not.toHaveBeenCalled(); + }); + + it("rejects empty manual input without exchanging a code", async () => { + const fetchMock = vi.fn(async () => jsonResponse({ key: "sk-or-unexpected" })); + vi.stubGlobal("fetch", fetchMock); + + await expect( + openRouterOAuth.login({ + signal: neverAbortedSignal, + prompt: async () => " ", + notify: () => {}, + }), + ).rejects.toThrow("Missing authorization code"); + expect(fetchMock).not.toHaveBeenCalled(); + }); + + it("closes the pending callback when login is cancelled", async () => { + const controller = new AbortController(); + let callbackUrl: URL | undefined; + const login = openRouterOAuth.login({ + signal: controller.signal, + prompt: () => new Promise(() => {}), + notify: (event) => { + if (event.type !== "auth_url") return; + callbackUrl = new URL(new URL(event.url).searchParams.get("callback_url") ?? ""); + controller.abort(); + }, + }); + + await expect(login).rejects.toThrow("Login cancelled"); + expect(callbackUrl).toBeDefined(); + await expect(nativeFetch(callbackUrl!)).rejects.toThrow(); + }); + + it("rejects before opening a callback server when login is already cancelled", async () => { + const controller = new AbortController(); + controller.abort(); + + await expect( + openRouterOAuth.login({ + signal: controller.signal, + prompt: async () => "", + notify: () => { + throw new Error("Cancelled login must not emit events"); + }, + }), + ).rejects.toThrow("Login cancelled"); + }); + + it("uses the configured OAuth callback host", async () => { + vi.stubEnv("PI_OAUTH_CALLBACK_HOST", "localhost"); + const controller = new AbortController(); + let callbackUrl: URL | undefined; + const login = openRouterOAuth.login({ + signal: controller.signal, + prompt: () => new Promise(() => {}), + notify: (event) => { + if (event.type !== "auth_url") return; + callbackUrl = new URL(new URL(event.url).searchParams.get("callback_url") ?? ""); + controller.abort(); + }, + }); + + await expect(login).rejects.toThrow("Login cancelled"); + expect(callbackUrl?.hostname).toBe("localhost"); + }); +}); diff --git a/packages/ai/test/pi-messages.test.ts b/packages/ai/test/pi-messages.test.ts new file mode 100644 index 0000000000000000000000000000000000000000..041929ada0ce36b106600e82a43fa12e42326873 --- /dev/null +++ b/packages/ai/test/pi-messages.test.ts @@ -0,0 +1,255 @@ +import { createServer, type IncomingMessage, type Server, type ServerResponse } from "node:http"; +import type { AddressInfo } from "node:net"; +import { afterEach, describe, expect, it } from "vitest"; +import { type PiMessagesOptions, stream, streamSimple } from "../src/api/pi-messages.ts"; +import type { Api, AssistantMessageEvent, Context, Model, StopReason } from "../src/types.ts"; + +type RecordedRequest = { + url: string; + headers: IncomingMessage["headers"]; + body: unknown; +}; + +type ResponderOptions = { + status?: number; + headers?: Record; + events?: unknown[]; + rawBody?: string; +}; + +let server: Server | undefined; + +afterEach(() => { + server?.close(); + server = undefined; +}); + +async function startServer(options: ResponderOptions): Promise<{ baseUrl: string; requests: RecordedRequest[] }> { + const requests: RecordedRequest[] = []; + + server = createServer((request: IncomingMessage, response: ServerResponse) => { + const chunks: Buffer[] = []; + request.on("data", (chunk: Buffer) => chunks.push(chunk)); + request.on("end", () => { + const raw = Buffer.concat(chunks).toString("utf-8"); + requests.push({ + url: request.url ?? "", + headers: request.headers, + body: raw ? JSON.parse(raw) : undefined, + }); + + if (options.status && options.status !== 200) { + response.statusCode = options.status; + response.setHeader("content-type", "application/json"); + response.end(options.rawBody ?? "{}"); + return; + } + + response.statusCode = 200; + response.setHeader("content-type", "text/event-stream"); + for (const [name, value] of Object.entries(options.headers ?? {})) { + response.setHeader(name, value); + } + for (const event of options.events ?? []) { + response.write(`data: ${JSON.stringify(event)}\n\n`); + } + response.end(); + }); + }); + + await new Promise((resolve) => { + server!.listen(0, "127.0.0.1", () => resolve()); + }); + + const address = server!.address() as AddressInfo; + return { baseUrl: `http://127.0.0.1:${address.port}/v1`, requests }; +} + +function createModel(baseUrl: string): Model<"pi-messages"> { + return { + id: "auto", + name: "Radius Auto", + api: "pi-messages", + provider: "radius", + baseUrl, + reasoning: false, + input: ["text"], + cost: { input: 1, output: 2, cacheRead: 0.1, cacheWrite: 0.2 }, + contextWindow: 128000, + maxTokens: 16384, + }; +} + +const context: Context = { + messages: [{ role: "user", content: "Hello", timestamp: Date.now() }], +}; + +const usage = { + input: 10, + output: 5, + cacheRead: 0, + cacheWrite: 0, + totalTokens: 15, + cost: { input: 0.1, output: 0.2, cacheRead: 0, cacheWrite: 0, total: 0.3 }, +}; + +describe("pi-messages", () => { + it("streams text and tool calls and resolves the terminal message", async () => { + const { baseUrl, requests } = await startServer({ + events: [ + { type: "start" }, + { type: "text_start", contentIndex: 0 }, + { type: "text_delta", contentIndex: 0, delta: "Hel" }, + { type: "text_delta", contentIndex: 0, delta: "lo" }, + { type: "text_end", contentIndex: 0, content: "Hello" }, + { type: "toolcall_start", contentIndex: 1, id: "call_1", toolName: "read" }, + { type: "toolcall_delta", contentIndex: 1, delta: '{"path":' }, + { type: "toolcall_delta", contentIndex: 1, delta: '"a.txt"}' }, + { + type: "toolcall_end", + contentIndex: 1, + toolCall: { type: "toolCall", id: "call_1", name: "read", arguments: { path: "a.txt" } }, + }, + { + type: "done", + reason: "toolUse", + usage, + responseId: "resp_1", + providerThinkingLevel: "high", + }, + ], + }); + const model = createModel(baseUrl); + + const events: AssistantMessageEvent[] = []; + const partialStopReasons: StopReason[] = []; + const eventStream = stream(model, context, { + apiKey: "test-key", + sessionId: "session-1", + toolChoice: "auto", + maxTokens: 100, + headers: { "x-custom": "1" }, + }); + for await (const event of eventStream) { + if ("partial" in event) { + partialStopReasons.push(event.partial.stopReason); + } + events.push(event); + } + const message = await eventStream.result(); + + expect(partialStopReasons[0]).toBe("pending"); + expect(message.stopReason).toBe("toolUse"); + expect(message.usage).toEqual(usage); + expect(message.responseId).toBe("resp_1"); + expect(message.providerThinkingLevel).toBe("high"); + expect(message.model).toBe("auto"); + expect(message.provider).toBe("radius"); + expect(message.content).toEqual([ + { type: "text", text: "Hello", textSignature: undefined }, + { type: "toolCall", id: "call_1", name: "read", arguments: { path: "a.txt" } }, + ]); + expect(events.some((event) => event.type === "text_delta")).toBe(true); + expect(events.filter((event) => event.type === "toolcall_end")).toHaveLength(1); + + expect(requests).toHaveLength(1); + const request = requests[0]; + expect(request.url).toBe("/v1/messages"); + expect(request.headers.authorization).toBe("Bearer test-key"); + expect(request.headers["x-custom"]).toBe("1"); + expect(request.body).toEqual({ + model: "auto", + context, + options: { maxTokens: 100, sessionId: "session-1", toolChoice: "auto" }, + }); + }); + + it("appends debug=1 and reports response headers via onResponse", async () => { + const { baseUrl, requests } = await startServer({ + headers: { "x-pi-gateway-upstream-provider": "anthropic" }, + events: [{ type: "done", reason: "stop", usage }], + }); + const model = createModel(baseUrl); + + let observedHeaders: Record | undefined; + const options = { + apiKey: "test-key", + debug: true, + onResponse: (response) => { + observedHeaders = response.headers; + }, + } satisfies PiMessagesOptions; + const message = await streamSimple(model, context, options).result(); + + expect(message.stopReason).toBe("stop"); + expect(requests[0].url).toBe("/v1/messages?debug=1"); + expect(observedHeaders?.["x-pi-gateway-upstream-provider"]).toBe("anthropic"); + }); + + it("surfaces backend error responses with diagnostics", async () => { + const { baseUrl } = await startServer({ + status: 401, + rawBody: JSON.stringify({ error: { message: "Token expired", code: "unauthorized" } }), + }); + const model = createModel(baseUrl); + + const message = await stream(model, context, { apiKey: "stale" }).result(); + + expect(message.stopReason).toBe("error"); + expect(message.errorMessage).toContain("401"); + expect(message.errorMessage).toContain("Token expired"); + expect(message.errorMessage).toContain("unauthorized"); + expect(message.diagnostics?.[0]?.type).toBe("pi_messages_response_failure"); + expect(message.diagnostics?.[0]?.details?.status).toBe(401); + }); + + it("propagates server-sent error events", async () => { + const { baseUrl } = await startServer({ + events: [{ type: "start" }, { type: "error", reason: "error", usage, errorMessage: "Upstream failed" }], + }); + const model = createModel(baseUrl); + + const message = await stream(model, context, { apiKey: "test-key" }).result(); + + expect(message.stopReason).toBe("error"); + expect(message.errorMessage).toBe("Upstream failed"); + expect(message.usage).toEqual(usage); + }); + + it("errors when no API key is provided", async () => { + const model = createModel("http://127.0.0.1:1/v1"); + + const message = await stream(model, context).result(); + + expect(message.stopReason).toBe("error"); + expect(message.errorMessage).toContain("No API key provided"); + }); + + it("errors when the stream ends without a terminal event", async () => { + const { baseUrl } = await startServer({ + events: [ + { type: "start" }, + { type: "text_start", contentIndex: 0 }, + { type: "text_delta", contentIndex: 0, delta: "partial" }, + ], + }); + const model = createModel(baseUrl); + + const message = await stream(model, context, { apiKey: "test-key" }).result(); + + expect(message.stopReason).toBe("error"); + expect(message.errorMessage).toContain("stream ended without a terminal event"); + }); +}); + +describe("pi-messages api registration", () => { + it("is registered as a builtin api provider", async () => { + const { getApiProvider } = await import("../src/compat.ts"); + expect(getApiProvider("pi-messages")).toBeDefined(); + }); + + it("is a known api usable on models", () => { + const api: Api = "pi-messages"; + expect(api).toBe("pi-messages"); + }); +}); diff --git a/packages/ai/test/pre-generation-error.test.ts b/packages/ai/test/pre-generation-error.test.ts new file mode 100644 index 0000000000000000000000000000000000000000..991f37f858a3e01127f297ef3882a3ed680789eb --- /dev/null +++ b/packages/ai/test/pre-generation-error.test.ts @@ -0,0 +1,40 @@ +import { describe, expect, it } from "vitest"; +import { streamSimple as streamAnthropic } from "../src/api/anthropic-messages.ts"; +import { streamSimple as streamAzure } from "../src/api/azure-openai-responses.ts"; +import { streamSimple as streamGoogle } from "../src/api/google-generative-ai.ts"; +import { streamSimple as streamMistral } from "../src/api/mistral-conversations.ts"; +import { streamSimple as streamCodex } from "../src/api/openai-codex-responses.ts"; +import { streamSimple as streamOpenAICompletions } from "../src/api/openai-completions.ts"; +import { streamSimple as streamOpenAIResponses } from "../src/api/openai-responses.ts"; +import type { Api, AssistantMessageEventStream, Model } from "../src/types.ts"; + +function model(api: TApi): Model { + return { + id: "test-model", + name: "Test", + api, + provider: "test-provider", + baseUrl: "https://example.invalid", + reasoning: false, + input: ["text"], + cost: { input: 0, output: 0, cacheRead: 0, cacheWrite: 0 }, + contextWindow: 1_000, + maxTokens: 100, + }; +} + +function expectMissingAuthThrows(create: () => AssistantMessageEventStream): void { + expect(create).toThrow("No API key for provider: test-provider"); +} + +describe("direct API authentication", () => { + it("throws synchronously when auth is missing", () => { + expectMissingAuthThrows(() => streamAnthropic(model("anthropic-messages"), { messages: [] }, {})); + expectMissingAuthThrows(() => streamAzure(model("azure-openai-responses"), { messages: [] }, {})); + expectMissingAuthThrows(() => streamGoogle(model("google-generative-ai"), { messages: [] }, {})); + expectMissingAuthThrows(() => streamMistral(model("mistral-conversations"), { messages: [] }, {})); + expectMissingAuthThrows(() => streamCodex(model("openai-codex-responses"), { messages: [] }, {})); + expectMissingAuthThrows(() => streamOpenAICompletions(model("openai-completions"), { messages: [] }, {})); + expectMissingAuthThrows(() => streamOpenAIResponses(model("openai-responses"), { messages: [] }, {})); + }); +}); diff --git a/packages/ai/test/provider-error-body-regression.test.ts b/packages/ai/test/provider-error-body-regression.test.ts new file mode 100644 index 0000000000000000000000000000000000000000..d894c3db004a87be6475be95950debd11494683e --- /dev/null +++ b/packages/ai/test/provider-error-body-regression.test.ts @@ -0,0 +1,213 @@ +// Per-tier provider regression for issues/provider-error-body-passthrough. +// +// Routes a 403-with-body error through the real provider catch path for one +// representative per tier (Success Criterion 7): a body-blind text provider +// (openai-completions), a status-only provider (openai-responses), and a +// body-blind Bedrock provider. Each asserts the resulting errorMessage carries +// both the HTTP status and the body reason. The image-provider tier is covered +// by provider-error-body-passthrough.test.ts; the already-correct happy path +// (no double body / no duplicated status) is asserted via the shared helper in +// error-body.test.ts. + +import { beforeEach, describe, expect, it, vi } from "vitest"; +import { streamSimple as streamSimpleBedrock } from "../src/api/bedrock-converse-stream.ts"; +import { stream as streamOpenAICompletions } from "../src/api/openai-completions.ts"; +import { stream as streamOpenAIResponses } from "../src/api/openai-responses.ts"; +import type { Context, Model } from "../src/types.ts"; + +// openai SDK APIError shape: " status code (no body)" message, the +// parsed body kept on `.error`. +class FakeAPIError extends Error { + status: number; + error: unknown; + constructor(status: number, parsedBody: unknown) { + super(`${status} status code (no body)`); + this.name = "PermissionDeniedError"; + this.status = status; + this.error = parsedBody; + } +} + +const bedrockMock = vi.hoisted(() => ({ + sendError: undefined as unknown, +})); + +const openaiMock = vi.hoisted(() => ({ + // Default parsed body; individual tests may override before invoking. + parsedBody: { error: "blocked by gateway WAF" } as unknown, +})); + +vi.mock("openai", () => { + function throwingCreate() { + const promise = Promise.resolve(undefined) as unknown as { withResponse: () => Promise }; + promise.withResponse = async () => { + throw new FakeAPIError(403, openaiMock.parsedBody); + }; + return promise; + } + class FakeOpenAI { + chat = { completions: { create: throwingCreate } }; + responses = { create: throwingCreate }; + } + return { default: FakeOpenAI }; +}); + +vi.mock("@aws-sdk/client-bedrock-runtime", () => { + class BedrockRuntimeServiceException extends Error {} + + class BedrockRuntimeClient { + middlewareStack = { add: () => {} }; + send(): Promise { + return Promise.reject(bedrockMock.sendError); + } + } + + class ConverseStreamCommand { + readonly input: unknown; + constructor(input: unknown) { + this.input = input; + } + } + + return { + BedrockRuntimeClient, + BedrockRuntimeServiceException, + ConverseStreamCommand, + StopReason: { + END_TURN: "end_turn", + STOP_SEQUENCE: "stop_sequence", + MAX_TOKENS: "max_tokens", + MODEL_CONTEXT_WINDOW_EXCEEDED: "model_context_window_exceeded", + TOOL_USE: "tool_use", + }, + CachePointType: { DEFAULT: "default" }, + CacheTTL: { ONE_HOUR: "ONE_HOUR" }, + ConversationRole: { ASSISTANT: "assistant", USER: "user" }, + ImageFormat: { JPEG: "jpeg", PNG: "png", GIF: "gif", WEBP: "webp" }, + ToolResultStatus: { ERROR: "error", SUCCESS: "success" }, + }; +}); + +import { getModel } from "../src/compat.ts"; + +const context: Context = { + systemPrompt: "", + messages: [{ role: "user", content: [{ type: "text", text: "hi" }], timestamp: 0 }], + tools: [], +}; + +const completionsModel: Model<"openai-completions"> = { + id: "test-model", + name: "Test Model", + api: "openai-completions", + provider: "openrouter", + baseUrl: "https://openrouter.ai/api/v1", + reasoning: false, + input: ["text"], + cost: { input: 0, output: 0, cacheRead: 0, cacheWrite: 0 }, + contextWindow: 1000, + maxTokens: 100, +}; + +const responsesModel: Model<"openai-responses"> = { + id: "gpt-test", + name: "GPT Test", + api: "openai-responses", + provider: "openai", + baseUrl: "https://api.openai.com/v1", + reasoning: false, + input: ["text"], + cost: { input: 0, output: 0, cacheRead: 0, cacheWrite: 0 }, + contextWindow: 1000, + maxTokens: 100, +}; + +async function drainResult(stream: { + [Symbol.asyncIterator](): AsyncIterator; + result(): Promise<{ errorMessage?: string; stopReason?: string }>; +}) { + for await (const _event of stream) { + void _event; + } + return stream.result(); +} + +describe("provider error body passthrough (per-tier regression)", () => { + beforeEach(() => { + openaiMock.parsedBody = { error: "blocked by gateway WAF" }; + }); + + it("openai-completions (body-blind text) surfaces status + body", async () => { + const output = await drainResult(streamOpenAICompletions(completionsModel, context, { apiKey: "test" })); + + expect(output.stopReason).toBe("error"); + expect(output.errorMessage).toContain("403"); + expect(output.errorMessage).toContain("blocked by gateway WAF"); + expect(output.errorMessage).not.toBe("403 status code (no body)"); + }); + + it("openai-completions does not double-print the OpenRouter metadata.raw extra", async () => { + // OpenRouter returns the extra reason under error.error.metadata.raw, which + // is part of the parsed body normalizeProviderError already surfaces. The + // manual append must not duplicate it. + openaiMock.parsedBody = { + message: "Provider returned error", + code: 403, + metadata: { raw: "upstream WAF blocked policy XYZ" }, + }; + + const output = await drainResult(streamOpenAICompletions(completionsModel, context, { apiKey: "test" })); + + expect(output.errorMessage).toContain("upstream WAF blocked policy XYZ"); + const occurrences = output.errorMessage?.match(/upstream WAF blocked policy XYZ/g) ?? []; + expect(occurrences).toHaveLength(1); + }); + + it("openai-responses (status-only) keeps the prefix and surfaces the body", async () => { + const output = await drainResult(streamOpenAIResponses(responsesModel, context, { apiKey: "test" })); + + expect(output.stopReason).toBe("error"); + expect(output.errorMessage).toContain("OpenAI API error (403)"); + expect(output.errorMessage).toContain("blocked by gateway WAF"); + }); + + it("bedrock (body-blind) surfaces the gateway body instead of Unknown: UnknownError", async () => { + bedrockMock.sendError = Object.assign(new Error("UnknownError"), { + name: "UnknownError", + $metadata: { httpStatusCode: 403 }, + $response: { statusCode: 403, body: '{"message":"blocked by gateway WAF"}' }, + }); + + const model = getModel("amazon-bedrock", "us.anthropic.claude-opus-4-8"); + const output = await drainResult(streamSimpleBedrock(model, { messages: context.messages }, {})); + + expect(output.stopReason).toBe("error"); + expect(output.errorMessage).toContain("403"); + expect(output.errorMessage).toContain("blocked by gateway WAF"); + expect(output.errorMessage).not.toContain("Unknown: UnknownError"); + }); + + it("bedrock preserves the SDK validation message when the response body is a stream", async () => { + bedrockMock.sendError = Object.assign( + new Error( + "Invocation of model ID anthropic.claude-opus-5 with on-demand throughput isn't supported. Retry with an inference profile.", + ), + { + name: "ValidationException", + $metadata: { httpStatusCode: 400 }, + $response: { + statusCode: 400, + body: { pipe: () => undefined, _readableState: { buffer: [], length: 0 } }, + }, + }, + ); + + const model = getModel("amazon-bedrock", "global.anthropic.claude-opus-5"); + const output = await drainResult(streamSimpleBedrock(model, { messages: context.messages }, {})); + + expect(output.stopReason).toBe("error"); + expect(output.errorMessage).toContain("on-demand throughput isn't supported"); + expect(output.errorMessage).toContain("inference profile"); + expect(output.errorMessage).not.toContain("_readableState"); + }); +}); diff --git a/packages/ai/test/providers.test.ts b/packages/ai/test/providers.test.ts new file mode 100644 index 0000000000000000000000000000000000000000..14f0b20a09cf11a8cbf430ff0a276c3a3db0fbaf --- /dev/null +++ b/packages/ai/test/providers.test.ts @@ -0,0 +1,629 @@ +import { describe, expect, it } from "vitest"; +import { lazyApi } from "../src/api/lazy.ts"; +import { envApiKeyAuth } from "../src/auth/helpers.ts"; +import type { AuthContext, AuthEvent } from "../src/auth/types.ts"; +import { createModels, createProvider } from "../src/models.ts"; +import { InMemoryModelsStore } from "../src/models-store.ts"; +import { builtinModels, builtinProviders, getBuiltinModel } from "../src/providers/all.ts"; +import { amazonBedrockProvider } from "../src/providers/amazon-bedrock.ts"; +import { anthropicProvider } from "../src/providers/anthropic.ts"; +import { cloudflareAIGatewayProvider } from "../src/providers/cloudflare-ai-gateway.ts"; +import { cloudflareWorkersAIProvider } from "../src/providers/cloudflare-workers-ai.ts"; +import { fauxAssistantMessage, fauxProvider } from "../src/providers/faux.ts"; +import { googleVertexProvider } from "../src/providers/google-vertex.ts"; +import type { + Api, + Context, + DeferredCancelOptions, + DeferredFetchOptions, + DeferredHandle, + Model, + ProviderStreams, +} from "../src/types.ts"; +import { AssistantMessageEventStream } from "../src/utils/event-stream.ts"; + +function fakeAuthContext(env: Record, files: string[] = []): AuthContext { + return { + env: async (name) => env[name], + fileExists: async (path) => files.includes(path), + }; +} + +const neverAbortedSignal = new AbortController().signal; + +const context: Context = { messages: [{ role: "user", content: "hi", timestamp: Date.now() }] }; + +describe("builtin providers", () => { + it("builtinModels registers every builtin provider with models", async () => { + const models = builtinModels(); + const providers = models.getProviders(); + expect(providers.length).toBe(builtinProviders().length); + expect(providers.map((p) => p.id)).toContain("anthropic"); + + const anthropic = models.getModel("anthropic", "claude-haiku-4-5"); + expect(anthropic?.api).toBe("anthropic-messages"); + + const all = models.getModels(); + expect(all.length).toBeGreaterThan(500); + + // Static providers list models immediately; Radius is purely dynamic. + for (const provider of providers) { + const list = models.getModels(provider.id); + if (provider.id === "radius") expect(list).toEqual([]); + else expect(list.length).toBeGreaterThan(0); + expect(list.every((m) => m.provider === provider.id)).toBe(true); + } + }); + + it("stores native constrained-sampling capabilities in model metadata", () => { + const gpt4o = getBuiltinModel("openai", "gpt-4o"); + expect(gpt4o.compat?.supportsStrictMode).toBe(true); + expect(gpt4o.compat?.supportsOpenAIGrammarTools).toBeUndefined(); + expect(getBuiltinModel("openai", "gpt-5.4").compat).toMatchObject({ + supportsStrictMode: true, + supportsOpenAIGrammarTools: true, + }); + expect(getBuiltinModel("anthropic", "claude-haiku-4-5").compat?.supportsStrictTools).toBe(true); + }); + + it("uses official Kimi K3 pricing for Moonshot providers", () => { + const models = builtinModels(); + for (const provider of ["moonshotai", "moonshotai-cn"]) { + expect(models.getModel(provider, "kimi-k3")?.cost).toEqual({ + input: 3, + output: 15, + cacheRead: 0.3, + cacheWrite: 0, + }); + } + }); + + it("uses API-equivalent implied pricing for Kimi Coding subscription models", () => { + const models = builtinModels(); + const expectedCosts = { + k3: { input: 3, output: 15, cacheRead: 0.3, cacheWrite: 0 }, + "kimi-for-coding-highspeed": { input: 1.9, output: 8, cacheRead: 0.38, cacheWrite: 0 }, + }; + + for (const [modelId, cost] of Object.entries(expectedCosts)) { + expect(models.getModel("kimi-coding", modelId)?.cost).toEqual(cost); + } + }); + + it("resolves Anthropic bearer auth from env with auth token precedence", async () => { + const models = createModels({ + authContext: fakeAuthContext({ + ANTHROPIC_AUTH_TOKEN: "auth-token", + ANTHROPIC_OAUTH_TOKEN: "oauth-token", + ANTHROPIC_API_KEY: "api-key", + }), + }); + models.setProvider(anthropicProvider()); + + expect(await models.getAuth("anthropic")).toEqual({ + auth: { headers: { Authorization: "Bearer auth-token" } }, + source: "ANTHROPIC_AUTH_TOKEN", + }); + }); + + it("preserves Anthropic OAuth token precedence over the API key", async () => { + const models = createModels({ + authContext: fakeAuthContext({ ANTHROPIC_API_KEY: "key", ANTHROPIC_OAUTH_TOKEN: "oauth-token" }), + }); + models.setProvider(anthropicProvider()); + + const result = await models.getAuth("anthropic"); + expect(result?.auth.apiKey).toBe("oauth-token"); + expect(result?.source).toBe("ANTHROPIC_OAUTH_TOKEN"); + }); + + it("runs provider-owned Bedrock bearer token and AWS profile login flows", async () => { + const auth = amazonBedrockProvider().auth.apiKey!; + const bearerAnswers = ["bearer-token", "bedrock-token"]; + expect( + await auth.login?.({ + signal: neverAbortedSignal, + prompt: async () => bearerAnswers.shift()!, + notify: () => {}, + }), + ).toEqual({ type: "api_key", key: "bedrock-token" }); + + const profileAnswers = ["aws-profile", "work"]; + const events: AuthEvent[] = []; + expect( + await auth.login?.({ + signal: neverAbortedSignal, + prompt: async () => profileAnswers.shift()!, + notify: (event) => events.push(event), + }), + ).toEqual({ type: "api_key", env: { AWS_PROFILE: "work" } }); + expect(events).toEqual([ + expect.objectContaining({ + type: "info", + links: [expect.objectContaining({ label: "AWS credential provider chain" })], + }), + ]); + expect( + await auth.resolve({ + ctx: fakeAuthContext({}), + credential: { type: "api_key", env: { AWS_PROFILE: "work" } }, + signal: neverAbortedSignal, + }), + ).toMatchObject({ auth: {}, env: { AWS_PROFILE: "work" } }); + }); + + it("reports bedrock as configured from ambient AWS credentials without an api key", async () => { + const models = createModels({ authContext: fakeAuthContext({ AWS_PROFILE: "dev" }) }); + models.setProvider(amazonBedrockProvider()); + const model = models.getModels("amazon-bedrock")[0]; + + const result = await models.getAuth(model.provider); + expect(result?.auth).toEqual({}); + expect(result?.source).toBe("AWS_PROFILE"); + + const unconfigured = createModels({ authContext: fakeAuthContext({}) }); + unconfigured.setProvider(amazonBedrockProvider()); + expect(await unconfigured.getAuth(model.provider)).toBeUndefined(); + }); + + it("requires Cloudflare Workers AI account config and returns scoped env", async () => { + const missingAccount = createModels({ authContext: fakeAuthContext({ CLOUDFLARE_API_KEY: "cf-key" }) }); + missingAccount.setProvider(cloudflareWorkersAIProvider()); + const model = missingAccount.getModels("cloudflare-workers-ai")[0]; + expect(await missingAccount.getAuth(model.provider)).toBeUndefined(); + + const configured = createModels({ + authContext: fakeAuthContext({ CLOUDFLARE_API_KEY: "cf-key", CLOUDFLARE_ACCOUNT_ID: "account-id" }), + }); + configured.setProvider(cloudflareWorkersAIProvider()); + const result = await configured.getAuth(model.provider); + expect(result?.auth).toEqual({ apiKey: "cf-key" }); + expect(result?.env).toEqual({ CLOUDFLARE_ACCOUNT_ID: "account-id" }); + }); + + it("requires Cloudflare AI Gateway account and gateway config and returns scoped env headers", async () => { + const missingGateway = createModels({ + authContext: fakeAuthContext({ CLOUDFLARE_API_KEY: "cf-key", CLOUDFLARE_ACCOUNT_ID: "account-id" }), + }); + missingGateway.setProvider(cloudflareAIGatewayProvider()); + const model = missingGateway.getModels("cloudflare-ai-gateway")[0]; + expect(await missingGateway.getAuth(model.provider)).toBeUndefined(); + + const configured = createModels({ + authContext: fakeAuthContext({ + CLOUDFLARE_API_KEY: "cf-key", + CLOUDFLARE_ACCOUNT_ID: "account-id", + CLOUDFLARE_GATEWAY_ID: "gateway-id", + }), + }); + configured.setProvider(cloudflareAIGatewayProvider()); + const result = await configured.getAuth(model.provider); + expect(result?.auth).toEqual({ + headers: { + "cf-aig-authorization": "Bearer cf-key", + Authorization: null, + "x-api-key": null, + }, + }); + expect(result?.env).toEqual({ + CLOUDFLARE_ACCOUNT_ID: "account-id", + CLOUDFLARE_GATEWAY_ID: "gateway-id", + }); + }); + + it("runs provider-owned Vertex API key and ADC login flows", async () => { + const auth = googleVertexProvider().auth.apiKey!; + const keyAnswers = ["api-key", "vertex-key"]; + expect( + await auth.login?.({ + signal: neverAbortedSignal, + prompt: async () => keyAnswers.shift()!, + notify: () => {}, + }), + ).toEqual({ type: "api_key", key: "vertex-key" }); + + const adcAnswers = ["adc", "project-id", "us-central1"]; + const events: AuthEvent[] = []; + expect( + await auth.login?.({ + signal: neverAbortedSignal, + prompt: async () => adcAnswers.shift()!, + notify: (event) => events.push(event), + }), + ).toEqual({ + type: "api_key", + env: { GOOGLE_CLOUD_PROJECT: "project-id", GOOGLE_CLOUD_LOCATION: "us-central1" }, + }); + expect(events).toEqual([ + expect.objectContaining({ + type: "info", + links: [expect.objectContaining({ label: "Application Default Credentials" })], + }), + ]); + expect( + await auth.resolve({ + ctx: fakeAuthContext({}, ["~/.config/gcloud/application_default_credentials.json"]), + credential: { + type: "api_key", + env: { GOOGLE_CLOUD_PROJECT: "project-id", GOOGLE_CLOUD_LOCATION: "us-central1" }, + }, + signal: neverAbortedSignal, + }), + ).toMatchObject({ + auth: {}, + env: { GOOGLE_CLOUD_PROJECT: "project-id", GOOGLE_CLOUD_LOCATION: "us-central1" }, + }); + }); + + it("resolves vertex via ADC file plus project and location", async () => { + const adc = "~/.config/gcloud/application_default_credentials.json"; + const configured = createModels({ + authContext: fakeAuthContext({ GOOGLE_CLOUD_PROJECT: "proj", GOOGLE_CLOUD_LOCATION: "us-central1" }, [adc]), + }); + configured.setProvider(googleVertexProvider()); + const model = configured.getModels("google-vertex")[0]; + + const result = await configured.getAuth(model.provider); + expect(result?.auth).toEqual({}); + expect(result?.source).toContain("application default"); + + // ADC without project/location is not configured + const partial = createModels({ authContext: fakeAuthContext({ GOOGLE_CLOUD_PROJECT: "proj" }, [adc]) }); + partial.setProvider(googleVertexProvider()); + expect(await partial.getAuth(model.provider)).toBeUndefined(); + + // explicit key wins over ADC + const keyed = createModels({ authContext: fakeAuthContext({ GOOGLE_CLOUD_API_KEY: "vertex-key" }) }); + keyed.setProvider(googleVertexProvider()); + expect((await keyed.getAuth(model.provider))?.auth.apiKey).toBe("vertex-key"); + }); +}); + +describe("envApiKeyAuth", () => { + it("prefers the stored credential key and falls back through env vars in order", async () => { + const auth = envApiKeyAuth("Test key", ["FIRST_KEY", "SECOND_KEY"]); + + const stored = await auth.resolve({ + ctx: fakeAuthContext({ FIRST_KEY: "env" }), + credential: { type: "api_key", key: "stored" }, + signal: neverAbortedSignal, + }); + expect(stored?.auth.apiKey).toBe("stored"); + expect(stored?.source).toBe("stored credential"); + + const second = await auth.resolve({ ctx: fakeAuthContext({ SECOND_KEY: "second" }), signal: neverAbortedSignal }); + expect(second?.auth.apiKey).toBe("second"); + expect(second?.source).toBe("SECOND_KEY"); + + expect(await auth.resolve({ ctx: fakeAuthContext({}), signal: neverAbortedSignal })).toBeUndefined(); + }); + + it("login prompts for a secret and returns an api-key credential", async () => { + const auth = envApiKeyAuth("Test key", ["TEST_KEY"]); + const credential = await auth.login?.({ + signal: neverAbortedSignal, + prompt: async (prompt) => { + expect(prompt.type).toBe("secret"); + return "entered-key"; + }, + notify: () => {}, + }); + expect(credential).toEqual({ type: "api_key", key: "entered-key" }); + }); +}); + +describe("createProvider", () => { + function recordingStreams(label: string, calls: string[]): ProviderStreams { + const respond = (model: Model) => { + calls.push(`${label}:${model.id}`); + const stream = new AssistantMessageEventStream(); + const message = fauxAssistantMessage("ok"); + stream.push({ type: "start", partial: message }); + stream.push({ type: "done", reason: "stop", message }); + stream.end(message); + return stream; + }; + return { stream: respond, streamSimple: respond }; + } + + function testModel(api: string, id: string): Model { + return { + id, + name: id, + api, + provider: "mixed", + baseUrl: "https://example.test/v1", + reasoning: false, + input: ["text"], + cost: { input: 0, output: 0, cacheRead: 0, cacheWrite: 0 }, + contextWindow: 10000, + maxTokens: 1000, + }; + } + + it("lazily exposes only declared deferred capabilities", async () => { + let loads = 0; + const streams = recordingStreams("deferred", []); + streams.fetchDeferred = (model) => streams.streamSimple(model, context); + const api = lazyApi( + async () => { + loads++; + return streams; + }, + { fetchDeferred: true }, + ); + const model = testModel("api-a", "model-a"); + const handle: DeferredHandle = { + provider: model.provider, + modelId: model.id, + api: model.api, + id: "response-1", + }; + + expect(loads).toBe(0); + expect(api.cancelDeferred).toBeUndefined(); + expect((await api.fetchDeferred!(model, handle).result()).stopReason).toBe("stop"); + expect(loads).toBe(1); + }); + + it("dispatches on model.api for mixed-API providers", async () => { + const calls: string[] = []; + const provider = createProvider({ + id: "mixed", + auth: { apiKey: { name: "Test", resolve: async () => ({ auth: {} }) } }, + models: [testModel("api-a", "model-a"), testModel("api-b", "model-b")], + api: { "api-a": recordingStreams("a", calls), "api-b": recordingStreams("b", calls) }, + }); + const models = createModels(); + models.setProvider(provider); + + await models.completeSimple(testModel("api-a", "model-a"), context); + await models.completeSimple(testModel("api-b", "model-b"), context); + expect(calls).toEqual(["a:model-a", "b:model-b"]); + }); + + it("merges provider-resolved env into stream options", async () => { + let capturedEnv: Record | undefined; + let capturedApiKey: string | undefined; + const envModel = { ...testModel("api-a", "model-a"), provider: "env-provider" }; + const provider = createProvider({ + id: "env-provider", + auth: { + apiKey: { + name: "Test", + resolve: async () => ({ + auth: { apiKey: "provider-key" }, + env: { PROVIDER_ONLY: "provider", SHARED: "provider" }, + }), + }, + }, + models: [envModel], + api: { + stream: (model, _context, options) => { + capturedEnv = options?.env; + capturedApiKey = options?.apiKey; + return recordingStreams("a", []).stream(model, _context, options); + }, + streamSimple: (model, _context, options) => { + capturedEnv = options?.env; + capturedApiKey = options?.apiKey; + return recordingStreams("a", []).streamSimple(model, _context, options); + }, + }, + }); + const models = createModels(); + models.setProvider(provider); + + await models.completeSimple(envModel, context, { + apiKey: "request-key", + env: { REQUEST_ONLY: "request", SHARED: "request" }, + }); + + expect(capturedApiKey).toBe("request-key"); + expect(capturedEnv).toEqual({ PROVIDER_ONLY: "provider", REQUEST_ONLY: "request", SHARED: "request" }); + }); + + it("applies resolved request options to deferred fetch and cancellation", async () => { + let fetchedModel: Model | undefined; + let fetchedOptions: DeferredFetchOptions | undefined; + let cancelledOptions: DeferredCancelOptions | undefined; + const deferredModel = { ...testModel("api-a", "model-a"), provider: "deferred-provider" }; + const streams = recordingStreams("deferred", []); + streams.fetchDeferred = (model, _handle, options) => { + fetchedModel = model; + fetchedOptions = options; + return streams.streamSimple(model, context); + }; + streams.cancelDeferred = async (_model, _handle, options) => { + cancelledOptions = options; + }; + const provider = createProvider({ + id: "deferred-provider", + auth: { + apiKey: { + name: "Test", + resolve: async () => ({ + auth: { + apiKey: "provider-key", + baseUrl: "https://resolved.test/v1", + headers: { Authorization: "Bearer provider", "X-Shared": "provider" }, + }, + env: { PROVIDER_ONLY: "provider", SHARED: "provider" }, + }), + }, + }, + models: [deferredModel], + api: streams, + }); + const models = createModels(); + models.setProvider(provider); + const handle: DeferredHandle = { + provider: deferredModel.provider, + modelId: deferredModel.id, + api: deferredModel.api, + id: "response-1", + }; + + await models.fetchDeferred(deferredModel, handle, { + wait: 50, + timeoutMs: 100, + apiKey: "request-key", + headers: { "X-Request": "request", "x-shared": "request" }, + env: { REQUEST_ONLY: "request", SHARED: "request" }, + transformHeaders: (headers) => ({ ...headers, "X-Transformed": "yes" }), + }); + await models.cancelDeferred(deferredModel, handle, { + timeoutMs: 200, + transformHeaders: (headers) => ({ ...headers, "X-Cancel": "yes" }), + }); + + expect(fetchedModel?.baseUrl).toBe("https://resolved.test/v1"); + expect(fetchedOptions).toMatchObject({ + wait: 50, + timeoutMs: 100, + apiKey: "request-key", + headers: { + Authorization: "Bearer provider", + "X-Request": "request", + "x-shared": "request", + "X-Transformed": "yes", + }, + env: { PROVIDER_ONLY: "provider", REQUEST_ONLY: "request", SHARED: "request" }, + }); + expect(cancelledOptions).toMatchObject({ + timeoutMs: 200, + apiKey: "provider-key", + headers: { Authorization: "Bearer provider", "X-Shared": "provider", "X-Cancel": "yes" }, + env: { PROVIDER_ONLY: "provider", SHARED: "provider" }, + }); + }); + + it("produces a stream error for a model whose api has no implementation", async () => { + const provider = createProvider({ + id: "mixed", + auth: { apiKey: { name: "Test", resolve: async () => ({ auth: {} }) } }, + models: [testModel("api-a", "model-a")], + api: { "api-a": recordingStreams("a", []) }, + }); + const result = await provider.streamSimple(testModel("api-ghost", "model-x"), context).result(); + expect(result.stopReason).toBe("error"); + expect(result.errorMessage).toContain("no API implementation"); + }); + + it("lets a newer dynamic refresh bypass and supersede older network work", async () => { + let fetches = 0; + let markFirstStarted: (() => void) | undefined; + let finishFirst: (() => void) | undefined; + const firstStarted = new Promise((resolve) => { + markFirstStarted = resolve; + }); + const firstBlocked = new Promise((resolve) => { + finishFirst = resolve; + }); + const provider = createProvider({ + id: "dynamic", + auth: { apiKey: { name: "Test", resolve: async () => ({ auth: {} }) } }, + models: [], + fetchModels: async () => { + fetches++; + const current = fetches; + if (current === 1) { + markFirstStarted?.(); + await firstBlocked; + } + return [testModel("api-a", `listed-${current}`)]; + }, + api: recordingStreams("a", []), + }); + + const store = new InMemoryModelsStore(); + const models = createModels({ modelsStore: store }); + models.setProvider(provider); + expect(provider.getModels()).toEqual([]); + + const first = models.refresh({ providers: ["dynamic"] }); + await firstStarted; + const second = models.refresh({ providers: ["dynamic"] }); + await expect(second).resolves.toMatchObject({ aborted: false }); + await expect(first).resolves.toMatchObject({ aborted: false }); + expect(fetches).toBe(2); + expect(provider.getModels().map((model) => model.id)).toEqual(["listed-2"]); + expect((await store.read("dynamic"))?.models.map((model) => model.id)).toEqual(["listed-2"]); + + finishFirst?.(); + await new Promise((resolve) => setTimeout(resolve, 0)); + expect(provider.getModels().map((model) => model.id)).toEqual(["listed-2"]); + expect((await store.read("dynamic"))?.models.map((model) => model.id)).toEqual(["listed-2"]); + }); +}); + +describe("fauxProvider", () => { + it("streams queued responses through a Models collection", async () => { + const faux = fauxProvider(); + const models = createModels(); + models.setProvider(faux.provider); + faux.setResponses([fauxAssistantMessage("hello from faux")]); + + const model = models.getModels(faux.provider.id)[0]; + const result = await models.completeSimple(model, context); + expect(result.stopReason).toBe("stop"); + expect(result.content).toEqual([{ type: "text", text: "hello from faux" }]); + expect(faux.state.callCount).toBe(1); + }); + + it("submits, polls, and redeems deferred responses", async () => { + const faux = fauxProvider({ deferred: { pendingFetches: 1, pollAfterMs: 25 } }); + const models = createModels(); + models.setProvider(faux.provider); + faux.setResponses([fauxAssistantMessage("ready")]); + const model = faux.getModel(); + + const submission = models.streamSimple(model, context, { deferred: { window: "1h" } }); + const eventTypes: string[] = []; + for await (const event of submission) eventTypes.push(event.type); + const deferred = await submission.result(); + expect(eventTypes).toEqual(["start", "done"]); + expect(deferred).toMatchObject({ stopReason: "deferred", content: [] }); + expect(deferred.deferred).toEqual({ + provider: model.provider, + modelId: model.id, + api: model.api, + id: expect.any(String), + pollAfterMs: 25, + }); + if (!deferred.deferred) throw new Error("Faux response did not include a deferred handle"); + + const pending = await models.fetchDeferred(model, deferred.deferred); + expect(pending.stopReason).toBe("deferred"); + expect(pending.deferred).toEqual(deferred.deferred); + + const ready = await models.fetchDeferred(model, deferred.deferred, { wait: 0 }); + expect(ready.stopReason).toBe("stop"); + expect(ready.content).toEqual([{ type: "text", text: "ready" }]); + expect(ready.usage.totalTokens).toBeGreaterThan(0); + expect(faux.state).toMatchObject({ callCount: 1, deferredFetchCount: 2 }); + }); + + it("records cancellation and returns deferred fetch failures in-band", async () => { + const faux = fauxProvider(); + const models = createModels(); + models.setProvider(faux.provider); + faux.setResponses([() => Promise.reject(new Error("deferred failed")), fauxAssistantMessage("cancelled")]); + const model = faux.getModel(); + + const failedSubmission = await models.completeSimple(model, context, { deferred: true }); + if (!failedSubmission.deferred) throw new Error("Faux response did not include a deferred handle"); + const failed = await models.fetchDeferred(model, failedSubmission.deferred); + expect(failed).toMatchObject({ stopReason: "error", errorMessage: "deferred failed" }); + + const cancelledSubmission = await models.completeSimple(model, context, { deferred: true }); + if (!cancelledSubmission.deferred) throw new Error("Faux response did not include a deferred handle"); + await models.cancelDeferred(model, cancelledSubmission.deferred); + expect(faux.state.cancelledDeferred).toEqual([cancelledSubmission.deferred]); + const cancelled = await models.fetchDeferred(model, cancelledSubmission.deferred); + expect(cancelled).toMatchObject({ + stopReason: "error", + errorMessage: expect.stringContaining("was cancelled"), + }); + }); +}); diff --git a/packages/ai/test/qwen-token-plan-models.test.ts b/packages/ai/test/qwen-token-plan-models.test.ts new file mode 100644 index 0000000000000000000000000000000000000000..a3d463684cdf1aba923f245f443e3566b54feb63 --- /dev/null +++ b/packages/ai/test/qwen-token-plan-models.test.ts @@ -0,0 +1,293 @@ +import { describe, expect, it, vi } from "vitest"; +import { getModels, streamSimple } from "../src/compat.ts"; +import { findEnvKeys } from "../src/env-api-keys.ts"; + +vi.mock("openai", () => { + class FakeOpenAI { + chat = { + completions: { + create: () => { + const stream = { + async *[Symbol.asyncIterator]() { + yield { + choices: [{ delta: {}, finish_reason: "stop" }], + usage: { + prompt_tokens: 1, + completion_tokens: 1, + prompt_tokens_details: { cached_tokens: 0 }, + completion_tokens_details: { reasoning_tokens: 0 }, + }, + }; + }, + }; + const promise = Promise.resolve(stream) as Promise & { + withResponse: () => Promise<{ + data: typeof stream; + response: { status: number; headers: Headers }; + }>; + }; + promise.withResponse = async () => ({ + data: stream, + response: { status: 200, headers: new Headers() }, + }); + return promise; + }, + }, + }; + } + + return { default: FakeOpenAI }; +}); + +const TEXT_MODELS = [ + "MiniMax-M2.5", + "deepseek-v3.2", + "deepseek-v4-flash", + "deepseek-v4-pro", + "glm-5", + "glm-5.1", + "glm-5.2", + "kimi-k2.5", + "kimi-k2.6", + "kimi-k2.7-code", + "qwen3.6-flash", + "qwen3.6-plus", + "qwen3.7-max", + "qwen3.7-plus", + "qwen3.8-flash", + "qwen3.8-max", +]; + +const INDIVIDUAL_TEXT_MODELS = [ + "deepseek-v4-flash-0731", + "deepseek-v4-pro", + "deepseek-v4-pro-0813", + "glm-5.2", + "qwen3.6-flash", + "qwen3.7-max", + "qwen3.7-plus", + "qwen3.8-flash", + "qwen3.8-max", +]; + +const IMAGE_MODELS = ["qwen-image-2.0", "qwen-image-2.0-pro", "wan2.7-image", "wan2.7-image-pro"]; + +const QWEN_THINKING_MODELS = [ + "deepseek-v3.2", + "deepseek-v4-flash", + "deepseek-v4-pro", + "glm-5", + "glm-5.1", + "glm-5.2", + "kimi-k2.5", + "kimi-k2.6", + "kimi-k2.7-code", + "qwen3.6-flash", + "qwen3.6-plus", + "qwen3.7-max", + "qwen3.7-plus", + "qwen3.8-flash", + "qwen3.8-max", +] as const; + +type QwenTokenPlanProvider = "qwen-token-plan" | "qwen-token-plan-cn" | "qwen-token-plan-individual"; +type QwenTokenPlanModelCase = { provider: QwenTokenPlanProvider; modelId: string }; + +const QWEN_THINKING_MODEL_CASES: QwenTokenPlanModelCase[] = [ + ...(["qwen-token-plan", "qwen-token-plan-cn"] as const).flatMap((provider) => + QWEN_THINKING_MODELS.map((modelId) => ({ provider, modelId })), + ), + ...INDIVIDUAL_TEXT_MODELS.map((modelId) => ({ provider: "qwen-token-plan-individual" as const, modelId })), +]; + +const QWEN_REASONING_EFFORT_MODELS = ["deepseek-v4-flash", "deepseek-v4-pro", "glm-5", "glm-5.1", "glm-5.2"] as const; +const QWEN38_MODELS = ["qwen3.8-flash", "qwen3.8-max"] as const; + +const QWEN_REASONING_EFFORT_MODEL_CASES: QwenTokenPlanModelCase[] = [ + ...(["qwen-token-plan", "qwen-token-plan-cn"] as const).flatMap((provider) => + QWEN_REASONING_EFFORT_MODELS.map((modelId) => ({ provider, modelId })), + ), + ...["deepseek-v4-flash-0731", "deepseek-v4-pro", "deepseek-v4-pro-0813", "glm-5.2"].map((modelId) => ({ + provider: "qwen-token-plan-individual" as const, + modelId, + })), +]; + +const QWEN38_MODEL_CASES: QwenTokenPlanModelCase[] = ( + ["qwen-token-plan", "qwen-token-plan-cn", "qwen-token-plan-individual"] as const +).flatMap((provider) => QWEN38_MODELS.map((modelId) => ({ provider, modelId }))); + +describe("Qwen Token Plan models", () => { + // #9021 + it("exposes exactly the documented Individual text models", () => { + const modelIds = getModels("qwen-token-plan-individual") + .map((model) => model.id) + .sort(); + + expect(modelIds).toEqual([...INDIVIDUAL_TEXT_MODELS].sort()); + }); + + it("reuses the international Token Plan environment variable", () => { + expect(findEnvKeys("qwen-token-plan-individual", { QWEN_TOKEN_PLAN_API_KEY: "test" })).toEqual([ + "QWEN_TOKEN_PLAN_API_KEY", + ]); + }); + + it.each(["qwen-token-plan", "qwen-token-plan-cn"] as const)("exposes all text models on %s", (provider) => { + const modelIds = getModels(provider).map((model) => model.id); + for (const expected of TEXT_MODELS) { + expect(modelIds, `${provider} should include ${expected}`).toContain(expected); + } + }); + + it.each(["qwen-token-plan", "qwen-token-plan-cn"] as const)("omits image models from %s", (provider) => { + const modelIds = getModels(provider).map((model) => model.id); + for (const excluded of IMAGE_MODELS) { + expect(modelIds, `${provider} should not include ${excluded}`).not.toContain(excluded); + } + }); + + // docs: https://modelstudio.console.alibabacloud.com/ap-southeast-1?tab=api&commonbuy=1#/api/?type=model&url=3016807 + it.each(QWEN_THINKING_MODEL_CASES)( + "sends Qwen thinking fields for $provider/$modelId", + async ({ provider, modelId }) => { + const model = getModels(provider).find((candidate) => candidate.id === modelId); + expect(model).toBeDefined(); + if (!model) throw new Error(`Missing model: ${provider}/${modelId}`); + + let payload: unknown; + await streamSimple( + model, + { + messages: [ + { + role: "user", + content: "Hi", + timestamp: Date.now(), + }, + ], + }, + { + apiKey: "test", + reasoning: "high", + onPayload: (params) => { + payload = params; + }, + }, + ).result(); + + expect(payload).toHaveProperty("enable_thinking", true); + expect(payload).not.toHaveProperty("thinking"); + }, + ); + + it.each(QWEN_REASONING_EFFORT_MODEL_CASES)( + "exposes Qwen reasoning_effort levels for $provider/$modelId", + ({ provider, modelId }) => { + const model = getModels(provider).find((candidate) => candidate.id === modelId); + expect(model).toBeDefined(); + if (!model) throw new Error(`Missing model: ${provider}/${modelId}`); + + expect(model.thinkingLevelMap).toMatchObject({ + minimal: null, + low: null, + medium: null, + high: "high", + xhigh: null, + max: "max", + }); + }, + ); + + it.each(QWEN38_MODEL_CASES)( + "exposes qwen3.8 reasoning_effort levels for $provider/$modelId", + ({ provider, modelId }) => { + const model = getModels(provider).find((candidate) => candidate.id === modelId); + expect(model).toBeDefined(); + if (!model) throw new Error(`Missing model: ${provider}/${modelId}`); + + expect(model.thinkingLevelMap).toMatchObject({ + minimal: null, + low: "low", + medium: "medium", + high: null, + xhigh: "xhigh", + max: null, + }); + }, + ); + + it.each(["qwen-token-plan", "qwen-token-plan-cn", "qwen-token-plan-individual"] as const)( + "omits retired qwen3.8-max-preview on %s", + (provider) => { + const modelIds = getModels(provider).map((model) => model.id); + expect(modelIds).not.toContain("qwen3.8-max-preview"); + }, + ); + + it.each(QWEN_REASONING_EFFORT_MODEL_CASES)( + "sends Qwen reasoning_effort for $provider/$modelId", + async ({ provider, modelId }) => { + const model = getModels(provider).find((candidate) => candidate.id === modelId); + expect(model).toBeDefined(); + if (!model) throw new Error(`Missing model: ${provider}/${modelId}`); + + let payload: unknown; + await streamSimple( + model, + { + messages: [ + { + role: "user", + content: "Hi", + timestamp: Date.now(), + }, + ], + }, + { + apiKey: "test", + reasoning: "high", + onPayload: (params) => { + payload = params; + }, + }, + ).result(); + + expect(payload).toHaveProperty("reasoning_effort", "high"); + }, + ); + + it.each(QWEN38_MODEL_CASES)( + "sends qwen3.8 xhigh reasoning_effort for $provider/$modelId", + async ({ provider, modelId }) => { + const model = getModels(provider).find((candidate) => candidate.id === modelId); + expect(model).toBeDefined(); + if (!model) throw new Error(`Missing model: ${provider}/${modelId}`); + + let payload: unknown; + await streamSimple( + model, + { + messages: [ + { + role: "user", + content: "Hi", + timestamp: Date.now(), + }, + ], + }, + { + apiKey: "test", + reasoning: "xhigh", + onPayload: (params) => { + payload = params; + }, + }, + ).result(); + + expect(payload).toHaveProperty("enable_thinking", true); + expect(payload).toHaveProperty("reasoning_effort", "xhigh"); + expect(payload).not.toHaveProperty("thinking"); + }, + ); +}); diff --git a/packages/ai/test/reasoning-options.test.ts b/packages/ai/test/reasoning-options.test.ts new file mode 100644 index 0000000000000000000000000000000000000000..fafa9fa5ed1c1a54060b1d3aa8e3acfd780bd949 --- /dev/null +++ b/packages/ai/test/reasoning-options.test.ts @@ -0,0 +1,36 @@ +import { describe, expect, it } from "vitest"; +import { getEffortThinkingLevelMap } from "../scripts/models-dev-reasoning-options.ts"; + +describe("getEffortThinkingLevelMap", () => { + it("exposes only verified effort values and none", () => { + expect( + getEffortThinkingLevelMap([{ type: "toggle" }, { type: "effort", values: ["none", "low", "high", "max"] }]), + ).toEqual({ + off: "none", + minimal: null, + low: "low", + medium: null, + high: "high", + xhigh: null, + max: "max", + }); + }); + + it("does not infer thinking-off from an effort list", () => { + expect(getEffortThinkingLevelMap([{ type: "effort", values: ["low", "high", "max"] }])).toEqual({ + off: null, + minimal: null, + low: "low", + medium: null, + high: "high", + xhigh: null, + max: "max", + }); + }); + + it("leaves toggle and budget controls for their adapter-specific implementations", () => { + expect(getEffortThinkingLevelMap([{ type: "toggle" }])).toBeUndefined(); + expect(getEffortThinkingLevelMap([{ type: "budget_tokens", min: 1024, max: 32000 }])).toBeUndefined(); + expect(getEffortThinkingLevelMap([{ type: "effort", values: [null, "default"] }])).toBeUndefined(); + }); +}); diff --git a/packages/ai/test/responseid.test.ts b/packages/ai/test/responseid.test.ts new file mode 100644 index 0000000000000000000000000000000000000000..cd5660575776a8fa59123cd771552a3205c7db17 --- /dev/null +++ b/packages/ai/test/responseid.test.ts @@ -0,0 +1,120 @@ +import { describe, expect, it } from "vitest"; +import { complete, getModel } from "../src/compat.ts"; +import type { Api, Context, Model, StreamOptions } from "../src/types.ts"; +import { hasAzureOpenAICredentials, resolveAzureDeploymentName } from "./azure-utils.ts"; +import { resolveApiKey } from "./oauth.ts"; + +type StreamOptionsWithExtras = StreamOptions & Record; + +const oauthTokens = await Promise.all([resolveApiKey("github-copilot"), resolveApiKey("openai-codex")]); +const [githubCopilotToken, openaiCodexToken] = oauthTokens; + +async function expectResponseId(model: Model, options: StreamOptionsWithExtras = {}) { + const context: Context = { + systemPrompt: "You are a helpful assistant. Be concise.", + messages: [{ role: "user", content: "Reply with exactly: response id test", timestamp: Date.now() }], + }; + + const response = await complete(model, context, options); + + expect(response.stopReason, response.errorMessage).not.toBe("error"); + expect(response.responseId).toBeTruthy(); + expect(typeof response.responseId).toBe("string"); +} + +describe("responseId E2E Tests", () => { + describe.skipIf(!process.env.GEMINI_API_KEY)("Google Provider", () => { + const llm = getModel("google", "gemini-2.5-flash"); + + it("should expose responseId", { retry: 3, timeout: 30000 }, async () => { + await expectResponseId(llm); + }); + }); + + describe("Google Vertex Provider", () => { + const vertexProject = process.env.GOOGLE_CLOUD_PROJECT || process.env.GCLOUD_PROJECT; + const vertexLocation = process.env.GOOGLE_CLOUD_LOCATION; + const vertexApiKey = process.env.GOOGLE_CLOUD_API_KEY; + const isVertexConfigured = Boolean(vertexProject && vertexLocation); + const vertexOptions = { project: vertexProject, location: vertexLocation } as const; + const llm = getModel("google-vertex", "gemini-3-flash-preview"); + + it.skipIf(!isVertexConfigured)("should expose responseId with ADC", { retry: 3, timeout: 30000 }, async () => { + await expectResponseId(llm, vertexOptions); + }); + + it.skipIf(!vertexApiKey)("should expose responseId with API key", { retry: 3, timeout: 30000 }, async () => { + await expectResponseId(llm, { apiKey: vertexApiKey! }); + }); + }); + + describe.skipIf(!process.env.OPENAI_API_KEY)("OpenAI Completions Provider", () => { + const { compat: _compat, ...baseModel } = getModel("openai", "gpt-4o-mini"); + void _compat; + const llm: Model<"openai-completions"> = { + ...baseModel, + api: "openai-completions", + }; + + it("should expose responseId", { retry: 3, timeout: 30000 }, async () => { + await expectResponseId(llm); + }); + }); + + describe.skipIf(!process.env.OPENAI_API_KEY)("OpenAI Responses Provider", () => { + const llm = getModel("openai", "gpt-5-mini"); + + it("should expose responseId", { retry: 3, timeout: 30000 }, async () => { + await expectResponseId(llm); + }); + }); + + describe.skipIf(!process.env.ANTHROPIC_API_KEY)("Anthropic Provider", () => { + const llm = getModel("anthropic", "claude-sonnet-4-5"); + + it("should expose responseId", { retry: 3, timeout: 30000 }, async () => { + await expectResponseId(llm); + }); + }); + + describe.skipIf(!hasAzureOpenAICredentials())("Azure OpenAI Responses Provider", () => { + const llm = getModel("azure-openai-responses", "gpt-4o-mini"); + const azureDeploymentName = resolveAzureDeploymentName(llm.id); + const azureOptions = azureDeploymentName ? { azureDeploymentName } : {}; + + it("should expose responseId", { retry: 3, timeout: 30000 }, async () => { + await expectResponseId(llm, azureOptions); + }); + }); + + describe.skipIf(!process.env.MISTRAL_API_KEY)("Mistral Provider", () => { + const llm = getModel("mistral", "devstral-medium-latest"); + + it("should expose responseId", { retry: 3, timeout: 30000 }, async () => { + await expectResponseId(llm); + }); + }); + + describe("GitHub Copilot Provider", () => { + it.skipIf(!githubCopilotToken)("OpenAI path should expose responseId", { retry: 3, timeout: 30000 }, async () => { + const llm = getModel("github-copilot", "gpt-5.3-codex"); + await expectResponseId(llm, { apiKey: githubCopilotToken }); + }); + + it.skipIf(!githubCopilotToken)( + "Anthropic path should expose responseId", + { retry: 3, timeout: 30000 }, + async () => { + const llm = getModel("github-copilot", "claude-sonnet-4.6"); + await expectResponseId(llm, { apiKey: githubCopilotToken }); + }, + ); + }); + + describe("OpenAI Codex Provider", () => { + it.skipIf(!openaiCodexToken)("should expose responseId", { retry: 3, timeout: 30000 }, async () => { + const llm = getModel("openai-codex", "gpt-5.5"); + await expectResponseId(llm, { apiKey: openaiCodexToken }); + }); + }); +}); diff --git a/packages/ai/test/retry.test.ts b/packages/ai/test/retry.test.ts new file mode 100644 index 0000000000000000000000000000000000000000..96ed4a81d78e567861c1122ca26416fdba4329c9 --- /dev/null +++ b/packages/ai/test/retry.test.ts @@ -0,0 +1,249 @@ +import { describe, expect, it, vi } from "vitest"; +import { fauxAssistantMessage } from "../src/providers/faux.ts"; +import { isRetryableAssistantError, type RetryPolicy, retryAssistantCall, retryDelayMs } from "../src/utils/retry.ts"; + +const openAIExplicitRetryMessage = + "An error occurred while processing your request. You can retry your request, or contact us through our help center at help.openai.com if the error persists. Please include the request ID req_******** in your message."; +const bedrockExplicitRetryMessage = + '{"message":"The system encountered an unexpected error during processing. Try your request again."}'; +const nvidiaNIMResourceExhaustedMessage = "ResourceExhausted: Worker local total request limit reached (288/48)"; +const bunFetchSocketClosedMessage = + "The socket connection was closed unexpectedly. For more information, pass `verbose: true` in the second argument to fetch()"; +const openAIResponsesEarlyEofMessage = "OpenAI Responses stream ended before a terminal response event"; +const wrappedDnsLookupError = + "The pending stream has been canceled (caused by: getaddrinfo ENOTFOUND bedrock-runtime.us-east-1.amazonaws.com)"; + +describe("provider retry classification", () => { + it("matches explicit provider retry guidance", () => { + expect( + isRetryableAssistantError( + fauxAssistantMessage("", { stopReason: "error", errorMessage: openAIExplicitRetryMessage }), + ), + ).toBe(true); + expect( + isRetryableAssistantError( + fauxAssistantMessage("", { stopReason: "error", errorMessage: bedrockExplicitRetryMessage }), + ), + ).toBe(true); + expect( + isRetryableAssistantError( + fauxAssistantMessage("", { stopReason: "error", errorMessage: nvidiaNIMResourceExhaustedMessage }), + ), + ).toBe(true); + }); + + it("matches Bun fetch socket drop wording", () => { + expect( + isRetryableAssistantError( + fauxAssistantMessage("", { stopReason: "error", errorMessage: bunFetchSocketClosedMessage }), + ), + ).toBe(true); + }); + + it("matches upstream request buffer exhaustion wording", () => { + expect( + isRetryableAssistantError( + fauxAssistantMessage("", { + stopReason: "error", + errorMessage: "Error: exceeded request buffer limit while retrying upstream", + }), + ), + ).toBe(true); + }); + + it.each([ + wrappedDnsLookupError, + "connect ENOTFOUND api.example.com", + "EAI_AGAIN api.example.com", + "getaddrinfo failed for api.example.com", + ])("matches DNS transport failure wording: %s", (errorMessage) => { + expect(isRetryableAssistantError(fauxAssistantMessage("", { stopReason: "error", errorMessage }))).toBe(true); + }); + + it("matches OpenAI Responses streams that end before terminal events", () => { + expect( + isRetryableAssistantError( + fauxAssistantMessage("", { stopReason: "error", errorMessage: openAIResponsesEarlyEofMessage }), + ), + ).toBe(true); + }); + + it("keeps provider limit errors non-retryable", () => { + expect( + isRetryableAssistantError( + fauxAssistantMessage("", { stopReason: "error", errorMessage: "429 quota exceeded" }), + ), + ).toBe(false); + }); + + it("classifies assistant error messages", () => { + expect( + isRetryableAssistantError(fauxAssistantMessage("", { stopReason: "error", errorMessage: "overloaded_error" })), + ).toBe(true); + expect( + isRetryableAssistantError( + fauxAssistantMessage("", { stopReason: "error", errorMessage: "524 status code (no body)" }), + ), + ).toBe(true); + expect(isRetryableAssistantError(fauxAssistantMessage("not an error"))).toBe(false); + }); +}); + +describe("retryDelayMs", () => { + it("caps agent retry delay", () => { + // Regression for #8826. + expect(retryDelayMs({ baseDelayMs: 2000 }, 6)).toBe(60000); + expect(retryDelayMs({ baseDelayMs: 2000, maxAgentDelayMs: 5000 }, 5)).toBe(5000); + expect(retryDelayMs({ baseDelayMs: 2000, maxAgentDelayMs: 0 }, 5)).toBe(0); + }); +}); + +describe("retryAssistantCall", () => { + const disabled: RetryPolicy = { enabled: false, maxRetries: 3, baseDelayMs: 0 }; + const enabled: RetryPolicy = { enabled: true, maxRetries: 3, baseDelayMs: 0 }; + + it("returns a successful response immediately without retrying", async () => { + const produce = vi.fn(async () => fauxAssistantMessage("ok")); + const res = await retryAssistantCall(produce, enabled, undefined); + expect(res.content).toEqual([{ type: "text", text: "ok" }]); + expect(produce).toHaveBeenCalledTimes(1); + }); + + it("does not retry an aborted message", async () => { + const produce = vi.fn(async () => fauxAssistantMessage("", { stopReason: "aborted" })); + const onRetryScheduled = vi.fn(); + const res = await retryAssistantCall(produce, enabled, undefined, { onRetryScheduled }); + expect(res.stopReason).toBe("aborted"); + expect(produce).toHaveBeenCalledTimes(1); + expect(onRetryScheduled).not.toHaveBeenCalled(); + }); + + it("does not retry a non-retryable error (quota/billing)", async () => { + const produce = vi.fn(async () => + fauxAssistantMessage("", { stopReason: "error", errorMessage: "insufficient_quota" }), + ); + const onRetryScheduled = vi.fn(); + const onRetryFinished = vi.fn(); + const res = await retryAssistantCall(produce, enabled, undefined, { onRetryScheduled, onRetryFinished }); + expect(res.stopReason).toBe("error"); + expect(produce).toHaveBeenCalledTimes(1); + expect(onRetryScheduled).not.toHaveBeenCalled(); + expect(onRetryFinished).not.toHaveBeenCalled(); + }); + + it("retries a transient error up to maxRetries then returns the final error", async () => { + const produce = vi.fn(async () => fauxAssistantMessage("", { stopReason: "error", errorMessage: "terminated" })); + const onRetryScheduled = vi.fn(); + const onRetryFinished = vi.fn(); + const res = await retryAssistantCall(produce, enabled, undefined, { onRetryScheduled, onRetryFinished }); + expect(res.stopReason).toBe("error"); + expect(produce).toHaveBeenCalledTimes(4); // 1 initial + 3 retries + expect(onRetryScheduled).toHaveBeenCalledTimes(3); + expect(onRetryFinished).toHaveBeenCalledWith(false, 3, "terminated"); + }); + + it("reports capped retry delays", async () => { + // Regression for #8826. + let n = 0; + const policy: RetryPolicy = { enabled: true, maxRetries: 4, baseDelayMs: 10, maxAgentDelayMs: 15 }; + const produce = vi.fn(async () => { + n++; + return n < 5 + ? fauxAssistantMessage("", { stopReason: "error", errorMessage: "terminated" }) + : fauxAssistantMessage("recovered"); + }); + const onRetryScheduled = vi.fn(); + + await retryAssistantCall(produce, policy, undefined, { onRetryScheduled }); + + expect(onRetryScheduled.mock.calls.map((call) => call[2])).toEqual([10, 15, 15, 15]); + }); + + it("stops retrying once a call succeeds", async () => { + let n = 0; + const produce = vi.fn(async () => { + n++; + return n < 3 + ? fauxAssistantMessage("", { stopReason: "error", errorMessage: "terminated" }) + : fauxAssistantMessage("recovered"); + }); + const onRetryFinished = vi.fn(); + const res = await retryAssistantCall(produce, enabled, undefined, { onRetryFinished }); + expect(res.content).toEqual([{ type: "text", text: "recovered" }]); + expect(produce).toHaveBeenCalledTimes(3); + expect(onRetryFinished).toHaveBeenCalledWith(true, 2); + }); + + it("reports an aborted retried call as unsuccessful", async () => { + let n = 0; + const produce = vi.fn(async () => { + n++; + return n === 1 + ? fauxAssistantMessage("", { stopReason: "error", errorMessage: "terminated" }) + : fauxAssistantMessage("", { stopReason: "aborted" }); + }); + const onRetryFinished = vi.fn(); + const res = await retryAssistantCall(produce, enabled, undefined, { onRetryFinished }); + expect(res.stopReason).toBe("aborted"); + expect(produce).toHaveBeenCalledTimes(2); + expect(onRetryFinished).toHaveBeenCalledWith(false, 1); + }); + + it("does not retry when policy is disabled", async () => { + const produce = vi.fn(async () => fauxAssistantMessage("", { stopReason: "error", errorMessage: "terminated" })); + const onRetryScheduled = vi.fn(); + const onRetryFinished = vi.fn(); + const res = await retryAssistantCall(produce, disabled, undefined, { onRetryScheduled, onRetryFinished }); + expect(res.stopReason).toBe("error"); + expect(produce).toHaveBeenCalledTimes(1); + expect(onRetryScheduled).not.toHaveBeenCalled(); + expect(onRetryFinished).not.toHaveBeenCalled(); + }); + + it("emits onRetryAttemptStart after backoff before each retried call", async () => { + const events: string[] = []; + let n = 0; + const produce = vi.fn(async () => { + events.push(`produce:${n}`); + n++; + return n < 3 + ? fauxAssistantMessage("", { stopReason: "error", errorMessage: "terminated" }) + : fauxAssistantMessage("recovered"); + }); + const onRetryScheduled = vi.fn((attempt: number) => { + events.push(`retry:${attempt}`); + }); + const onRetryAttemptStart = vi.fn(() => { + events.push("attempt-start"); + }); + const res = await retryAssistantCall(produce, enabled, undefined, { onRetryScheduled, onRetryAttemptStart }); + expect(res.content).toEqual([{ type: "text", text: "recovered" }]); + expect(onRetryScheduled).toHaveBeenCalledTimes(2); + expect(onRetryAttemptStart).toHaveBeenCalledTimes(2); + expect(events).toEqual([ + "produce:0", + "retry:1", + "attempt-start", + "produce:1", + "retry:2", + "attempt-start", + "produce:2", + ]); + }); + + it("aborts backoff sleep via signal, returns an aborted message, and emits onRetryFinished(false)", async () => { + const controller = new AbortController(); + const produce = vi.fn(async () => fauxAssistantMessage("", { stopReason: "error", errorMessage: "terminated" })); + const policy: RetryPolicy = { enabled: true, maxRetries: 5, baseDelayMs: 10_000 }; + const onRetryFinished = vi.fn(); + const p = retryAssistantCall(produce, policy, controller.signal, { onRetryFinished }); + // Let one error call resolve and the first backoff sleep start, then abort. + await vi.waitFor(() => expect(produce).toHaveBeenCalled()); + controller.abort(); + const res = await p; + expect(res.stopReason).toBe("aborted"); + expect(res.errorMessage).toBeUndefined(); + expect(produce).toHaveBeenCalledTimes(1); + expect(onRetryFinished).toHaveBeenCalledWith(false, 1, "terminated"); + }); +}); diff --git a/packages/ai/test/scratch.ts b/packages/ai/test/scratch.ts new file mode 100644 index 0000000000000000000000000000000000000000..fec7b31aab4460f86f30dfe99d0e0fbd0da0447c --- /dev/null +++ b/packages/ai/test/scratch.ts @@ -0,0 +1,57 @@ +// Scratch script showing real-world use of the new Models API. +// Run from packages/ai: node test/scratch.ts +// Requires ANTHROPIC_API_KEY. + +import { createModels } from "../src/models.ts"; +import { anthropicProvider } from "../src/providers/anthropic.ts"; +import type { Context } from "../src/types.ts"; + +// --------------------------------------------------------------------------- +// 1. Build a Models runtime and register a built-in provider factory. +// (Apps wanting everything use `builtinModels()` from providers/all.) +// --------------------------------------------------------------------------- + +const models = createModels(); +models.setProvider(anthropicProvider()); + +// --------------------------------------------------------------------------- +// 2. Look up a model and check auth. +// --------------------------------------------------------------------------- + +const model = models.getModel("anthropic", "claude-haiku-4-5"); +if (!model) throw new Error("model not found"); + +const auth = await models.getAuth(model.provider); +console.log(`model: ${model.provider}/${model.id}`); +console.log(`auth: ${auth ? `configured via ${auth.source}` : "not configured"}\n`); +if (!auth) process.exit(1); + +const context: Context = { + systemPrompt: "You are terse.", + messages: [{ role: "user", content: "Say exactly: ok", timestamp: Date.now() }], +}; + +// --------------------------------------------------------------------------- +// 3. Simple completion (request-level auth resolution happens inside). +// --------------------------------------------------------------------------- + +const message = await models.completeSimple(model, context); +console.log(`completeSimple -> [${message.stopReason}]`, message.content); + +// --------------------------------------------------------------------------- +// 4. Streaming with deltas. +// --------------------------------------------------------------------------- + +context.messages.push(message, { + role: "user", + content: "Now count from 1 to 5, one number per line.", + timestamp: Date.now(), +}); + +process.stdout.write("streamSimple -> "); +const stream = models.streamSimple(model, context); +for await (const event of stream) { + if (event.type === "text_delta") process.stdout.write(event.delta.replaceAll("\n", " ")); +} +const final = await stream.result(); +console.log(`[${final.stopReason}] cost: $${final.usage.cost.total.toFixed(6)}`); diff --git a/packages/ai/test/supports-xhigh.test.ts b/packages/ai/test/supports-xhigh.test.ts new file mode 100644 index 0000000000000000000000000000000000000000..a1e43c40b4a9ca200c904592bfc3d04e46e9df59 --- /dev/null +++ b/packages/ai/test/supports-xhigh.test.ts @@ -0,0 +1,163 @@ +import { describe, expect, it } from "vitest"; +import { getModel, getSupportedThinkingLevels } from "../src/compat.ts"; + +describe("getSupportedThinkingLevels", () => { + it("includes max but not xhigh for Anthropic Opus 4.6 on anthropic-messages API", () => { + const model = getModel("anthropic", "claude-opus-4-6"); + expect(model).toBeDefined(); + expect(getSupportedThinkingLevels(model!)).toContain("max"); + expect(getSupportedThinkingLevels(model!)).not.toContain("xhigh"); + }); + + it("includes xhigh and max for Anthropic Opus 4.8 on anthropic-messages API", () => { + const model = getModel("anthropic", "claude-opus-4-8"); + expect(model).toBeDefined(); + expect(getSupportedThinkingLevels(model!)).toContain("xhigh"); + expect(getSupportedThinkingLevels(model!)).toContain("max"); + }); + + it("includes xhigh and max for Anthropic Opus 5 on anthropic-messages API", () => { + const model = getModel("anthropic", "claude-opus-5"); + expect(model).toBeDefined(); + expect(getSupportedThinkingLevels(model!)).toContain("xhigh"); + expect(getSupportedThinkingLevels(model!)).toContain("max"); + }); + + it("includes max but not xhigh for Anthropic Sonnet 4.6 on anthropic-messages API", () => { + const model = getModel("anthropic", "claude-sonnet-4-6"); + expect(model).toBeDefined(); + expect(getSupportedThinkingLevels(model!)).toContain("max"); + expect(getSupportedThinkingLevels(model!)).not.toContain("xhigh"); + }); + + it("includes xhigh and max for Anthropic Sonnet 5 on anthropic-messages API", () => { + const model = getModel("anthropic", "claude-sonnet-5"); + expect(model).toBeDefined(); + expect(getSupportedThinkingLevels(model!)).toContain("xhigh"); + expect(getSupportedThinkingLevels(model!)).toContain("max"); + }); + + it("includes xhigh and max but not off for Anthropic Claude Fable 5 on anthropic-messages API", () => { + const model = getModel("anthropic", "claude-fable-5"); + expect(model).toBeDefined(); + expect(getSupportedThinkingLevels(model!)).toContain("xhigh"); + expect(getSupportedThinkingLevels(model!)).toContain("max"); + expect(getSupportedThinkingLevels(model!)).not.toContain("off"); + }); + + it("does not include xhigh or max for Claude Sonnet 4.5", () => { + const model = getModel("anthropic", "claude-sonnet-4-5"); + expect(model).toBeDefined(); + expect(getSupportedThinkingLevels(model!)).not.toContain("xhigh"); + expect(getSupportedThinkingLevels(model!)).not.toContain("max"); + }); + + it.each(["gpt-5.5", "gpt-5.6-sol", "gpt-5.6-terra", "gpt-5.6-luna", "gpt-6-astra"] as const)( + "includes xhigh for openai-codex %s models", + (modelId) => { + const model = getModel("openai-codex", modelId); + expect(model).toBeDefined(); + expect(getSupportedThinkingLevels(model!)).toContain("xhigh"); + }, + ); + + it.each(["gpt-5.6-sol", "gpt-5.6-terra", "gpt-5.6-luna"] as const)( + "includes xhigh and max for OpenAI %s models", + (modelId) => { + const model = getModel("openai", modelId); + expect(model).toBeDefined(); + expect(getSupportedThinkingLevels(model!)).toEqual(["off", "low", "medium", "high", "xhigh", "max"]); + }, + ); + + it("includes only medium/high/xhigh for OpenAI GPT-5.5 Pro", () => { + const model = getModel("openai", "gpt-5.5-pro"); + expect(model).toBeDefined(); + expect(getSupportedThinkingLevels(model!)).toEqual(["medium", "high", "xhigh"]); + }); + + it("includes only medium/high/xhigh for OpenRouter GPT-5.5 Pro", () => { + const model = getModel("openrouter", "openai/gpt-5.5-pro"); + expect(model).toBeDefined(); + expect(getSupportedThinkingLevels(model!)).toEqual(["medium", "high", "xhigh"]); + }); + + it("includes low/high/max plus off for DeepSeek V4.1 Flash on the DeepSeek provider", () => { + const model = getModel("deepseek", "deepseek-flash"); + expect(model).toBeDefined(); + expect(getSupportedThinkingLevels(model!)).toEqual(["off", "low", "high", "max"]); + }); + + it("includes low/high/max plus off for DeepSeek V4 Flash on opencode-go", () => { + const model = getModel("opencode-go", "deepseek-v4-flash"); + expect(model).toBeDefined(); + expect(getSupportedThinkingLevels(model!)).toEqual(["off", "low", "high", "max"]); + }); + + it("includes only high plus off for OpenCode Go Kimi K2.6", () => { + const model = getModel("opencode-go", "kimi-k2.6"); + expect(model).toBeDefined(); + expect(getSupportedThinkingLevels(model!)).toEqual(["off", "high"]); + }); + + it("excludes thinking off for Moonshot Kimi K2.7 Code models", () => { + const cases = [getModel("moonshotai", "kimi-k2.7-code"), getModel("moonshotai-cn", "kimi-k2.7-code")]; + + for (const model of cases) { + expect(model).toBeDefined(); + expect(getSupportedThinkingLevels(model!)).toEqual(["minimal", "low", "medium", "high"]); + } + }); + + it.each(["moonshotai", "moonshotai-cn"] as const)("uses the verified effort options for %s Kimi K3", (provider) => { + const model = getModel(provider, "kimi-k3"); + expect(model).toBeDefined(); + expect(getSupportedThinkingLevels(model!)).toEqual(["low", "high", "max"]); + }); + + it("includes only low, high, max for Kimi Coding K3", () => { + const model = getModel("kimi-coding", "k3"); + expect(model).toBeDefined(); + expect(getSupportedThinkingLevels(model!)).toEqual(["low", "high", "max"]); + }); + + it("includes only high for OpenCode Grok Build", () => { + const model = getModel("opencode", "grok-build-0.1"); + expect(model).toBeDefined(); + expect(getSupportedThinkingLevels(model!)).toEqual(["high"]); + }); + + it("includes only high/xhigh plus off for DeepSeek V4 Flash on OpenRouter", () => { + const model = getModel("openrouter", "deepseek/deepseek-v4-flash"); + expect(model).toBeDefined(); + expect(getSupportedThinkingLevels(model!)).toEqual(["off", "high", "xhigh"]); + }); + + it("includes max but not xhigh for OpenRouter Opus 4.6 (openai-completions API)", () => { + const model = getModel("openrouter", "anthropic/claude-opus-4.6"); + expect(model).toBeDefined(); + expect(getSupportedThinkingLevels(model!)).toContain("max"); + expect(getSupportedThinkingLevels(model!)).not.toContain("xhigh"); + }); + + it("includes xhigh and max for Bedrock Claude Opus 5", () => { + const model = getModel("amazon-bedrock", "global.anthropic.claude-opus-5"); + expect(model).toBeDefined(); + expect(getSupportedThinkingLevels(model!)).toContain("xhigh"); + expect(getSupportedThinkingLevels(model!)).toContain("max"); + }); + + it("includes xhigh but not off or max for xAI Grok 4.6", () => { + const model = getModel("xai", "grok-4.6"); + expect(model).toBeDefined(); + expect(getSupportedThinkingLevels(model!)).toEqual(["low", "medium", "high", "xhigh"]); + }); + + it("includes xhigh and max but not off for Bedrock Claude Fable 5", () => { + const model = getModel("amazon-bedrock", "global.anthropic.claude-fable-5"); + expect(model).toBeDefined(); + expect(getSupportedThinkingLevels(model!)).toContain("xhigh"); + expect(getSupportedThinkingLevels(model!)).toContain("max"); + expect(getSupportedThinkingLevels(model!)).not.toContain("off"); + }); +}); diff --git a/packages/ai/test/text.test.ts b/packages/ai/test/text.test.ts new file mode 100644 index 0000000000000000000000000000000000000000..43dc21c84636c5362990684c643d48823236595b --- /dev/null +++ b/packages/ai/test/text.test.ts @@ -0,0 +1,33 @@ +import { describe, expect, it } from "vitest"; +import { type AssistantMessage, contentText, type ToolResultMessage } from "../src/index.ts"; + +const content: AssistantMessage["content"] = [ + { type: "thinking", thinking: "reasoning" }, + { type: "text", text: "first" }, + { type: "toolCall", id: "1", name: "read", arguments: {} }, + { type: "text", text: "second" }, +]; + +describe("contentText", () => { + it("extracts assistant text blocks", () => { + expect(contentText(content)).toBe("first\nsecond"); + }); + + it("supports custom separators", () => { + expect(contentText(content, "")).toBe("firstsecond"); + }); + + it("passes string content through", () => { + expect(contentText("hello")).toBe("hello"); + }); + + it("extracts text from tool-result content", () => { + const toolResultContent: ToolResultMessage["content"] = [ + { type: "text", text: "first" }, + { type: "image", data: "...", mimeType: "image/png" }, + { type: "text", text: "second" }, + ]; + + expect(contentText(toolResultContent, "")).toBe("firstsecond"); + }); +}); diff --git a/packages/ai/test/tool-call-id-normalization.test.ts b/packages/ai/test/tool-call-id-normalization.test.ts new file mode 100644 index 0000000000000000000000000000000000000000..031a33085b2558cc7c9c899bf49ce78ff9f7f245 --- /dev/null +++ b/packages/ai/test/tool-call-id-normalization.test.ts @@ -0,0 +1,290 @@ +/** + * Tool Call ID Normalization Tests + * + * Tests that tool call IDs from OpenAI Responses API (github-copilot, openai-codex, opencode) + * are properly normalized when sent to other providers. + * + * OpenAI Responses API generates IDs in format: {call_id}|{id} + * where {id} can be 400+ chars with special characters (+, /, =). + * + * Regression test for: https://github.com/earendil-works/pi/issues/1022 + */ + +import { Type } from "typebox"; +import { describe, expect, it } from "vitest"; +import { completeSimple, getEnvApiKey, getModel } from "../src/compat.ts"; +import type { AssistantMessage, Message, Tool, ToolResultMessage } from "../src/types.ts"; +import { resolveApiKey } from "./oauth.ts"; + +// Resolve API keys +const copilotToken = await resolveApiKey("github-copilot"); +const openrouterKey = getEnvApiKey("openrouter"); +const codexToken = await resolveApiKey("openai-codex"); + +// Simple echo tool for testing +const echoToolSchema = Type.Object({ + message: Type.String({ description: "Message to echo back" }), +}); + +const echoTool: Tool = { + name: "echo", + description: "Echoes the message back", + parameters: echoToolSchema, +}; + +/** + * Test 1: Live cross-provider handoff + * + * 1. Use github-copilot gpt-5.5 to generate a tool call + * 2. Switch to openrouter openai/gpt-5.5 and complete + * 3. Switch to openai-codex gpt-5.5 and complete + * + * Both should succeed without "call_id too long" errors. + */ +describe("Tool Call ID Normalization - Live Handoff", () => { + it.skipIf(!copilotToken || !openrouterKey)( + "github-copilot -> openrouter should normalize pipe-separated IDs", + async () => { + const copilotModel = getModel("github-copilot", "gpt-5.5"); + const openrouterModel = getModel("openrouter", "openai/gpt-5.5"); + + // Step 1: Generate tool call with github-copilot + const userMessage: Message = { + role: "user", + content: "Use the echo tool to echo 'hello world'", + timestamp: Date.now(), + }; + + const assistantResponse = await completeSimple( + copilotModel, + { + systemPrompt: "You are a helpful assistant. Use the echo tool when asked.", + messages: [userMessage], + tools: [echoTool], + }, + { apiKey: copilotToken }, + ); + + expect(assistantResponse.stopReason, `Copilot error: ${assistantResponse.errorMessage}`).toBe("toolUse"); + + const toolCall = assistantResponse.content.find((c) => c.type === "toolCall"); + expect(toolCall).toBeDefined(); + expect(toolCall!.type).toBe("toolCall"); + + // Verify it's a pipe-separated ID (OpenAI Responses format) + if (toolCall?.type === "toolCall") { + expect(toolCall.id).toContain("|"); + console.log(`Tool call ID from github-copilot: ${toolCall.id.slice(0, 80)}...`); + } + + // Create tool result + const toolResult: ToolResultMessage = { + role: "toolResult", + toolCallId: (toolCall as any).id, + toolName: "echo", + content: [{ type: "text", text: "hello world" }], + isError: false, + timestamp: Date.now(), + }; + + // Step 2: Complete with openrouter (uses openai-completions API) + const openrouterResponse = await completeSimple( + openrouterModel, + { + systemPrompt: "You are a helpful assistant.", + messages: [ + userMessage, + assistantResponse, + toolResult, + { role: "user", content: "Say hi", timestamp: Date.now() }, + ], + tools: [echoTool], + }, + { apiKey: openrouterKey }, + ); + + // Should NOT fail with "call_id too long" error + expect(openrouterResponse.stopReason, `OpenRouter error: ${openrouterResponse.errorMessage}`).not.toBe( + "error", + ); + expect(openrouterResponse.errorMessage).toBeUndefined(); + }, + 60000, + ); + + it.skipIf(!copilotToken || !codexToken)( + "github-copilot -> openai-codex should normalize pipe-separated IDs", + async () => { + const copilotModel = getModel("github-copilot", "gpt-5.5"); + const codexModel = getModel("openai-codex", "gpt-5.5"); + + // Step 1: Generate tool call with github-copilot + const userMessage: Message = { + role: "user", + content: "Use the echo tool to echo 'test message'", + timestamp: Date.now(), + }; + + const assistantResponse = await completeSimple( + copilotModel, + { + systemPrompt: "You are a helpful assistant. Use the echo tool when asked.", + messages: [userMessage], + tools: [echoTool], + }, + { apiKey: copilotToken }, + ); + + expect(assistantResponse.stopReason, `Copilot error: ${assistantResponse.errorMessage}`).toBe("toolUse"); + + const toolCall = assistantResponse.content.find((c) => c.type === "toolCall"); + expect(toolCall).toBeDefined(); + + // Create tool result + const toolResult: ToolResultMessage = { + role: "toolResult", + toolCallId: (toolCall as any).id, + toolName: "echo", + content: [{ type: "text", text: "test message" }], + isError: false, + timestamp: Date.now(), + }; + + // Step 2: Complete with openai-codex (uses openai-codex-responses API) + const codexResponse = await completeSimple( + codexModel, + { + systemPrompt: "You are a helpful assistant.", + messages: [ + userMessage, + assistantResponse, + toolResult, + { role: "user", content: "Say hi", timestamp: Date.now() }, + ], + tools: [echoTool], + }, + { apiKey: codexToken }, + ); + + // Should NOT fail with ID validation error + expect(codexResponse.stopReason, `Codex error: ${codexResponse.errorMessage}`).not.toBe("error"); + expect(codexResponse.errorMessage).toBeUndefined(); + }, + 60000, + ); +}); + +/** + * Test 2: Prefilled context with exact failing IDs from issue #1022 + * + * Uses the exact tool call ID format that caused the error: + * "call_xxx|very_long_base64_with_special_chars+/=" + */ +describe("Tool Call ID Normalization - Prefilled Context", () => { + // Exact tool call ID from issue #1022 JSONL + const FAILING_TOOL_CALL_ID = + "call_pAYbIr76hXIjncD9UE4eGfnS|t5nnb2qYMFWGSsr13fhCd1CaCu3t3qONEPuOudu4HSVEtA8YJSL6FAZUxvoOoD792VIJWl91g87EdqsCWp9krVsdBysQoDaf9lMCLb8BS4EYi4gQd5kBQBYLlgD71PYwvf+TbMD9J9/5OMD42oxSRj8H+vRf78/l2Xla33LWz4nOgsddBlbvabICRs8GHt5C9PK5keFtzyi3lsyVKNlfduK3iphsZqs4MLv4zyGJnvZo/+QzShyk5xnMSQX/f98+aEoNflEApCdEOXipipgeiNWnpFSHbcwmMkZoJhURNu+JEz3xCh1mrXeYoN5o+trLL3IXJacSsLYXDrYTipZZbJFRPAucgbnjYBC+/ZzJOfkwCs+Gkw7EoZR7ZQgJ8ma+9586n4tT4cI8DEhBSZsWMjrCt8dxKg=="; + + // Build prefilled context with the failing ID + function buildPrefilledMessages(): Message[] { + const userMessage: Message = { + role: "user", + content: "Use the echo tool to echo 'hello'", + timestamp: Date.now() - 2000, + }; + + const assistantMessage: AssistantMessage = { + role: "assistant", + content: [ + { + type: "toolCall", + id: FAILING_TOOL_CALL_ID, + name: "echo", + arguments: { message: "hello" }, + }, + ], + api: "openai-responses", + provider: "github-copilot", + model: "gpt-5.2-codex", + usage: { + input: 100, + output: 50, + cacheRead: 0, + cacheWrite: 0, + totalTokens: 150, + cost: { input: 0, output: 0, cacheRead: 0, cacheWrite: 0, total: 0 }, + }, + stopReason: "toolUse", + timestamp: Date.now() - 1500, + }; + + const toolResult: ToolResultMessage = { + role: "toolResult", + toolCallId: FAILING_TOOL_CALL_ID, + toolName: "echo", + content: [{ type: "text", text: "hello" }], + isError: false, + timestamp: Date.now() - 1000, + }; + + const followUpUser: Message = { + role: "user", + content: "Say hi", + timestamp: Date.now(), + }; + + return [userMessage, assistantMessage, toolResult, followUpUser]; + } + + it.skipIf(!openrouterKey)( + "openrouter should handle prefilled context with long pipe-separated IDs", + async () => { + const model = getModel("openrouter", "openai/gpt-5.5"); + const messages = buildPrefilledMessages(); + + const response = await completeSimple( + model, + { + systemPrompt: "You are a helpful assistant.", + messages, + tools: [echoTool], + }, + { apiKey: openrouterKey }, + ); + + // Should NOT fail with "call_id too long" error + expect(response.stopReason, `OpenRouter error: ${response.errorMessage}`).not.toBe("error"); + if (response.errorMessage) { + expect(response.errorMessage).not.toContain("call_id"); + expect(response.errorMessage).not.toContain("too long"); + } + }, + 30000, + ); + + it.skipIf(!codexToken)( + "openai-codex should handle prefilled context with long pipe-separated IDs", + async () => { + const model = getModel("openai-codex", "gpt-5.5"); + const messages = buildPrefilledMessages(); + + const response = await completeSimple( + model, + { + systemPrompt: "You are a helpful assistant.", + messages, + tools: [echoTool], + }, + { apiKey: codexToken }, + ); + + // Should NOT fail with ID validation error + expect(response.stopReason, `Codex error: ${response.errorMessage}`).not.toBe("error"); + if (response.errorMessage) { + expect(response.errorMessage).not.toContain("id"); + expect(response.errorMessage).not.toContain("additional characters"); + } + }, + 30000, + ); +}); diff --git a/packages/ai/test/tool-call-without-result.test.ts b/packages/ai/test/tool-call-without-result.test.ts new file mode 100644 index 0000000000000000000000000000000000000000..57945ccadb9a8993ea1ebe73fbed087dc842999b --- /dev/null +++ b/packages/ai/test/tool-call-without-result.test.ts @@ -0,0 +1,359 @@ +import { Type } from "typebox"; +import { describe, expect, it } from "vitest"; +import { complete, getModel } from "../src/compat.ts"; +import type { Api, Context, Model, StreamOptions, Tool } from "../src/types.ts"; + +type StreamOptionsWithExtras = StreamOptions & Record; + +import { hasAzureOpenAICredentials, resolveAzureDeploymentName } from "./azure-utils.ts"; +import { hasBedrockCredentials } from "./bedrock-utils.ts"; +import { hasCloudflareAiGatewayCredentials, hasCloudflareWorkersAICredentials } from "./cloudflare-utils.ts"; +import { resolveApiKey } from "./oauth.ts"; + +// Resolve OAuth tokens at module level (async, runs before tests) +const oauthTokens = await Promise.all([ + resolveApiKey("anthropic"), + resolveApiKey("github-copilot"), + resolveApiKey("openai-codex"), +]); +const [anthropicOAuthToken, githubCopilotToken, openaiCodexToken] = oauthTokens; + +// Simple calculate tool +const calculateSchema = Type.Object({ + expression: Type.String({ description: "The mathematical expression to evaluate" }), +}); + +const calculateTool: Tool = { + name: "calculate", + description: "Evaluate mathematical expressions", + parameters: calculateSchema, +}; + +async function testToolCallWithoutResult(model: Model, options: StreamOptionsWithExtras = {}) { + // Step 1: Create context with the calculate tool + const context: Context = { + systemPrompt: "You are a helpful assistant. Use the calculate tool when asked to perform calculations.", + messages: [], + tools: [calculateTool], + }; + + // Step 2: Ask the LLM to make a tool call + context.messages.push({ + role: "user", + content: "Please calculate 25 * 18 using the calculate tool.", + timestamp: Date.now(), + }); + + // Step 3: Get the assistant's response (should contain a tool call) + const firstResponse = await complete(model, context, options); + context.messages.push(firstResponse); + + console.log("First response:", JSON.stringify(firstResponse, null, 2)); + + // Verify the response contains a tool call + const hasToolCall = firstResponse.content.some((block) => block.type === "toolCall"); + expect(hasToolCall).toBe(true); + + if (!hasToolCall) { + throw new Error("Expected assistant to make a tool call, but none was found"); + } + + // Step 4: Send a user message WITHOUT providing tool result + // This simulates the scenario where a tool call was aborted/cancelled + context.messages.push({ + role: "user", + content: "Never mind, just tell me what is 2+2?", + timestamp: Date.now(), + }); + + // Step 5: The fix should filter out the orphaned tool call, and the request should succeed + const secondResponse = await complete(model, context, options); + console.log("Second response:", JSON.stringify(secondResponse, null, 2)); + + // The request should succeed (not error) - that's the main thing we're testing + expect(secondResponse.stopReason).not.toBe("error"); + + // Should have some content in the response + expect(secondResponse.content.length).toBeGreaterThan(0); + + // The LLM may choose to answer directly or make a new tool call - either is fine + // The important thing is it didn't fail with the orphaned tool call error + const textContent = secondResponse.content + .filter((block) => block.type === "text") + .map((block) => (block.type === "text" ? block.text : "")) + .join(" "); + const toolCalls = secondResponse.content.filter((block) => block.type === "toolCall").length; + expect(toolCalls || textContent.length).toBeGreaterThan(0); + console.log("Answer:", textContent); + + // Verify the stop reason is either "stop" or "toolUse" (new tool call) + expect(["stop", "toolUse"]).toContain(secondResponse.stopReason); +} + +describe("Tool Call Without Result Tests", () => { + // ========================================================================= + // API Key-based providers + // ========================================================================= + + describe.skipIf(!process.env.GEMINI_API_KEY)("Google Provider", () => { + const model = getModel("google", "gemini-2.5-flash"); + + it("should filter out tool calls without corresponding tool results", { retry: 3, timeout: 30000 }, async () => { + await testToolCallWithoutResult(model); + }); + }); + + describe.skipIf(!process.env.OPENAI_API_KEY)("OpenAI Completions Provider", () => { + const { compat: _compat, ...baseModel } = getModel("openai", "gpt-4o-mini")!; + void _compat; + const model: Model<"openai-completions"> = { + ...baseModel, + api: "openai-completions", + }; + + it("should filter out tool calls without corresponding tool results", { retry: 3, timeout: 30000 }, async () => { + await testToolCallWithoutResult(model); + }); + }); + + describe.skipIf(!process.env.OPENAI_API_KEY)("OpenAI Responses Provider", () => { + const model = getModel("openai", "gpt-5-mini"); + + it("should filter out tool calls without corresponding tool results", { retry: 3, timeout: 30000 }, async () => { + await testToolCallWithoutResult(model); + }); + }); + + describe.skipIf(!hasAzureOpenAICredentials())("Azure OpenAI Responses Provider", () => { + const model = getModel("azure-openai-responses", "gpt-4o-mini"); + const azureDeploymentName = resolveAzureDeploymentName(model.id); + const azureOptions = azureDeploymentName ? { azureDeploymentName } : {}; + + it("should filter out tool calls without corresponding tool results", { retry: 3, timeout: 30000 }, async () => { + await testToolCallWithoutResult(model, azureOptions); + }); + }); + + describe.skipIf(!process.env.ANTHROPIC_API_KEY)("Anthropic Provider", () => { + const model = getModel("anthropic", "claude-haiku-4-5"); + + it("should filter out tool calls without corresponding tool results", { retry: 3, timeout: 30000 }, async () => { + await testToolCallWithoutResult(model); + }); + }); + + describe.skipIf(!process.env.XAI_API_KEY)("xAI Provider", () => { + const model = getModel("xai", "grok-4.3"); + + it("should filter out tool calls without corresponding tool results", { retry: 3, timeout: 30000 }, async () => { + await testToolCallWithoutResult(model); + }); + }); + + describe.skipIf(!process.env.GROQ_API_KEY)("Groq Provider", () => { + const model = getModel("groq", "openai/gpt-oss-20b"); + + it("should filter out tool calls without corresponding tool results", { retry: 3, timeout: 30000 }, async () => { + await testToolCallWithoutResult(model); + }); + }); + + describe.skipIf(!process.env.CEREBRAS_API_KEY)("Cerebras Provider", () => { + const model = getModel("cerebras", "gpt-oss-120b"); + + it("should filter out tool calls without corresponding tool results", { retry: 3, timeout: 30000 }, async () => { + await testToolCallWithoutResult(model); + }); + }); + + describe.skipIf(!hasCloudflareWorkersAICredentials())("Cloudflare Workers AI Provider", () => { + const model = getModel("cloudflare-workers-ai", "@cf/moonshotai/kimi-k2.6"); + + it("should filter out tool calls without corresponding tool results", { retry: 3, timeout: 30000 }, async () => { + await testToolCallWithoutResult(model); + }); + }); + + describe.skipIf(!hasCloudflareAiGatewayCredentials())("Cloudflare AI Gateway Provider", () => { + const model = getModel("cloudflare-ai-gateway", "workers-ai/@cf/moonshotai/kimi-k2.6"); + + it("should filter out tool calls without corresponding tool results", { retry: 3, timeout: 30000 }, async () => { + await testToolCallWithoutResult(model); + }); + }); + + describe.skipIf(!process.env.HF_TOKEN)("Hugging Face Provider", () => { + const model = getModel("huggingface", "moonshotai/Kimi-K2.5"); + + it("should filter out tool calls without corresponding tool results", { retry: 3, timeout: 30000 }, async () => { + await testToolCallWithoutResult(model); + }); + }); + + describe.skipIf(!process.env.TOGETHER_API_KEY)("Together AI Provider", () => { + const model = getModel("together", "moonshotai/Kimi-K2.6"); + + it("should filter out tool calls without corresponding tool results", { retry: 3, timeout: 30000 }, async () => { + await testToolCallWithoutResult(model, { reasoningEffort: "high" }); + }); + }); + + describe.skipIf(!process.env.BASETEN_API_KEY)("Baseten Provider", () => { + const model = getModel("baseten", "zai-org/GLM-5.2"); + + it("should filter out tool calls without corresponding tool results", { retry: 3, timeout: 30000 }, async () => { + await testToolCallWithoutResult(model, { reasoningEffort: "high" }); + }); + }); + + describe.skipIf(!process.env.ZAI_API_KEY)("zAI Provider", () => { + const model = getModel("zai", "glm-5.2"); + + it("should filter out tool calls without corresponding tool results", { retry: 3, timeout: 30000 }, async () => { + await testToolCallWithoutResult(model); + }); + }); + + describe.skipIf(!process.env.MISTRAL_API_KEY)("Mistral Provider", () => { + const model = getModel("mistral", "devstral-medium-latest"); + + it("should filter out tool calls without corresponding tool results", { retry: 3, timeout: 30000 }, async () => { + await testToolCallWithoutResult(model); + }); + }); + + describe.skipIf(!process.env.MINIMAX_API_KEY)("MiniMax Provider", () => { + const model = getModel("minimax", "MiniMax-M2.7"); + + it("should filter out tool calls without corresponding tool results", { retry: 3, timeout: 30000 }, async () => { + await testToolCallWithoutResult(model); + }); + }); + + describe.skipIf(!process.env.XIAOMI_API_KEY)("Xiaomi MiMo (API billing) Provider", () => { + const model = getModel("xiaomi", "mimo-v2.5-pro"); + + it("should filter out tool calls without corresponding tool results", { retry: 3, timeout: 30000 }, async () => { + await testToolCallWithoutResult(model); + }); + }); + + describe.skipIf(!process.env.XIAOMI_TOKEN_PLAN_CN_API_KEY)("Xiaomi MiMo Token Plan (CN) Provider", () => { + const model = getModel("xiaomi-token-plan-cn", "mimo-v2.5-pro"); + + it("should filter out tool calls without corresponding tool results", { retry: 3, timeout: 30000 }, async () => { + await testToolCallWithoutResult(model); + }); + }); + + describe.skipIf(!process.env.XIAOMI_TOKEN_PLAN_AMS_API_KEY)("Xiaomi MiMo Token Plan (AMS) Provider", () => { + const model = getModel("xiaomi-token-plan-ams", "mimo-v2.5-pro"); + + it("should filter out tool calls without corresponding tool results", { retry: 3, timeout: 30000 }, async () => { + await testToolCallWithoutResult(model); + }); + }); + + describe.skipIf(!process.env.XIAOMI_TOKEN_PLAN_SGP_API_KEY)("Xiaomi MiMo Token Plan (SGP) Provider", () => { + const model = getModel("xiaomi-token-plan-sgp", "mimo-v2.5-pro"); + + it("should filter out tool calls without corresponding tool results", { retry: 3, timeout: 30000 }, async () => { + await testToolCallWithoutResult(model); + }); + }); + + describe.skipIf(!process.env.QWEN_TOKEN_PLAN_API_KEY)("Qwen Token Plan Provider", () => { + const model = getModel("qwen-token-plan", "qwen3.7-max"); + + it("should filter out tool calls without corresponding tool results", { retry: 3, timeout: 30000 }, async () => { + await testToolCallWithoutResult(model); + }); + }); + + describe.skipIf(!process.env.QWEN_TOKEN_PLAN_API_KEY)("Qwen Token Plan Individual Provider", () => { + const model = getModel("qwen-token-plan-individual", "qwen3.8-max"); + + it("should filter out tool calls without corresponding tool results", { retry: 3, timeout: 30000 }, async () => { + await testToolCallWithoutResult(model); + }); + }); + + describe.skipIf(!process.env.QWEN_TOKEN_PLAN_CN_API_KEY)("Qwen Token Plan (CN) Provider", () => { + const model = getModel("qwen-token-plan-cn", "qwen3.7-max"); + + it("should filter out tool calls without corresponding tool results", { retry: 3, timeout: 30000 }, async () => { + await testToolCallWithoutResult(model); + }); + }); + + describe.skipIf(!process.env.KIMI_API_KEY)("Kimi For Coding Provider", () => { + const model = getModel("kimi-coding", "kimi-for-coding"); + + it("should filter out tool calls without corresponding tool results", { retry: 3, timeout: 30000 }, async () => { + await testToolCallWithoutResult(model); + }); + }); + + describe.skipIf(!process.env.AI_GATEWAY_API_KEY)("Vercel AI Gateway Provider", () => { + const model = getModel("vercel-ai-gateway", "google/gemini-2.5-flash"); + + it("should filter out tool calls without corresponding tool results", { retry: 3, timeout: 30000 }, async () => { + await testToolCallWithoutResult(model); + }); + }); + + describe.skipIf(!hasBedrockCredentials())("Amazon Bedrock Provider", () => { + const model = getModel("amazon-bedrock", "global.anthropic.claude-sonnet-4-5-20250929-v1:0"); + + it("should filter out tool calls without corresponding tool results", { retry: 3, timeout: 30000 }, async () => { + await testToolCallWithoutResult(model); + }); + }); + + // ========================================================================= + // OAuth-based providers (credentials from ~/.pi/agent/oauth.json) + // ========================================================================= + + describe("Anthropic OAuth Provider", () => { + const model = getModel("anthropic", "claude-haiku-4-5"); + + it.skipIf(!anthropicOAuthToken)( + "should filter out tool calls without corresponding tool results", + { retry: 3, timeout: 30000 }, + async () => { + await testToolCallWithoutResult(model, { apiKey: anthropicOAuthToken }); + }, + ); + }); + + describe("GitHub Copilot Provider", () => { + it.skipIf(!githubCopilotToken)( + "claude-haiku-4.5 - should filter out tool calls without corresponding tool results", + { retry: 3, timeout: 30000 }, + async () => { + const model = getModel("github-copilot", "claude-haiku-4.5"); + await testToolCallWithoutResult(model, { apiKey: githubCopilotToken }); + }, + ); + + it.skipIf(!githubCopilotToken)( + "claude-sonnet-4 - should filter out tool calls without corresponding tool results", + { retry: 3, timeout: 30000 }, + async () => { + const model = getModel("github-copilot", "claude-sonnet-4.6"); + await testToolCallWithoutResult(model, { apiKey: githubCopilotToken }); + }, + ); + }); + + describe("OpenAI Codex Provider", () => { + it.skipIf(!openaiCodexToken)( + "gpt-5.5 - should filter out tool calls without corresponding tool results", + { retry: 3, timeout: 30000 }, + async () => { + const model = getModel("openai-codex", "gpt-5.5"); + await testToolCallWithoutResult(model, { apiKey: openaiCodexToken }); + }, + ); + }); +}); diff --git a/packages/ai/test/total-tokens.test.ts b/packages/ai/test/total-tokens.test.ts new file mode 100644 index 0000000000000000000000000000000000000000..fabff7f69622300b743e953994a62de30cc5ab52 --- /dev/null +++ b/packages/ai/test/total-tokens.test.ts @@ -0,0 +1,882 @@ +/** + * Test totalTokens field across all providers. + * + * totalTokens represents the total number of tokens processed by the LLM, + * including input (with cache) and output (with thinking). This is the + * base for calculating context size for the next request. + * + * - OpenAI Completions: Uses native total_tokens field + * - OpenAI Responses: Uses native total_tokens field + * - Google: Uses native totalTokenCount field + * - Anthropic: Computed as input + output + cacheRead + cacheWrite + * - Other OpenAI-compatible providers: Uses native total_tokens field + */ + +import { describe, expect, it } from "vitest"; +import { complete, getModel } from "../src/compat.ts"; +import type { Api, Context, Model, StreamOptions, Usage } from "../src/types.ts"; + +type StreamOptionsWithExtras = StreamOptions & Record; + +import { hasAzureOpenAICredentials, resolveAzureDeploymentName } from "./azure-utils.ts"; +import { hasBedrockCredentials } from "./bedrock-utils.ts"; +import { hasCloudflareAiGatewayCredentials, hasCloudflareWorkersAICredentials } from "./cloudflare-utils.ts"; +import { resolveApiKey } from "./oauth.ts"; + +// Resolve OAuth tokens at module level (async, runs before tests) +const oauthTokens = await Promise.all([ + resolveApiKey("anthropic"), + resolveApiKey("github-copilot"), + resolveApiKey("openai-codex"), +]); +const [anthropicOAuthToken, githubCopilotToken, openaiCodexToken] = oauthTokens; + +// Generate a long system prompt to trigger caching (>2k bytes for most providers) +const LONG_SYSTEM_PROMPT = `You are a helpful assistant. Be concise in your responses. + +Here is some additional context that makes this system prompt long enough to trigger caching: + +${Array(50) + .fill( + "Lorem ipsum dolor sit amet, consectetur adipiscing elit. Sed do eiusmod tempor incididunt ut labore et dolore magna aliqua. Ut enim ad minim veniam, quis nostrud exercitation ullamco laboris.", + ) + .join("\n\n")} + +Remember: Always be helpful and concise.`; + +async function testTotalTokensWithCache( + llm: Model, + options: StreamOptionsWithExtras = {}, +): Promise<{ first: Usage; second: Usage }> { + // First request - no cache + const context1: Context = { + systemPrompt: LONG_SYSTEM_PROMPT, + messages: [ + { + role: "user", + content: "What is 2 + 2? Reply with just the number.", + timestamp: Date.now(), + }, + ], + }; + + const response1 = await complete(llm, context1, options); + expect(response1.stopReason).toBe("stop"); + + // Second request - should trigger cache read (same system prompt, add conversation) + const context2: Context = { + systemPrompt: LONG_SYSTEM_PROMPT, + messages: [ + ...context1.messages, + response1, // Include previous assistant response + { + role: "user", + content: "What is 3 + 3? Reply with just the number.", + timestamp: Date.now(), + }, + ], + }; + + const response2 = await complete(llm, context2, options); + expect(response2.stopReason).toBe("stop"); + + return { first: response1.usage, second: response2.usage }; +} + +function logUsage(label: string, usage: Usage) { + const computed = usage.input + usage.output + usage.cacheRead + usage.cacheWrite; + console.log(` ${label}:`); + console.log( + ` input: ${usage.input}, output: ${usage.output}, cacheRead: ${usage.cacheRead}, cacheWrite: ${usage.cacheWrite}`, + ); + console.log(` totalTokens: ${usage.totalTokens}, computed: ${computed}`); +} + +function assertTotalTokensEqualsComponents(usage: Usage) { + const computed = usage.input + usage.output + usage.cacheRead + usage.cacheWrite; + expect(usage.totalTokens).toBe(computed); +} + +describe("totalTokens field", () => { + // ========================================================================= + // Anthropic + // ========================================================================= + + describe.skipIf(!process.env.ANTHROPIC_API_KEY)("Anthropic (API Key)", () => { + it( + "claude-sonnet-4-5 - should return totalTokens equal to sum of components", + { retry: 3, timeout: 60000 }, + async () => { + const llm = getModel("anthropic", "claude-sonnet-4-5"); + + console.log(`\nAnthropic / ${llm.id}:`); + const { first, second } = await testTotalTokensWithCache(llm, { apiKey: process.env.ANTHROPIC_API_KEY }); + + logUsage("First request", first); + logUsage("Second request", second); + + assertTotalTokensEqualsComponents(first); + assertTotalTokensEqualsComponents(second); + + // Anthropic should have cache activity + const hasCache = second.cacheRead > 0 || second.cacheWrite > 0 || first.cacheWrite > 0; + expect(hasCache).toBe(true); + }, + ); + }); + + describe("Anthropic (OAuth)", () => { + it.skipIf(!anthropicOAuthToken)( + "claude-sonnet-4 - should return totalTokens equal to sum of components", + { retry: 3, timeout: 60000 }, + async () => { + const llm = getModel("anthropic", "claude-sonnet-4-6"); + + console.log(`\nAnthropic OAuth / ${llm.id}:`); + const { first, second } = await testTotalTokensWithCache(llm, { apiKey: anthropicOAuthToken }); + + logUsage("First request", first); + logUsage("Second request", second); + + assertTotalTokensEqualsComponents(first); + assertTotalTokensEqualsComponents(second); + + // Anthropic should have cache activity + const hasCache = second.cacheRead > 0 || second.cacheWrite > 0 || first.cacheWrite > 0; + expect(hasCache).toBe(true); + }, + ); + }); + + // ========================================================================= + // OpenAI + // ========================================================================= + + describe.skipIf(!process.env.OPENAI_API_KEY)("OpenAI Completions", () => { + it( + "gpt-4o-mini - should return totalTokens equal to sum of components", + { retry: 3, timeout: 60000 }, + async () => { + const { compat: _compat, ...baseModel } = getModel("openai", "gpt-4o-mini")!; + void _compat; + const llm: Model<"openai-completions"> = { + ...baseModel, + api: "openai-completions", + }; + + console.log(`\nOpenAI Completions / ${llm.id}:`); + const { first, second } = await testTotalTokensWithCache(llm); + + logUsage("First request", first); + logUsage("Second request", second); + + assertTotalTokensEqualsComponents(first); + assertTotalTokensEqualsComponents(second); + }, + ); + }); + + describe.skipIf(!process.env.OPENAI_API_KEY)("OpenAI Responses", () => { + it( + "claude-haiku-4.5 - should return totalTokens equal to sum of components", + { retry: 3, timeout: 60000 }, + async () => { + const llm = getModel("openai", "gpt-4o"); + + console.log(`\nOpenAI Responses / ${llm.id}:`); + const { first, second } = await testTotalTokensWithCache(llm); + + logUsage("First request", first); + logUsage("Second request", second); + + assertTotalTokensEqualsComponents(first); + assertTotalTokensEqualsComponents(second); + }, + ); + }); + + describe.skipIf(!hasAzureOpenAICredentials())("Azure OpenAI Responses", () => { + it( + "gpt-4o-mini - should return totalTokens equal to sum of components", + { retry: 3, timeout: 60000 }, + async () => { + const llm = getModel("azure-openai-responses", "gpt-4o-mini"); + const azureDeploymentName = resolveAzureDeploymentName(llm.id); + const azureOptions = azureDeploymentName ? { azureDeploymentName } : {}; + + console.log(`\nAzure OpenAI Responses / ${llm.id}:`); + const { first, second } = await testTotalTokensWithCache(llm, azureOptions); + + logUsage("First request", first); + logUsage("Second request", second); + + assertTotalTokensEqualsComponents(first); + assertTotalTokensEqualsComponents(second); + }, + ); + }); + + // ========================================================================= + // Google + // ========================================================================= + + describe.skipIf(!process.env.GEMINI_API_KEY)("Google", () => { + it( + "gemini-2.5-flash - should return totalTokens equal to sum of components", + { retry: 3, timeout: 60000 }, + async () => { + const llm = getModel("google", "gemini-2.5-flash"); + + console.log(`\nGoogle / ${llm.id}:`); + const { first, second } = await testTotalTokensWithCache(llm); + + logUsage("First request", first); + logUsage("Second request", second); + + assertTotalTokensEqualsComponents(first); + assertTotalTokensEqualsComponents(second); + }, + ); + }); + + // ========================================================================= + // xAI + // ========================================================================= + + describe.skipIf(!process.env.XAI_API_KEY)("xAI", () => { + it("grok-4.3 - should return totalTokens equal to sum of components", { retry: 3, timeout: 60000 }, async () => { + const llm = getModel("xai", "grok-4.3"); + + console.log(`\nxAI / ${llm.id}:`); + const { first, second } = await testTotalTokensWithCache(llm, { apiKey: process.env.XAI_API_KEY }); + + logUsage("First request", first); + logUsage("Second request", second); + + assertTotalTokensEqualsComponents(first); + assertTotalTokensEqualsComponents(second); + }); + }); + + // ========================================================================= + // Groq + // ========================================================================= + + describe.skipIf(!process.env.GROQ_API_KEY)("Groq", () => { + it( + "openai/gpt-oss-120b - should return totalTokens equal to sum of components", + { retry: 3, timeout: 60000 }, + async () => { + const llm = getModel("groq", "openai/gpt-oss-120b"); + + console.log(`\nGroq / ${llm.id}:`); + const { first, second } = await testTotalTokensWithCache(llm, { apiKey: process.env.GROQ_API_KEY }); + + logUsage("First request", first); + logUsage("Second request", second); + + assertTotalTokensEqualsComponents(first); + assertTotalTokensEqualsComponents(second); + }, + ); + }); + + // ========================================================================= + // Cerebras + // ========================================================================= + + describe.skipIf(!process.env.CEREBRAS_API_KEY)("Cerebras", () => { + it( + "gpt-oss-120b - should return totalTokens equal to sum of components", + { retry: 3, timeout: 60000 }, + async () => { + const llm = getModel("cerebras", "gpt-oss-120b"); + + console.log(`\nCerebras / ${llm.id}:`); + const { first, second } = await testTotalTokensWithCache(llm, { apiKey: process.env.CEREBRAS_API_KEY }); + + logUsage("First request", first); + logUsage("Second request", second); + + assertTotalTokensEqualsComponents(first); + assertTotalTokensEqualsComponents(second); + }, + ); + }); + + // ========================================================================= + // Cloudflare Workers AI + // ========================================================================= + + describe.skipIf(!hasCloudflareWorkersAICredentials())("Cloudflare Workers AI", () => { + it( + "@cf/moonshotai/kimi-k2.6 - should return totalTokens equal to sum of components", + { retry: 3, timeout: 60000 }, + async () => { + const llm = getModel("cloudflare-workers-ai", "@cf/moonshotai/kimi-k2.6"); + + console.log(`\nCloudflare Workers AI / ${llm.id}:`); + const { first, second } = await testTotalTokensWithCache(llm, { + apiKey: process.env.CLOUDFLARE_API_KEY, + }); + + logUsage("First request", first); + logUsage("Second request", second); + + assertTotalTokensEqualsComponents(first); + assertTotalTokensEqualsComponents(second); + }, + ); + }); + + // ========================================================================= + // Cloudflare AI Gateway + // ========================================================================= + + describe.skipIf(!hasCloudflareAiGatewayCredentials())("Cloudflare AI Gateway", () => { + it( + "workers-ai/@cf/moonshotai/kimi-k2.6 - should return totalTokens equal to sum of components", + { retry: 3, timeout: 60000 }, + async () => { + const llm = getModel("cloudflare-ai-gateway", "workers-ai/@cf/moonshotai/kimi-k2.6"); + + console.log(`\nCloudflare AI Gateway / ${llm.id}:`); + const { first, second } = await testTotalTokensWithCache(llm, { + apiKey: process.env.CLOUDFLARE_API_KEY, + }); + + logUsage("First request", first); + logUsage("Second request", second); + + assertTotalTokensEqualsComponents(first); + assertTotalTokensEqualsComponents(second); + }, + ); + }); + + // ========================================================================= + // Hugging Face + // ========================================================================= + + describe.skipIf(!process.env.HF_TOKEN)("Hugging Face", () => { + it("Kimi-K2.5 - should return totalTokens equal to sum of components", { retry: 3, timeout: 60000 }, async () => { + const llm = getModel("huggingface", "moonshotai/Kimi-K2.5"); + + console.log(`\nHugging Face / ${llm.id}:`); + const { first, second } = await testTotalTokensWithCache(llm, { apiKey: process.env.HF_TOKEN }); + + logUsage("First request", first); + logUsage("Second request", second); + + assertTotalTokensEqualsComponents(first); + assertTotalTokensEqualsComponents(second); + }); + }); + + // ========================================================================= + // Together AI + // ========================================================================= + + describe.skipIf(!process.env.TOGETHER_API_KEY)("Together AI", () => { + it("Kimi-K2.6 - should return totalTokens equal to sum of components", { retry: 3, timeout: 60000 }, async () => { + const llm = getModel("together", "moonshotai/Kimi-K2.6"); + + console.log(`\nTogether AI / ${llm.id}:`); + const { first, second } = await testTotalTokensWithCache(llm, { + apiKey: process.env.TOGETHER_API_KEY, + reasoningEffort: "high", + }); + + logUsage("First request", first); + logUsage("Second request", second); + + assertTotalTokensEqualsComponents(first); + assertTotalTokensEqualsComponents(second); + }); + }); + + // ========================================================================= + // Baseten + // ========================================================================= + + describe.skipIf(!process.env.BASETEN_API_KEY)("Baseten", () => { + it("GLM 5.2 - should return totalTokens equal to sum of components", { retry: 3, timeout: 60000 }, async () => { + const llm = getModel("baseten", "zai-org/GLM-5.2"); + + console.log(`\nBaseten / ${llm.id}:`); + const { first, second } = await testTotalTokensWithCache(llm, { + apiKey: process.env.BASETEN_API_KEY, + reasoningEffort: "high", + }); + + logUsage("First request", first); + logUsage("Second request", second); + + assertTotalTokensEqualsComponents(first); + assertTotalTokensEqualsComponents(second); + }); + }); + + // ========================================================================= + // z.ai + // ========================================================================= + + describe.skipIf(!process.env.ZAI_API_KEY)("z.ai", () => { + it("glm-5.2 - should return totalTokens equal to sum of components", { retry: 3, timeout: 60000 }, async () => { + const llm = getModel("zai", "glm-5.2"); + + console.log(`\nz.ai / ${llm.id}:`); + const { first, second } = await testTotalTokensWithCache(llm, { apiKey: process.env.ZAI_API_KEY }); + + logUsage("First request", first); + logUsage("Second request", second); + + assertTotalTokensEqualsComponents(first); + assertTotalTokensEqualsComponents(second); + }); + }); + + // ========================================================================= + // Mistral + // ========================================================================= + + describe.skipIf(!process.env.MISTRAL_API_KEY)("Mistral", () => { + it( + "devstral-medium-latest - should return totalTokens equal to sum of components", + { retry: 3, timeout: 60000 }, + async () => { + const llm = getModel("mistral", "devstral-medium-latest"); + + console.log(`\nMistral / ${llm.id}:`); + const { first, second } = await testTotalTokensWithCache(llm, { apiKey: process.env.MISTRAL_API_KEY }); + + logUsage("First request", first); + logUsage("Second request", second); + + assertTotalTokensEqualsComponents(first); + assertTotalTokensEqualsComponents(second); + }, + ); + }); + + // ========================================================================= + // MiniMax + // ========================================================================= + + describe.skipIf(!process.env.MINIMAX_API_KEY)("MiniMax", () => { + it( + "MiniMax-M2.7 - should return totalTokens equal to sum of components", + { retry: 3, timeout: 60000 }, + async () => { + const llm = getModel("minimax", "MiniMax-M2.7"); + + console.log(`\nMiniMax / ${llm.id}:`); + const { first, second } = await testTotalTokensWithCache(llm, { apiKey: process.env.MINIMAX_API_KEY }); + + logUsage("First request", first); + logUsage("Second request", second); + + assertTotalTokensEqualsComponents(first); + assertTotalTokensEqualsComponents(second); + }, + ); + }); + + // ========================================================================= + // Xiaomi MiMo + // ========================================================================= + + describe.skipIf(!process.env.XIAOMI_API_KEY)("Xiaomi MiMo (API billing)", () => { + it( + "mimo-v2.5-pro - should return totalTokens equal to sum of components", + { retry: 3, timeout: 60000 }, + async () => { + const llm = getModel("xiaomi", "mimo-v2.5-pro"); + + console.log(`\nXiaomi MiMo / ${llm.id}:`); + const { first, second } = await testTotalTokensWithCache(llm, { apiKey: process.env.XIAOMI_API_KEY }); + + logUsage("First request", first); + logUsage("Second request", second); + + assertTotalTokensEqualsComponents(first); + assertTotalTokensEqualsComponents(second); + }, + ); + }); + + // ========================================================================= + // Xiaomi MiMo Token Plan CN + // ========================================================================= + + describe.skipIf(!process.env.XIAOMI_TOKEN_PLAN_CN_API_KEY)("Xiaomi MiMo Token Plan (CN)", () => { + it( + "mimo-v2.5-pro - should return totalTokens equal to sum of components", + { retry: 3, timeout: 60000 }, + async () => { + const llm = getModel("xiaomi-token-plan-cn", "mimo-v2.5-pro"); + + console.log(`\nXiaomi MiMo Token Plan CN / ${llm.id}:`); + const { first, second } = await testTotalTokensWithCache(llm, { + apiKey: process.env.XIAOMI_TOKEN_PLAN_CN_API_KEY, + }); + + logUsage("First request", first); + logUsage("Second request", second); + + assertTotalTokensEqualsComponents(first); + assertTotalTokensEqualsComponents(second); + }, + ); + }); + + // ========================================================================= + // Xiaomi MiMo Token Plan AMS + // ========================================================================= + + describe.skipIf(!process.env.XIAOMI_TOKEN_PLAN_AMS_API_KEY)("Xiaomi MiMo Token Plan (AMS)", () => { + it( + "mimo-v2.5-pro - should return totalTokens equal to sum of components", + { retry: 3, timeout: 60000 }, + async () => { + const llm = getModel("xiaomi-token-plan-ams", "mimo-v2.5-pro"); + + console.log(`\nXiaomi MiMo Token Plan AMS / ${llm.id}:`); + const { first, second } = await testTotalTokensWithCache(llm, { + apiKey: process.env.XIAOMI_TOKEN_PLAN_AMS_API_KEY, + }); + + logUsage("First request", first); + logUsage("Second request", second); + + assertTotalTokensEqualsComponents(first); + assertTotalTokensEqualsComponents(second); + }, + ); + }); + + // ========================================================================= + // Xiaomi MiMo Token Plan SGP + // ========================================================================= + + describe.skipIf(!process.env.XIAOMI_TOKEN_PLAN_SGP_API_KEY)("Xiaomi MiMo Token Plan (SGP)", () => { + it( + "mimo-v2.5-pro - should return totalTokens equal to sum of components", + { retry: 3, timeout: 60000 }, + async () => { + const llm = getModel("xiaomi-token-plan-sgp", "mimo-v2.5-pro"); + + console.log(`\nXiaomi MiMo Token Plan SGP / ${llm.id}:`); + const { first, second } = await testTotalTokensWithCache(llm, { + apiKey: process.env.XIAOMI_TOKEN_PLAN_SGP_API_KEY, + }); + + logUsage("First request", first); + logUsage("Second request", second); + + assertTotalTokensEqualsComponents(first); + assertTotalTokensEqualsComponents(second); + }, + ); + }); + + // ========================================================================= + // Qwen Token Plan + // ========================================================================= + + describe.skipIf(!process.env.QWEN_TOKEN_PLAN_API_KEY)("Qwen Token Plan", () => { + it( + "qwen3.7-max - should return totalTokens equal to sum of components", + { retry: 3, timeout: 60000 }, + async () => { + const llm = getModel("qwen-token-plan", "qwen3.7-max"); + + console.log(`\nQwen Token Plan / ${llm.id}:`); + const { first, second } = await testTotalTokensWithCache(llm, { + apiKey: process.env.QWEN_TOKEN_PLAN_API_KEY, + }); + + logUsage("First request", first); + logUsage("Second request", second); + + assertTotalTokensEqualsComponents(first); + assertTotalTokensEqualsComponents(second); + }, + ); + }); + + // ========================================================================= + // Qwen Token Plan Individual + // ========================================================================= + + describe.skipIf(!process.env.QWEN_TOKEN_PLAN_API_KEY)("Qwen Token Plan Individual", () => { + it( + "qwen3.8-max - should return totalTokens equal to sum of components", + { retry: 3, timeout: 60000 }, + async () => { + const llm = getModel("qwen-token-plan-individual", "qwen3.8-max"); + + console.log(`\nQwen Token Plan Individual / ${llm.id}:`); + const { first, second } = await testTotalTokensWithCache(llm, { + apiKey: process.env.QWEN_TOKEN_PLAN_API_KEY, + }); + + logUsage("First request", first); + logUsage("Second request", second); + + assertTotalTokensEqualsComponents(first); + assertTotalTokensEqualsComponents(second); + }, + ); + }); + + // ========================================================================= + // Qwen Token Plan CN + // ========================================================================= + + describe.skipIf(!process.env.QWEN_TOKEN_PLAN_CN_API_KEY)("Qwen Token Plan (CN)", () => { + it( + "qwen3.7-max - should return totalTokens equal to sum of components", + { retry: 3, timeout: 60000 }, + async () => { + const llm = getModel("qwen-token-plan-cn", "qwen3.7-max"); + + console.log(`\nQwen Token Plan CN / ${llm.id}:`); + const { first, second } = await testTotalTokensWithCache(llm, { + apiKey: process.env.QWEN_TOKEN_PLAN_CN_API_KEY, + }); + + logUsage("First request", first); + logUsage("Second request", second); + + assertTotalTokensEqualsComponents(first); + assertTotalTokensEqualsComponents(second); + }, + ); + }); + + // ========================================================================= + // Kimi For Coding + // ========================================================================= + + describe.skipIf(!process.env.KIMI_API_KEY)("Kimi For Coding", () => { + it( + "kimi-for-coding - should return totalTokens equal to sum of components", + { retry: 3, timeout: 60000 }, + async () => { + const llm = getModel("kimi-coding", "kimi-for-coding"); + + console.log(`\nKimi For Coding / ${llm.id}:`); + const { first, second } = await testTotalTokensWithCache(llm, { apiKey: process.env.KIMI_API_KEY }); + + logUsage("First request", first); + logUsage("Second request", second); + + assertTotalTokensEqualsComponents(first); + assertTotalTokensEqualsComponents(second); + }, + ); + }); + + // ========================================================================= + // Vercel AI Gateway + // ========================================================================= + + describe.skipIf(!process.env.AI_GATEWAY_API_KEY)("Vercel AI Gateway", () => { + it( + "google/gemini-2.5-flash - should return totalTokens equal to sum of components", + { retry: 3, timeout: 60000 }, + async () => { + const llm = getModel("vercel-ai-gateway", "google/gemini-2.5-flash"); + + console.log(`\nVercel AI Gateway / ${llm.id}:`); + const { first, second } = await testTotalTokensWithCache(llm, { apiKey: process.env.AI_GATEWAY_API_KEY }); + + logUsage("First request", first); + logUsage("Second request", second); + + assertTotalTokensEqualsComponents(first); + assertTotalTokensEqualsComponents(second); + }, + ); + }); + + // ========================================================================= + // OpenRouter - Multiple backend providers + // ========================================================================= + + describe.skipIf(!process.env.OPENROUTER_API_KEY)("OpenRouter", () => { + it( + "anthropic/claude-sonnet-4 - should return totalTokens equal to sum of components", + { retry: 3, timeout: 60000 }, + async () => { + const llm = getModel("openrouter", "anthropic/claude-sonnet-4"); + + console.log(`\nOpenRouter / ${llm.id}:`); + const { first, second } = await testTotalTokensWithCache(llm, { apiKey: process.env.OPENROUTER_API_KEY }); + + logUsage("First request", first); + logUsage("Second request", second); + + assertTotalTokensEqualsComponents(first); + assertTotalTokensEqualsComponents(second); + }, + ); + + it( + "deepseek/deepseek-chat - should return totalTokens equal to sum of components", + { retry: 3, timeout: 60000 }, + async () => { + const llm = getModel("openrouter", "deepseek/deepseek-chat"); + + console.log(`\nOpenRouter / ${llm.id}:`); + const { first, second } = await testTotalTokensWithCache(llm, { apiKey: process.env.OPENROUTER_API_KEY }); + + logUsage("First request", first); + logUsage("Second request", second); + + assertTotalTokensEqualsComponents(first); + assertTotalTokensEqualsComponents(second); + }, + ); + + it( + "mistralai/mistral-small-3.2-24b-instruct - should return totalTokens equal to sum of components", + { retry: 3, timeout: 60000 }, + async () => { + const llm = getModel("openrouter", "mistralai/mistral-small-3.2-24b-instruct"); + + console.log(`\nOpenRouter / ${llm.id}:`); + const { first, second } = await testTotalTokensWithCache(llm, { apiKey: process.env.OPENROUTER_API_KEY }); + + logUsage("First request", first); + logUsage("Second request", second); + + assertTotalTokensEqualsComponents(first); + assertTotalTokensEqualsComponents(second); + }, + ); + + it( + "google/gemini-2.5-flash - should return totalTokens equal to sum of components", + { retry: 3, timeout: 60000 }, + async () => { + const llm = getModel("openrouter", "google/gemini-2.5-flash"); + + console.log(`\nOpenRouter / ${llm.id}:`); + const { first, second } = await testTotalTokensWithCache(llm, { apiKey: process.env.OPENROUTER_API_KEY }); + + logUsage("First request", first); + logUsage("Second request", second); + + assertTotalTokensEqualsComponents(first); + assertTotalTokensEqualsComponents(second); + }, + ); + + it( + "deepseek/deepseek-chat - should return totalTokens equal to sum of components", + { retry: 3, timeout: 60000 }, + async () => { + const llm = getModel("openrouter", "deepseek/deepseek-chat"); + + console.log(`\nOpenRouter / ${llm.id}:`); + const { first, second } = await testTotalTokensWithCache(llm, { apiKey: process.env.OPENROUTER_API_KEY }); + + logUsage("First request", first); + logUsage("Second request", second); + + assertTotalTokensEqualsComponents(first); + assertTotalTokensEqualsComponents(second); + }, + ); + }); + + // ========================================================================= + // GitHub Copilot (OAuth) + // ========================================================================= + + describe("GitHub Copilot (OAuth)", () => { + it.skipIf(!githubCopilotToken)( + "claude-haiku-4.5 - should return totalTokens equal to sum of components", + { retry: 3, timeout: 60000 }, + async () => { + const llm = getModel("github-copilot", "claude-haiku-4.5"); + + console.log(`\nGitHub Copilot / ${llm.id}:`); + const { first, second } = await testTotalTokensWithCache(llm, { apiKey: githubCopilotToken }); + + logUsage("First request", first); + logUsage("Second request", second); + + assertTotalTokensEqualsComponents(first); + assertTotalTokensEqualsComponents(second); + }, + ); + + it.skipIf(!githubCopilotToken)( + "claude-sonnet-4 - should return totalTokens equal to sum of components", + { retry: 3, timeout: 60000 }, + async () => { + const llm = getModel("github-copilot", "claude-sonnet-4.6"); + + console.log(`\nGitHub Copilot / ${llm.id}:`); + const { first, second } = await testTotalTokensWithCache(llm, { apiKey: githubCopilotToken }); + + logUsage("First request", first); + logUsage("Second request", second); + + assertTotalTokensEqualsComponents(first); + assertTotalTokensEqualsComponents(second); + }, + ); + }); + + // ========================================================================= + // ========================================================================= + + // ========================================================================= + // ========================================================================= + + describe.skipIf(!hasBedrockCredentials())("Amazon Bedrock", () => { + it( + "claude-sonnet-4-5 - should return totalTokens equal to sum of components", + { retry: 3, timeout: 60000 }, + async () => { + const llm = getModel("amazon-bedrock", "global.anthropic.claude-sonnet-4-5-20250929-v1:0"); + + console.log(`\nAmazon Bedrock / ${llm.id}:`); + const { first, second } = await testTotalTokensWithCache(llm); + + logUsage("First request", first); + logUsage("Second request", second); + + assertTotalTokensEqualsComponents(first); + assertTotalTokensEqualsComponents(second); + }, + ); + }); + + // ========================================================================= + // OpenAI Codex (OAuth) + // ========================================================================= + + describe("OpenAI Codex (OAuth)", () => { + it.skipIf(!openaiCodexToken)( + "gpt-5.5 - should return totalTokens equal to sum of components", + { retry: 3, timeout: 60000 }, + async () => { + const llm = getModel("openai-codex", "gpt-5.5"); + + console.log(`\nOpenAI Codex / ${llm.id}:`); + const { first, second } = await testTotalTokensWithCache(llm, { apiKey: openaiCodexToken }); + + logUsage("First request", first); + logUsage("Second request", second); + + assertTotalTokensEqualsComponents(first); + assertTotalTokensEqualsComponents(second); + }, + ); + }); +}); diff --git a/packages/ai/test/uuid.test.ts b/packages/ai/test/uuid.test.ts new file mode 100644 index 0000000000000000000000000000000000000000..6100104ceeca5520cd3b44485463de785ba664e3 --- /dev/null +++ b/packages/ai/test/uuid.test.ts @@ -0,0 +1,57 @@ +import { afterEach, describe, expect, it, vi } from "vitest"; +import { uuidv7 } from "../src/utils/uuid.ts"; + +const UUID_V7_RE = /^[0-9a-f]{8}-[0-9a-f]{4}-7[0-9a-f]{3}-[89ab][0-9a-f]{3}-[0-9a-f]{12}$/; +const TIMESTAMP = 0x0123456789ab; + +function parseTimestamp(uuid: string): number { + return Number.parseInt(uuid.replaceAll("-", "").slice(0, 12), 16); +} + +afterEach(() => { + vi.useRealTimers(); + vi.unstubAllGlobals(); +}); + +describe("uuidv7", () => { + it("generates ordered UUIDv7s while preserving follower timestamps", () => { + vi.useFakeTimers(); + vi.setSystemTime(TIMESTAMP); + + const first = uuidv7(); + const second = uuidv7(); + vi.setSystemTime(TIMESTAMP - 1); + const afterRollback = uuidv7(); + vi.setSystemTime(TIMESTAMP + 1); + const afterAdvance = uuidv7(); + const ordinaryIds = [first, second, afterRollback, afterAdvance]; + const followerTimestamp = TIMESTAMP - 1_000; + const followers = [uuidv7(followerTimestamp), uuidv7(followerTimestamp)]; + + for (const id of [...ordinaryIds, ...followers]) expect(id).toMatch(UUID_V7_RE); + expect(ordinaryIds).toEqual([...ordinaryIds].sort()); + expect(new Set(ordinaryIds)).toHaveLength(ordinaryIds.length); + expect(ordinaryIds.map(parseTimestamp)).toEqual([TIMESTAMP, TIMESTAMP, TIMESTAMP, TIMESTAMP + 1]); + expect(followers.map(parseTimestamp)).toEqual([followerTimestamp, followerTimestamp]); + expect(new Set(followers)).toHaveLength(followers.length); + }); + + it("uses fresh randomness for every UUID tail", () => { + let randomByte = 0; + vi.stubGlobal("crypto", { + getRandomValues(bytes: Uint8Array) { + return bytes.fill(++randomByte); + }, + }); + + expect([uuidv7(TIMESTAMP).slice(-8), uuidv7(TIMESTAMP).slice(-8)]).toEqual(["01010101", "02020202"]); + }); + + it.each([0, 2 ** 48 - 1])("accepts timestamp boundary %s", (timestamp) => { + expect(parseTimestamp(uuidv7(timestamp))).toBe(timestamp); + }); + + it.each([-1, 2 ** 48, 1.5, Number.NaN, Number.POSITIVE_INFINITY])("rejects invalid timestamp %s", (timestamp) => { + expect(() => uuidv7(timestamp)).toThrow(RangeError); + }); +}); diff --git a/packages/ai/test/xhigh.test.ts b/packages/ai/test/xhigh.test.ts new file mode 100644 index 0000000000000000000000000000000000000000..0803c3757afd25025ef97dbb9eac77def7d00094 --- /dev/null +++ b/packages/ai/test/xhigh.test.ts @@ -0,0 +1,70 @@ +import { describe, expect, it } from "vitest"; +import { getModel, stream } from "../src/compat.ts"; +import type { Context, Model } from "../src/types.ts"; + +function makeContext(): Context { + return { + messages: [ + { + role: "user", + content: `What is ${(Math.random() * 100) | 0} + ${(Math.random() * 100) | 0}? Think step by step.`, + timestamp: Date.now(), + }, + ], + }; +} + +describe.skipIf(!process.env.OPENAI_API_KEY)("xhigh reasoning", () => { + describe("gpt 5.5 (supports xhigh)", () => { + // Note: codex models only support the responses API, not chat completions + it("should work with openai-responses", async () => { + const model = getModel("openai", "gpt-5.5"); + const s = stream(model, makeContext(), { reasoningEffort: "xhigh" }); + let hasThinking = false; + + for await (const event of s) { + if (event.type === "thinking_start" || event.type === "thinking_delta") { + hasThinking = true; + } + } + + const response = await s.result(); + expect(response.stopReason, `Error: ${response.errorMessage}`).toBe("stop"); + expect(response.content.some((b) => b.type === "text")).toBe(true); + expect(hasThinking || response.content.some((b) => b.type === "thinking")).toBe(true); + }); + }); + + describe("gpt-5-mini (does not support xhigh)", () => { + it("should error with openai-responses when using xhigh", async () => { + const model = getModel("openai", "gpt-5-mini"); + const s = stream(model, makeContext(), { reasoningEffort: "xhigh" }); + + for await (const _ of s) { + // drain events + } + + const response = await s.result(); + expect(response.stopReason).toBe("error"); + expect(response.errorMessage).toContain("xhigh"); + }); + + it("should error with openai-completions when using xhigh", async () => { + const { compat: _compat, ...baseModel } = getModel("openai", "gpt-5-mini"); + void _compat; + const model: Model<"openai-completions"> = { + ...baseModel, + api: "openai-completions", + }; + const s = stream(model, makeContext(), { reasoningEffort: "xhigh" }); + + for await (const _ of s) { + // drain events + } + + const response = await s.result(); + expect(response.stopReason).toBe("error"); + expect(response.errorMessage).toContain("xhigh"); + }); + }); +}); diff --git a/packages/ai/test/zen.test.ts b/packages/ai/test/zen.test.ts new file mode 100644 index 0000000000000000000000000000000000000000..0f731b46928ef75d83a2dfffb13c514d0451cc94 --- /dev/null +++ b/packages/ai/test/zen.test.ts @@ -0,0 +1,25 @@ +import { describe, expect, it } from "vitest"; +import { complete } from "../src/compat.ts"; +import { MODELS } from "../src/models.generated.ts"; +import type { Model } from "../src/types.ts"; + +describe.skipIf(!process.env.OPENCODE_API_KEY)("OpenCode Models Smoke Test", () => { + const providers = [ + { key: "opencode", label: "OpenCode Zen" }, + { key: "opencode-go", label: "OpenCode Go" }, + ] as const; + + providers.forEach(({ key, label }) => { + const providerModels = Object.values(MODELS[key]); + providerModels.forEach((model) => { + it(`${label}: ${model.id}`, async () => { + const response = await complete(model as Model, { + messages: [{ role: "user", content: "Say hello.", timestamp: Date.now() }], + }); + + expect(response.content).toBeTruthy(); + expect(response.stopReason).toBe("stop"); + }, 60000); + }); + }); +});