| // Upstream-error classification shared by the proxy chat path, the responses | |
| // path, and the fusion panel. Pure functions over an error's message/status — | |
| // no I/O — so they live in a neutral lib module that any of those can import | |
| // without forming an import cycle (fusion ↔ proxy in particular). | |
| export function isRetryableError(err: any): boolean { | |
| const msg = (err.message ?? '').toLowerCase(); | |
| // Trust the upstream HTTP status the provider attached to the error first | |
| // (providerHttpError in providers/base.ts sets err.status on every adapter). | |
| // This structured check is the robust primary signal; the message-substring | |
| // rules below are the fallback for errors that carry a code in their text but | |
| // no numeric status. It's the fix for #337/#339: an Ollama "410 Gone", or any | |
| // upstream 5xx the substring allowlist never enumerated (502/504/507…), used to | |
| // fall through to a 502 and STRAND the healthy paid routes still queued later in | |
| // the chain — because the old code matched specific substrings and ignored | |
| // err.status for every code except 403. 408 (request timeout), 409 (conflict), | |
| // 410 (model pulled upstream), 429 (rate limit) and all 5xx are transient or | |
| // fail-over-able; 400/401 stay fatal (status 0 here, handled by the absence of a | |
| // matching rule) and 403 is handled by isModelAccessForbiddenError below. | |
| // 422 (notably Mistral's strict validation) is a provider-side request-shape | |
| // rejection: fail over to another provider, but if the whole chain rejects it | |
| // exhaustedRetryError renders a client-facing 400 via isProviderBadRequestError. | |
| const status = typeof err?.status === 'number' ? err.status : 0; | |
| if (status === 408 || status === 409 || status === 410 || status === 422 || status === 429 || status >= 500) return true; | |
| return msg.includes('429') || msg.includes('rate limit') || msg.includes('too many requests') | |
| || msg.includes('quota') || msg.includes('resource_exhausted') | |
| || msg.includes('aborted') || msg.includes('timeout') || msg.includes('etimedout') | |
| || msg.includes('econnrefused') || msg.includes('econnreset') | |
| || msg.includes('fetch failed') // undici transport error (proxy down, DNS, TLS, etc.) | |
| || msg.includes('503') || msg.includes('unavailable') | |
| // Provider marks the hosted deployment itself as sick (NVIDIA NIM's | |
| // "DEGRADED function cannot be invoked" arrives as a 400, #522). The | |
| // 'api error 400' rule below already catches the NIM shape; this keeps | |
| // degraded conditions retryable even when a provider words it differently. | |
| || msg.includes('degraded') | |
| || msg.includes('500') || msg.includes('internal server error') | |
| // 413: this model's payload limit is too small for the request, but another | |
| // provider in the fallback chain may have a larger limit. Same reasoning as 503. | |
| || msg.includes('413') || msg.includes('payload too large') || msg.includes('request body too large') | |
| || msg.includes('request entity too large') || msg.includes('content too large') | |
| // Context/prompt-too-large in all its provider wordings (Anthropic "prompt | |
| // is too long", Google "input token count ... exceeds", OpenAI "maximum | |
| // context length", …): another candidate may have a larger window, so fail | |
| // over; if EVERY candidate rejects it the exhaustion ladder renders an | |
| // honest 413 instead of a generic failure. | |
| || isContextTooLargeError(err) | |
| // 404: model deprecated/removed upstream (e.g. OpenRouter's "no endpoints found" | |
| // for a model that's been pulled). Rotate to the next model in the chain — | |
| // setCooldown + the health checker will avoid this model on subsequent requests. | |
| || msg.includes('404') || msg.includes('not found') || msg.includes('no endpoints found') | |
| // 410: the model/endpoint was permanently removed upstream (e.g. Ollama Cloud | |
| // "API error 410: Gone", #339). Like a 404 it won't return on this provider, so | |
| // rotate to the next route; isModelNotFoundError benches the whole model. The | |
| // structured status check above already catches the 410 when the provider | |
| // attaches err.status — this is the text fallback for errors that don't. | |
| || msg.includes('410') || msg.includes('gone') | |
| // 403: the key is valid (it passed validateKey, and the health checker | |
| // disables truly-forbidden keys) but this specific model is off-limits to | |
| // the key's tier — e.g. gpt-4o on GitHub Models' free tier, subscription-only | |
| // models on Cloudflare. Another model in the chain is reachable, so fail over | |
| // instead of 502-ing the whole request. Paired with isModelAccessForbiddenError | |
| // to rule the model out for this request and a day-long bench. See issue #256. | |
| || isModelAccessForbiddenError(err) | |
| // 400: one provider may reject parameters another accepts (e.g. max_tokens | |
| // limits, unsupported params). The matching pattern is "api error 400" | |
| // which comes from the OpenAI-compat provider's error formatting, not | |
| // a bare "400" which is deliberately non-retryable for validation errors. | |
| || msg.includes('api error 400') | |
| // 422: Mistral and other strict OpenAI-compatible endpoints use | |
| // Unprocessable Entity for request-shape validation. Rotate to a provider | |
| // that accepts the same OpenAI payload; if every provider rejects it, render | |
| // a clean invalid_request_error instead of a rate-limit exhaustion. | |
| || msg.includes('api error 422') || msg.includes('unprocessable entity') | |
| // 402: this provider/key is out of credits (e.g. HuggingFace Router | |
| // "API error 402: Payment required"). The SAME model often lives on another | |
| // provider (Kimi K2.6 is on HF + Cloudflare + NVIDIA), so fail over instead | |
| // of killing the workflow. Paired with a long cooldown (isPaymentRequiredError) | |
| // so we don't re-hammer the broke key every retry. | |
| || isPaymentRequiredError(err) | |
| // Dead-turn classes from the stream turn-integrity layer (#231 audit): | |
| // all thrown before any byte reached the client, so another model can | |
| // serve the request invisibly. | |
| || msg.includes('empty completion') | |
| // The model answered in prose despite a response_format request (and the | |
| // JSON couldn't be healed out of the text) — thrown before any byte | |
| // reached the client, so the next candidate can serve it invisibly. | |
| || msg.includes('ignored response_format') | |
| // The model DID emit JSON but ran out of max_tokens mid-value | |
| // (finish_reason 'length'). A terser model may fit the same schema in the | |
| // same budget, so fail over — but the thrower marks the model skipped for | |
| // this request: retrying the same model would truncate identically. | |
| || msg.includes('truncated json') | |
| || msg.includes('in-band provider error') | |
| || msg.includes('stream ended unexpectedly') | |
| || msg.includes('stream stalled') | |
| // First-byte timeout (#584): the grace budget expired before ANY byte | |
| // reached the client, so the next candidate can serve it invisibly. | |
| || msg.includes('no first byte') | |
| || msg.includes('unparseable inline tool-call dialect') | |
| // The model emitted a tool call whose arguments violate the schema the | |
| // caller declared (opt-in check, lib/tool-validate.ts). Thrown before any | |
| // byte reached the client, and a different model usually gets the same | |
| // call right — the thrower marks the model skipped for this request, since | |
| // a sibling key would misbehave identically. | |
| || msg.includes('invalid tool arguments') | |
| // A transport failure undici buried in `err.cause` rather than the | |
| // top-level message (see isTransportError). Purely additive: every rule | |
| // above is unchanged, and this is a second, structured signal for the | |
| // errors whose real cause never reaches the text we match on. | |
| || isTransportError(err); | |
| } | |
| // ── Transport failures hidden in the cause chain (undici) ──────────────────── | |
| // Every rule in isRetryableError above reads the TOP-LEVEL message, and undici | |
| // does not always put the real failure there. The INITIAL-CONNECT case is | |
| // covered by luck: undici words a connection that never opened "fetch failed", | |
| // and the allowlist matches that string. A socket that dies MID-request does | |
| // not get the same treatment — it surfaces as a generic wrapper (a bare | |
| // `TypeError: terminated`, or an adapter's own re-throw) whose `.cause` carries | |
| // the actual ECONNRESET / EPIPE / "socket hang up" / "premature close" / | |
| // "other side closed" error, sometimes nested a link or two deeper still. | |
| // Those fell through every rule and classified FATAL: the client got a 502 | |
| // while the healthy paid routes queued behind the dead one were never tried. | |
| // | |
| // Everything matched here is a transient network fault that says nothing about | |
| // the request or the model, so the next candidate in the chain can serve it. | |
| // The walk is bounded and cycle-safe: `cause` is attacker-adjacent data (it can | |
| // come from a provider's own error object) and a self-referential chain on this | |
| // hot path would hang the request, not just slow it. | |
| const TRANSPORT_CAUSE_MAX_DEPTH = 5; | |
| const TRANSPORT_ERROR_CODES = new Set([ | |
| // Node socket-level codes. A dead/refused/unreachable host is transient from | |
| // the chain's point of view either way: the next candidate is a DIFFERENT | |
| // host, so even a permanently bad one here is worth failing over rather than | |
| // 502-ing the caller. | |
| 'ECONNRESET', 'ECONNREFUSED', 'ECONNABORTED', 'EPIPE', 'ETIMEDOUT', | |
| 'EAI_AGAIN', 'ENOTFOUND', 'EHOSTUNREACH', 'ENETUNREACH', 'EADDRNOTAVAIL', | |
| ]); | |
| // undici stamps its own transport failures with a `UND_ERR_*` code | |
| // (UND_ERR_SOCKET, UND_ERR_CONNECT_TIMEOUT, UND_ERR_HEADERS_TIMEOUT, | |
| // UND_ERR_BODY_TIMEOUT, …). Matched by PREFIX rather than enumerated: the list | |
| // grows with undici releases, and every member of it is a transport condition. | |
| const UNDICI_ERROR_CODE_PREFIX = 'UND_ERR_'; | |
| const TRANSPORT_MESSAGE_HINTS = [ | |
| 'socket hang up', | |
| 'premature close', | |
| 'other side closed', | |
| // A socket dropped before the TLS handshake finished — undici's full wording | |
| // is "Client network socket disconnected before secure TLS connection was | |
| // established". The peer closed the TCP connection mid-handshake; transient. | |
| 'client network socket disconnected', | |
| // The same codes as above, for the links that carry them in text only (an | |
| // error stringified across a boundary keeps the code in its message but | |
| // loses the `code` property). | |
| 'econnreset', 'econnrefused', 'epipe', 'etimedout', 'eai_again', | |
| ]; | |
| /** Walk an error's `cause` chain, yielding each link's `code` and `message`. | |
| * Includes the error itself as the first link. Bounded to | |
| * TRANSPORT_CAUSE_MAX_DEPTH hops and cycle-safe. */ | |
| function errorChainLinks(err: unknown): Array<{ code: string; message: string }> { | |
| const links: Array<{ code: string; message: string }> = []; | |
| const seen = new Set<unknown>(); | |
| let cur: any = err; | |
| for (let depth = 0; cur != null && depth <= TRANSPORT_CAUSE_MAX_DEPTH; depth++) { | |
| if (typeof cur !== 'object' && typeof cur !== 'function') break; | |
| if (seen.has(cur)) break; | |
| seen.add(cur); | |
| links.push({ | |
| code: typeof cur.code === 'string' ? cur.code : '', | |
| message: typeof cur.message === 'string' ? cur.message : '', | |
| }); | |
| cur = cur.cause; | |
| } | |
| return links; | |
| } | |
| /** True when the error — or anything in its bounded cause chain — is a | |
| * transient network transport failure: a reset, dropped, refused or timed-out | |
| * socket, a failed TLS handshake, or an undici transport code. */ | |
| export function isTransportError(err: any): boolean { | |
| if (err == null) return false; | |
| // A client hang-up and the fallback time-budget hedge BOTH reach undici as an | |
| // aborted socket, and undici labels that abort UND_ERR_ABORTED — the very | |
| // same code a genuine mid-flight socket death carries. Neither is provider | |
| // health, so classifying either retryable would resurrect exactly what the | |
| // two marked abort errors exist to prevent (no cooldown, no health penalty, | |
| // no failure stats for a request the GATEWAY canceled). The structured | |
| // markers are authoritative and win over any transport evidence below. | |
| if (isClientAbortError(err) || isHedgeAbortError(err)) return false; | |
| for (const { code, message } of errorChainLinks(err)) { | |
| if (code && (TRANSPORT_ERROR_CODES.has(code) || code.startsWith(UNDICI_ERROR_CODE_PREFIX))) return true; | |
| const msg = message.toLowerCase(); | |
| if (!msg) continue; | |
| if (TRANSPORT_MESSAGE_HINTS.some(hint => msg.includes(hint))) return true; | |
| // undici's wording for a response body whose socket died mid-read is the | |
| // bare word "terminated". Matched as the WHOLE message and never as a | |
| // substring: "your account has been terminated" is a fatal billing/auth | |
| // condition, and retrying that around the entire chain would burn every | |
| // candidate on a request that can never succeed. | |
| if (msg.trim() === 'terminated') return true; | |
| } | |
| return false; | |
| } | |
| // A genuine provider QUOTA signal: a structured 429 or rate-limit/quota wording. | |
| // Distinct from the much broader isRetryableError: timeouts, 5xx, transport | |
| // failures and dead-turn classes are retryable but say nothing about quotas. | |
| // The null-limits exhaustion heuristic in getCooldownDecisionForLimit (services/ | |
| // ratelimit.ts) keys off this: only an actual quota signal may feed its | |
| // "effectively daily-exhausted" escalation ladder. Before this split, a slow | |
| // local Ollama route timing out twice classified retryable → recorded as a | |
| // null-limit "429" → escalated 2m→10m→1h→24h and 429'd every request while the | |
| // server was merely busy generating (#592). | |
| // Mirrors the other classifiers: trust the structured status when the adapter | |
| // attached one (providerHttpError sets err.status everywhere); fall back to | |
| // message markers only when there is no status. | |
| export function isRateLimitSignal(err: any): boolean { | |
| const status = typeof err?.status === 'number' ? err.status : 0; | |
| if (status !== 0) return status === 429; | |
| const msg = (err?.message ?? '').toLowerCase(); | |
| return msg.includes('429') || msg.includes('rate limit') || msg.includes('too many requests') | |
| || msg.includes('quota') || msg.includes('resource_exhausted'); | |
| } | |
| // ── Timeout wording ───────────────────────────────────────────────────────── | |
| // The message markers that mean "this attempt ran out of time", in every | |
| // wording the stack produces: the per-attempt HTTP deadline ("The operation was | |
| // aborted (groq, chat, 120s)" — enrichAbort in lib/proxy.ts), the mid-stream | |
| // inactivity watchdog ("… stream stalled: no data for 90000ms (timeout)"), the | |
| // first-byte grace budget, and a raw socket ETIMEDOUT. | |
| // | |
| // A client hang-up is deliberately NOT in here: newClientAbortError's message | |
| // ("client disconnected — upstream request canceled") carries none of these | |
| // markers precisely so a vanished client is never read as provider slowness. | |
| // | |
| // Single source of truth for two consumers: the attempt-trail classifier | |
| // (classifyAttemptError) and the analytics query that folds timeouts into the | |
| // speed score (refreshStatsCache, #619) — so the trail and the score always | |
| // agree about what a timeout is. | |
| export const TIMEOUT_ERROR_MARKERS = ['timeout', 'stalled', 'etimedout', 'aborted'] as const; | |
| /** True when an error message reads as a timeout. Expects raw text; caller need | |
| * not lowercase it. */ | |
| export function isTimeoutErrorText(message: unknown): boolean { | |
| const msg = (typeof message === 'string' ? message : String(message ?? '')).toLowerCase(); | |
| return TIMEOUT_ERROR_MARKERS.some(marker => msg.includes(marker)); | |
| } | |
| // ── Client-caused aborts ───────────────────────────────────────────────────── | |
| // When the gateway's OWN client hangs up, the proxy surfaces abort the composed | |
| // fetch signal with this marked error, and undici rejects the in-flight fetch / | |
| // body read / stream read with the same object (fetch propagates the signal's | |
| // abort reason). It is deliberately NOT an AbortError and its message contains | |
| // neither "aborted" nor "timeout": enrichAbort (lib/proxy.ts) must pass it | |
| // through untouched, and it must never classify as a retryable provider | |
| // timeout — a vanished client says nothing about provider health, so the | |
| // fallback loop stops without a cooldown, health penalty, or failure stats. | |
| export function newClientAbortError(): Error { | |
| const err = new Error('client disconnected — upstream request canceled'); | |
| (err as Error & { clientAbort?: boolean }).clientAbort = true; | |
| return err; | |
| } | |
| /** True when an error is (or wraps) the client-disconnect abort above. The | |
| * structured marker is the primary signal; `cause` is checked because some | |
| * transports re-wrap the abort reason, and the message substring is the last | |
| * resort for errors that were stringified across a boundary. */ | |
| export function isClientAbortError(err: any): boolean { | |
| if (err?.clientAbort === true) return true; | |
| if (err?.cause && (err.cause as { clientAbort?: boolean }).clientAbort === true) return true; | |
| const msg = (err?.message ?? '').toLowerCase(); | |
| return msg.includes('client disconnected'); | |
| } | |
| // ── Fallback time-budget hedging (fallback-v2) ────────────────────────────── | |
| // When the wall-clock retry budget expires MID-FLIGHT (the current attempt is | |
| // still waiting on a stalled upstream), the fallback loop aborts the composed | |
| // fetch signal with this marked error. Same rationale as the client abort: it | |
| // is NOT a provider-health signal, so it must never bench/cooldown/penalize | |
| // the model+key, and enrichAbort must pass it through untouched. The loop | |
| // catches it, renders timedOut exhaustion (the budget is spent — nothing left | |
| // to try), and stops without failure bookkeeping. | |
| export function newHedgeAbortError(): Error { | |
| const err = new Error('fallback time budget expired — upstream request canceled'); | |
| (err as Error & { hedgeAbort?: boolean }).hedgeAbort = true; | |
| return err; | |
| } | |
| /** True when an error is (or wraps) the time-budget hedge abort above. */ | |
| export function isHedgeAbortError(err: any): boolean { | |
| if (err?.hedgeAbort === true) return true; | |
| if (err?.cause && (err.cause as { hedgeAbort?: boolean }).hedgeAbort === true) return true; | |
| const msg = (err?.message ?? '').toLowerCase(); | |
| return msg.includes('fallback time budget expired'); | |
| } | |
| /** True for any fetch-abort rejection surfacing out of a body read — the | |
| * per-attempt timeout ('request'-bounds deadline in fetchWithTimeout), an | |
| * AbortSignal.timeout, or the client disconnect above. Adapters that wrap | |
| * res.json() in a diagnostic catch (openai-compat's non-JSON-body split) must | |
| * rethrow these instead of classifying them as a malformed provider body. */ | |
| export function isAbortLikeError(err: any): boolean { | |
| if (isClientAbortError(err)) return true; | |
| const name = (err as { name?: string })?.name; | |
| return name === 'AbortError' || name === 'TimeoutError' || /\baborted\b/i.test(err?.message ?? ''); | |
| } | |
| // A 401 / invalid-API-key error from a provider. KEY-fatal, not request-fatal: | |
| // the same request is fine on the provider's sibling key or on another provider, | |
| // so the fallback loop rotates past the bad key (and triggers an immediate | |
| // health revalidation) instead of 502-ing the whole request. Deliberately NOT | |
| // added to isRetryableError: a bad key must be skipped and revalidated, not | |
| // blindly re-benched like a rate limit — the loop handles it as its own class. | |
| // | |
| // Status handling: every provider adapter attaches err.status (providerHttpError | |
| // in providers/base.ts), so the structured status is the primary signal. | |
| // - 401 → always key-auth. | |
| // - 400 → key-auth ONLY for Google-style key-specific phrasings: Google | |
| // reports a bad/expired key as HTTP 400 INVALID_ARGUMENT with "API key not | |
| // valid" / "API key expired" / API_KEY_INVALID (#268, providers/google.ts). | |
| // Without this a dead Google key classified as a provider bad-request and | |
| // could exhaust into a client-blaming 400 "rejected the request as invalid". | |
| // Generic auth wording (unauthorized etc.) stays excluded on a 400 so | |
| // ordinary payload rejections never classify as key-auth. | |
| // - any other status → not key-auth (e.g. a 403 stays model-forbidden). | |
| // - no status at all → key-specific OR generic auth substrings. | |
| export function isKeyAuthError(err: any): boolean { | |
| const status = typeof err?.status === 'number' ? err.status : 0; | |
| if (status === 401) return true; | |
| const msg = (err?.message ?? '').toLowerCase(); | |
| const keySpecific = msg.includes('api key not valid') | |
| || msg.includes('api key expired') | |
| || msg.includes('api_key_invalid'); | |
| if (status === 400) return keySpecific; | |
| if (status !== 0) return false; | |
| return keySpecific | |
| || msg.includes('401') | |
| || msg.includes('unauthorized') | |
| || msg.includes('invalid api key') | |
| || msg.includes('invalid_api_key') | |
| || msg.includes('incorrect api key') | |
| || msg.includes('authentication failed'); | |
| } | |
| // A 429 whose body says the provider's DAILY free allocation is spent (observed | |
| // live: Cloudflare "you have used up your daily free allocation of 10,000 | |
| // neurons"; OpenRouter "free-models-per-day"). A transient 90s cooldown just | |
| // makes the router re-pick a dead-for-the-day provider all day, so the caller | |
| // benches until the next UTC midnight instead. Requires BOTH a daily marker and | |
| // a quota/allocation marker so an ordinary per-minute 429 never matches. | |
| // | |
| // Groq ends every limit error, per-minute and request-too-large alike, with | |
| // "Need more tokens? Upgrade to Dev Tier today at …". That "today" is sales | |
| // copy, not a window, so it is cut before looking for a daily marker; and a | |
| // request too big for the model is never a spent allowance. | |
| const UPGRADE_UPSELL = /\bupgrade to [^.?!]*?\btoday\b/g; | |
| export function isDailyQuotaExhaustedError(err: any): boolean { | |
| if (err?.dailyQuotaExhausted === true) return true; | |
| if (isContextTooLargeError(err)) return false; | |
| const msg = (err?.message ?? '').toLowerCase().replace(UPGRADE_UPSELL, ''); | |
| if (!/daily|per[ -_]?day|\btoday\b/.test(msg)) return false; | |
| return /allocation|quota|limit|exhaust|used up/.test(msg); | |
| } | |
| // A provider-side "this hosted model is temporarily degraded" condition dressed | |
| // up as a 400. Observed live on NVIDIA NIM (issue #522): a degraded function | |
| // returns `400 {"detail":"Function id '...': DEGRADED function cannot be | |
| // invoked"}` — the request is fine; the deployment is sick. Must NOT classify | |
| // as a provider bad-request: exhausting on it would render a client-blaming | |
| // 400 invalid_request_error for what is capacity/health, not request shape. | |
| export function isProviderDegradedError(err: any): boolean { | |
| const msg = (err?.message ?? '').toLowerCase(); | |
| return msg.includes('degraded'); | |
| } | |
| // #788: provider-level failures — 5xx, timeouts, transport/network errors, a | |
| // degraded deployment — are symptoms of the PROVIDER being sick, not of this | |
| // particular key. Retrying the same provider with a sibling key would fail | |
| // identically, so the fallback loop must skip the WHOLE platform for the | |
| // request instead of burning one failover hop per key. Key-scoped failures | |
| // (auth, quota, 403 tier) stay out of here so a dead key can still rotate to | |
| // a healthy sibling on the same platform. | |
| // | |
| // Classified on the STRUCTURED status — every adapter attaches one to an HTTP | |
| // failure (providerHttpError in providers/base.ts) — plus the two families that | |
| // carry no status at all: a timeout and a transport-level failure. Deliberately | |
| // NO bare '500' / '503' / 'unavailable' / 'internal server error' substrings: a | |
| // token count, a duration ("… took 5003ms") or a key-scoped message naming an | |
| // unavailable model would each condemn a healthy platform for the whole request. | |
| // A message check therefore only runs when there is no status to trust. | |
| export function isProviderLevelError(err: any): boolean { | |
| // A failure the thrower explicitly scoped to the MODEL (`skipModelForRequest` | |
| // — an ignored response_format, invalid tool arguments) is never provider | |
| // health, and its message quotes caller- and model-supplied text: a tool | |
| // named `set_timeout`, or an Ajv complaint about an instance path `/timeout`, | |
| // would otherwise trip the substring checks below and condemn a healthy | |
| // platform for the whole request. The structured marker is authoritative; | |
| // the text is not. | |
| if (err?.skipModelForRequest === true) return false; | |
| const status = typeof err?.status === 'number' ? err.status : 0; | |
| if (status >= 500) return true; | |
| // A DEGRADED hosted deployment (NVIDIA NIM, #522) is provider health wearing | |
| // a 400 — same source of truth as everywhere else, not a second substring. | |
| if (isProviderDegradedError(err)) return true; | |
| if (status !== 0) return false; | |
| const msg = (err?.message ?? '').toLowerCase(); | |
| return isTimeoutErrorText(msg) | |
| || msg.includes('econnrefused') || msg.includes('econnreset') | |
| || msg.includes('fetch failed'); // undici transport error (DNS/TLS/proxy down) | |
| } | |
| // #809: kilo's free relay occasionally answers with a bare classification | |
| // word ("safe" / "unsafe") instead of a real reply — an upstream content | |
| // filter, not the requested model. Such a turn is a dead turn: treat it like | |
| // an empty completion so the fallback loop fails over to the next provider | |
| // instead of surfacing "safe"/"unsafe" as the answer. | |
| // | |
| // Scoped to the relay that actually does this. "safe"/"unsafe" is a LEGITIMATE | |
| // one-word answer for moderation and guard-model workloads, so applying the | |
| // rule everywhere would throw away correct responses and burn the whole | |
| // fallback chain for anyone doing classification. Only platforms observed | |
| // injecting a filter verdict in place of the model's reply belong here. | |
| const CLASSIFICATION_RELAY_PLATFORMS = new Set(['kilo']); | |
| export function isUpstreamClassificationOutput(text: unknown, platform?: string): boolean { | |
| if (!platform || !CLASSIFICATION_RELAY_PLATFORMS.has(platform.toLowerCase())) return false; | |
| const t = (typeof text === 'string' ? text : '').trim().toLowerCase(); | |
| return t === 'safe' || t === 'unsafe'; | |
| } | |
| // Provider-side 400s are retryable because another provider may accept the same | |
| // request shape. If every routed provider rejects it, however, the client should | |
| // see an invalid-request error rather than a misleading rate-limit exhaustion. | |
| export function isProviderBadRequestError(err: any): boolean { | |
| if (isProviderDegradedError(err)) return false; | |
| const status = typeof err?.status === 'number' ? err.status : 0; | |
| const msg = (err?.message ?? '').toLowerCase(); | |
| if (status === 400) return msg.includes('api error 400'); | |
| if (status === 422) return msg.includes('api error 422') || msg.includes('unprocessable entity'); | |
| if (status !== 0) return false; | |
| return msg.includes('api error 400') || msg.includes('api error 422') || msg.includes('unprocessable entity'); | |
| } | |
| // A "this request/prompt is too large" rejection. Providers word it very | |
| // differently and even disagree on the HTTP status: | |
| // - OpenAI-compat: 400 with code `context_length_exceeded` — "This model's | |
| // maximum context length is 8192 tokens. However, your messages resulted | |
| // in 10001 tokens. Please reduce the length of the messages." | |
| // - Anthropic: 400 invalid_request_error — "prompt is too long: 210000 | |
| // tokens > 200000 maximum" (Bedrock words it "Input is too long for | |
| // requested model.") | |
| // - Google/Gemini: 400 INVALID_ARGUMENT — "The input token count (1200000) | |
| // exceeds the maximum number of tokens allowed (1048576)." | |
| // - Groq: a literal HTTP 413 — "Request too large for model `x` ... on | |
| // tokens per minute (TPM): Limit 30000, Requested 33476". Requested > | |
| // Limit can never fit that candidate's window, so it belongs here (too | |
| // large for the candidate), not with transient per-minute 429s. | |
| // - Reverse proxies / gateways: 413 "Payload Too Large" / "request entity | |
| // too large" / "content too large". | |
| // Structured status first (providerHttpError attaches err.status on every | |
| // adapter), then the code field, then message markers — mirroring how the | |
| // other classifiers in this file work. Retryable (a sibling candidate may have | |
| // a larger window), but when EVERY attempt dies this way the exhaustion ladder | |
| // renders an honest 413 instead of a generic failure. | |
| export function isContextTooLargeError(err: any): boolean { | |
| if (err?.status === 413) return true; | |
| if (typeof err?.code === 'string' && err.code.toLowerCase() === 'context_length_exceeded') return true; | |
| const msg = (err?.message ?? '').toLowerCase(); | |
| return msg.includes('context_length_exceeded') | |
| || msg.includes('maximum context length') | |
| || msg.includes('context length exceeded') | |
| || /exceeds[^.]*context (length|window|size)/.test(msg) | |
| || msg.includes('prompt is too long') | |
| || msg.includes('input is too long') | |
| || /input token count[^.]*exceeds/.test(msg) | |
| || msg.includes('exceeds the maximum number of tokens') | |
| || msg.includes('request too large') | |
| || msg.includes('payload too large') | |
| || msg.includes('request entity too large') | |
| || msg.includes('request body too large') | |
| || msg.includes('content too large') | |
| // Zhipu AI (bigmodel.cn) 400, error code 1261: "Prompt exceeds max length" | |
| // (#873). Its wording matches none of the markers above, so without this it | |
| // was mis-bucketed as provider_bad_request instead of context_too_large. | |
| || msg.includes('exceeds max length') | |
| || msg.includes('api error 413'); | |
| } | |
| // A 402 Payment Required / out-of-credits error. Distinct from a transient 429: | |
| // it won't recover on the next window, so the caller benches the model+key with | |
| // PAYMENT_REQUIRED_COOLDOWN_MS (a full day) rather than the 90s transient cooldown. | |
| // | |
| // The digits 402 only count as the STATUS. A bare `includes('402')` also | |
| // matched token counts and request ids ("Limit 30000, Requested 34026", | |
| // "14023 tokens used"), and since the 402 bench covers the key on every model | |
| // of the platform for a day (#1239), one unlucky number took a whole provider | |
| // out. So: an error that states another status is never a 402 by its digits, | |
| // and otherwise 402 has to stand alone rather than sit inside a longer number. | |
| const STATED_STATUS = /\bapi error (\d{3})\b|\(http (\d{3})\)/; | |
| const STANDALONE_402 = /(?<![\w.])402(?![\w.])/; | |
| export function isPaymentRequiredError(err: any): boolean { | |
| const msg = String(err?.message ?? '').toLowerCase(); | |
| if (msg.includes('payment required') | |
| || msg.includes('insufficient_quota') || msg.includes('insufficient credit') | |
| || msg.includes('insufficient balance')) return true; | |
| const stated = msg.match(STATED_STATUS); | |
| const status = typeof err?.status === 'number' ? err.status : Number(stated?.[1] ?? stated?.[2]); | |
| if (Number.isFinite(status)) return status === 402; | |
| return STANDALONE_402.test(msg); | |
| } | |
| // "model 'x' does not exist" / "model \"x\" does not exist" / "model x does not | |
| // exist" — one non-space token (optionally quoted) between the word "model" and | |
| // the verdict. Bounded so an unrelated sentence containing both words never | |
| // matches; the model id itself is never inspected. | |
| const MODEL_ID_DOES_NOT_EXIST = /\bmodel\b\s+['"`]?[^\s'"`]{1,200}['"`]?\s+(?:does\s+not|doesn't)\s+exist\b/; | |
| // A 404 "model removed/deprecated upstream" error. It's a MODEL-level failure, | |
| // not a key-level one: every key for the platform will 404 the same way, so the | |
| // retry loop skips the entire model for the rest of the request instead of | |
| // burning one fallback attempt per key on the same dead route. | |
| // (PR #111, credits @barbotkonv.) | |
| export function isModelNotFoundError(err: any): boolean { | |
| // 404 (removed/deprecated) and 410 (permanently Gone) are both MODEL-level: every | |
| // key for the platform fails the same way, so skip the whole model for the rest | |
| // of the request instead of burning one fallback attempt per sibling key. 410 | |
| // added for #339 (Ollama Cloud "Gone"); prefer the structured status when present. | |
| if (err?.status === 404 || err?.status === 410) return true; | |
| const msg = (err?.message ?? '').toLowerCase(); | |
| return msg.includes('404') || msg.includes('not found') || msg.includes('no endpoints found') | |
| || msg.includes('410') || msg.includes('gone') | |
| // Some aggregators report a removed/stale model with a 400 (not a 404) whose | |
| // body reads "No model found: <id>", "model not found", "unknown model" or | |
| // "model does not exist" (#: Routeway 400 "No model found: llama-3.3-70b-instruct:free"). | |
| // Note "No model found" does NOT contain the substring "not found" (words are | |
| // no/model/found), so it slipped past the checks above and fell through to | |
| // isProviderBadRequestError — surfacing as a request-blaming 400 instead of a | |
| // stale-catalog 404. These phrasings are MODEL-level (every sibling key fails | |
| // identically), so they belong here for the whole-model skip. | |
| || msg.includes('no model found') || msg.includes('model not found') | |
| || msg.includes('unknown model') || msg.includes('model does not exist') | |
| || msg.includes('no such model') | |
| // The same verdict with the model id quoted in the middle (#1239: NavyAI | |
| // 400 "The model 'o3-mini' does not exist or is not supported for chat | |
| // completions."). The bare "model does not exist" substring above never | |
| // matches that wording, so every one of a platform's stale rows was booked | |
| // as provider_bad_request — no whole-model skip, a hop burned per dead | |
| // model, and the exhaustion body blamed the caller's request. | |
| || MODEL_ID_DOES_NOT_EXIST.test(msg) | |
| // "not supported for chat completions" is MODEL-level too: the id is real | |
| // but this platform cannot serve it on the endpoint we use, and a sibling | |
| // key would be told the same. Route it out for the request like a 404. | |
| || msg.includes('not supported for chat completions'); | |
| } | |
| // A 403 that suspends the ACCOUNT, not one model: NavyAI answers every model | |
| // behind a benched free key with "The Free plan is temporarily disabled due to | |
| // abuse. You can purchase a plan ...". Every model of the platform fails the | |
| // same way, so classifying it as model-forbidden benched ONE model per attempt: | |
| // with 93 catalog rows on that platform, each request burned its whole failover | |
| // budget re-discovering the same dead account. Not key-auth either: the | |
| // credential itself is valid, the plan behind it is not, and validateKey's | |
| // /models probe still passes, so the health checker never demotes the key. | |
| // Status-gated to 403 (or a status-less message that names one) so provider | |
| // wording alone can never condemn a healthy key. | |
| export function isAccountSuspendedError(err: any): boolean { | |
| const status = typeof err?.status === 'number' ? err.status : 0; | |
| const msg = (err?.message ?? '').toLowerCase(); | |
| if (status !== 403 && !(status === 0 && msg.includes('403'))) return false; | |
| return /\b(plan|account|subscription) (is|has been|was) (temporarily )?(disabled|suspended|banned|deactivated)\b/.test(msg) | |
| || msg.includes('due to abuse'); | |
| } | |
| // A 403 Forbidden returned for a specific model behind an otherwise-valid key. | |
| // Drives the same whole-model skip as a 404: every key on this platform's tier | |
| // would be forbidden the same model, so rule it out for the rest of the request | |
| // rather than trying it again with a sibling key. Distinct from a dead key — | |
| // validateKey returns false on 401/403, so the health checker disables genuinely | |
| // forbidden keys; a 403 reaching here is model-not-on-this-tier. See issue #256. | |
| export function isModelAccessForbiddenError(err: any): boolean { | |
| if (err?.status === 403) return true; | |
| const msg = (err?.message ?? '').toLowerCase(); | |
| if (msg.includes('403') || msg.includes('forbidden')) return true; | |
| // Not every provider spells "this key may not use this model" as a 403 (issue | |
| // #618): observed live as a 400 whose body reads "user is not allowed to | |
| // access model kat-coder-pro-v2.5", and as plan-gate wordings like "action | |
| // plan limited". Classified transient, those took the 90s bench — and the | |
| // auto-router re-picked the same permanently-unreachable model the moment it | |
| // expired, failing and re-benching forever with an ever-growing penalty. | |
| // Deliberately narrow: only 400/401 (or a status-less error), and only for | |
| // phrasings that name access/permission to the MODEL. A bare "Bad Request", | |
| // a generic "Unauthorized" (which isKeyAuthError owns), or a parameter | |
| // rejection must keep their current classification. | |
| const status = typeof err?.status === 'number' ? err.status : 0; | |
| if (status !== 0 && status !== 400 && status !== 401) return false; | |
| return MODEL_ACCESS_DENIED_PHRASES.some(phrase => msg.includes(phrase)); | |
| } | |
| const MODEL_ACCESS_DENIED_PHRASES = [ | |
| 'not allowed to access', | |
| 'not allowed to use', | |
| 'not authorized to access', | |
| 'not authorized to use', | |
| 'unauthorized to access', | |
| 'not permitted to access', | |
| 'not permitted to use', | |
| 'do not have access to', | |
| 'does not have access to', | |
| 'no access to model', | |
| 'no access to this model', | |
| 'model access denied', | |
| 'access to this model is restricted', | |
| // Plan/subscription gates the provider words as a limit rather than a denial. | |
| 'action plan limited', | |
| ]; | |
| // ── Permanent upstream retirement (issue #634) ─────────────────────────────── | |
| // isModelNotFoundError above rules a 404/410 model out for ONE request. That is | |
| // right for a load balancer hiccup and wrong for a model the provider RETIRED: | |
| // NVIDIA's "The model 'minimaxai/minimax-m2.7' has reached its end of life on | |
| // 2026-07-27T00:00:00Z and is no longer available" is true forever, yet nothing | |
| // persisted it, so the corpse burned a fallback slot on every later request. | |
| // | |
| // This classifier says how much a failure should be TRUSTED as permanent: | |
| // - 'definitive' — an explicit 410 Gone, or unmistakable end-of-life wording. | |
| // Acting on one response is safe. | |
| // - 'probable' — a 404 whose body says the model is gone in so many words. | |
| // Real, but a provider can also word a transient outage this way, so the | |
| // caller corroborates across distinct requests before acting. | |
| // - null — everything else, including every bare 404 ("Not found", | |
| // "Provider returned error", OpenRouter's "No endpoints found", NVIDIA's | |
| // per-account "Function '<uuid>': Not found for account '<id>'"). A | |
| // transient 404 must NEVER retire a healthy model. | |
| // Note the 410 test is deliberately stricter than isModelNotFoundError's bare | |
| // `msg.includes('410')`: a digit run inside a token count must not read as Gone. | |
| export type ModelRetirementConfidence = 'definitive' | 'probable'; | |
| const END_OF_LIFE_PHRASES = [ | |
| 'end of life', | |
| 'end-of-life', | |
| 'has been retired', | |
| 'has been decommissioned', | |
| 'has been sunset', | |
| ]; | |
| const MODEL_GONE_PHRASES = [ | |
| 'no longer available', | |
| 'no longer offered', | |
| 'no longer supported', | |
| 'has been removed', | |
| 'was removed', | |
| 'has been discontinued', | |
| 'has been deprecated', | |
| 'is deprecated', | |
| ]; | |
| export function modelRetirementSignal(err: any): ModelRetirementConfidence | null { | |
| const msg = (err?.message ?? '').toLowerCase(); | |
| const status = typeof err?.status === 'number' ? err.status : 0; | |
| const gone = status === 410 || /\berror 410\b/.test(msg) || /\b410 gone\b/.test(msg); | |
| // An end-of-life phrase is only definitive when the status agrees the model | |
| // is not there. On its own it is just text, and 'definitive' skips the | |
| // corroboration gate in noteModelRetirementSignal — so a 429 or 500 whose | |
| // body happens to mention a retirement (an advisory notice, a status-page | |
| // quote, a message about a DIFFERENT model) disabled a live model on one | |
| // response. Unmatched here means it falls through and teaches nothing. | |
| if (gone || (isModelNotFoundError(err) && END_OF_LIFE_PHRASES.some(phrase => msg.includes(phrase)))) return 'definitive'; | |
| if (!isModelNotFoundError(err)) return null; | |
| return MODEL_GONE_PHRASES.some(phrase => msg.includes(phrase)) ? 'probable' : null; | |
| } | |
| // A stream that ended without its terminal marker (`[DONE]` and/or a | |
| // finish_reason): the upstream connection was reset or the response truncated | |
| // mid-generation. Observed live on Kilo Gateway (5 attempts in one session, | |
| // #1218): the gateway answers 200, streams content, then dies without | |
| // `data: [DONE]` — `readSseStream` throws | |
| // "…stream ended unexpectedly (no [DONE], no finish_reason)". | |
| // | |
| // The truncation is UPSTREAM transport, not request shape, so the plain | |
| // retryable path (a fresh attempt on a sibling key, or even the same key | |
| // later) is the right response — but a stream that dies with zero content | |
| // deltas is usually the ROUTE (platform+model+key edge) that is sick, not | |
| // bad luck: three empty-ended truncations in a row on the same route | |
| // reliably precede another one. Callers use this signal to bench the route | |
| // after a short streak instead of re-paying the round trip every request. | |
| export function isStreamTruncatedError(err: any): boolean { | |
| const msg = (err?.message ?? '').toLowerCase(); | |
| return msg.includes('stream ended unexpectedly') | |
| || msg.includes('no [done], no finish_reason') | |
| // undici surfaces an abrupt RST as "terminated" on the body read; with | |
| // our SSE reader the top-level message is "terminated" and the real | |
| // cause is buried in err.cause (see isTransportError). Count it too: | |
| // a terminated mid-stream body is the same dead route either way. | |
| || msg === 'terminated'; | |
| } | |