File size: 10,933 Bytes
c61f7ed | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 151 152 153 154 155 156 157 158 159 160 161 162 163 164 165 166 167 168 169 170 171 172 173 174 175 176 177 178 179 180 181 182 183 184 185 186 187 188 189 190 191 192 193 194 195 196 197 198 199 200 201 202 203 204 205 206 207 208 209 210 211 212 213 214 215 216 217 218 219 220 221 222 223 224 225 226 | """Provider API error summarising for ``AIAgent``.
Entitlement-failure detection, xAI subscription decoration, structured-detail coercion, and log-safe
redaction of provider error payloads.
Extracted from ``run_agent.py``; every method resolves through ``AIAgent``'s MRO unchanged.
"""
import json
import re
from typing import Any, Dict, Optional
from agent.redact import redact_sensitive_text
# Substrings of the plain ``ValueError`` the Anthropic SDK raises for a malformed event-stream
# frame (wire trouble, not local validation). Read by ``AIAgent._is_provider_stream_parse_error``.
PROVIDER_STREAM_PARSE_MARKERS = ("expected ident at line", "expected value at line")
# Offline DNS failures are wrapped in a generic "Connection error" by SDKs β inspect the chain.
_NETWORK_RESOLUTION_MARKERS = (
"temporary failure in name resolution",
"name or service not known",
"nodename nor servname provided, or not known",
"getaddrinfo failed",
"no address associated with hostname",
"network is unreachable",
)
_XAI_ENTITLEMENT_HINT = (
" β xAI rejected this OAuth account. NOTE: X Premium+ does NOT "
"include xAI API access β only standalone SuperGrok subscribers "
"can use this provider. Other possible causes: no Grok "
"subscription, your tier doesn't include this model, or your "
"quota is exhausted. Check https://grok.com/?_s=usage to see "
"which, or run `/model` to switch providers."
)
_ERROR_DETAIL_KEYS = ("message", "detail", "error", "code", "type")
def _is_xai_entitlement_text(lower: str) -> bool:
"""xAI's permission-denied body text for an unsubscribed / under-tiered / exhausted account."""
return (
"do not have an active grok subscription" in lower
or ("out of available resources" in lower and "grok" in lower)
or ("does not have permission" in lower and "grok" in lower)
)
def _http_prefix(error: Exception) -> str:
status_code = getattr(error, "status_code", None)
return f"HTTP {status_code}: " if status_code else ""
class ApiErrorSummaryMixin:
"""Provider error -> user/log-safe summary (see module docstring)."""
@staticmethod
def _is_entitlement_failure(
error_context: Optional[Dict[str, Any]], status_code: Optional[int]
) -> bool:
"""Detect subscription/entitlement 401/403s that masquerade as auth failures.
Refreshing a token cannot fix an unsubscribed account, so callers surface the error instead of looping
the pool. xAI returns the same permission-denied text for BOTH cases; a ``[WKE=unauthenticated:...]``
suffix (or "access token could not be validated") means stale token β return False so the refresh path
runs.
Disambiguator for xAI (#29344): the same ``code`` text ("The caller does not have permission to
execute the specified operation") is returned for BOTH an unsubscribed account AND a stale OAuth
access token. xAI ships an explicit signal in the ``error`` field that tells the two apart: a
``[WKE=unauthenticated:...]`` suffix (and/or the ``OAuth2 access token could not be validated``
phrasing) means the credentials failed validation β that's recoverable by refreshing the token, NOT
by surfacing an entitlement message. When either signal is present we return False eagerly so the
credential-pool refresh path runs, letting long-running TUI sessions recover from stale tokens
without an exit/reopen cycle.
"""
if status_code not in {401, 403, None}:
return False
if not isinstance(error_context, dict):
return False
# Single lowercase haystack over every field shape (message/reason and raw code/error).
haystack = " ".join(
str(error_context.get(k) or "").lower() for k in ("message", "reason", "code", "error")
)
if not haystack.strip():
return False
if "[wke=unauthenticated:" in haystack or "oauth2 access token could not be validated" in haystack:
return False
return _is_xai_entitlement_text(haystack)
@staticmethod
def _decorate_xai_entitlement_error(detail: str) -> str:
"""Append a neutral hint when xAI's OAuth surface returns the permission-denied 403.
xAI's ``/v1/responses`` uses one body for several causes (no subscription, tier lacks the model, quota
exhausted). The least obvious: X Premium+ does NOT include API access β only SuperGrok does. Lead with
that, keep the raw text, point at https://grok.com/?_s=usage. Idempotent: a substring unique to the
hint marks prior decoration.
"""
if not detail or not _is_xai_entitlement_text(detail.lower()):
return detail
if "X Premium+ does NOT include" in detail:
return detail
return f"{detail}{_XAI_ENTITLEMENT_HINT}"
@staticmethod
def _coerce_api_error_detail(value: Any) -> str:
"""Return a display-safe string for structured provider error fields."""
if isinstance(value, str):
return value
if isinstance(value, dict):
for key in _ERROR_DETAIL_KEYS:
nested = value.get(key)
if isinstance(nested, str) and nested.strip():
return nested
for key in _ERROR_DETAIL_KEYS:
if key in value:
nested_detail = ApiErrorSummaryMixin._coerce_api_error_detail(value[key])
if nested_detail:
return nested_detail
try:
return json.dumps(value, ensure_ascii=False, sort_keys=True)
except TypeError:
return str(value)
if isinstance(value, (list, tuple)):
parts = [ApiErrorSummaryMixin._coerce_api_error_detail(item) for item in value]
return "; ".join(part for part in parts if part)
if value is None:
return ""
return str(value)
@staticmethod
def _summarize_api_error(error: Exception) -> str:
"""Extract a human-readable one-liner from an API error.
Cloudflare HTML pages β ``<title>``; network/DNS failures (even SDK-wrapped) β offline hint; else
truncated str(error).
"""
raw = str(error)
current: Optional[BaseException] = error
seen: set[int] = set()
while current is not None and id(current) not in seen:
seen.add(id(current))
if any(marker in str(current).lower() for marker in _NETWORK_RESOLUTION_MARKERS):
return (
"Hermes can't reach the model provider. You may be offline. "
"Check your internet connection and try again."
)
current = current.__cause__ or current.__context__
if isinstance(error, ValueError) and any(marker in raw.lower() for marker in PROVIDER_STREAM_PARSE_MARKERS):
return f"Malformed provider streaming response: {raw[:300]}"
prefix = _http_prefix(error)
# Cloudflare / proxy HTML pages: grab the <title> (and Ray ID) for a clean summary
if "<!DOCTYPE" in raw or "<html" in raw:
m = re.search(r"<title[^>]*>([^<]+)</title>", raw, re.IGNORECASE)
title = m.group(1).strip() if m else "HTML error page (title not found)"
ray = re.search(r"Cloudflare Ray ID:\s*<strong[^>]*>([^<]+)</strong>", raw)
parts = [prefix[:-2]] if prefix else []
parts.append(title)
if ray:
parts.append(f"Ray {ray.group(1).strip()}")
return " β ".join(parts)
# GeminiAPIError already composes a clean one-liner with guidance; don't re-extract the raw body.
if type(error).__name__ == "GeminiAPIError":
return redact_sensitive_text(raw[:1000])
# JSON body errors from OpenAI/Anthropic SDKs
body = getattr(error, "body", None)
if isinstance(body, dict):
msg = body.get("error", {}).get("message") if isinstance(body.get("error"), dict) else body.get("message")
if msg:
msg = ApiErrorSummaryMixin._coerce_api_error_detail(msg)
return ApiErrorSummaryMixin._decorate_xai_entitlement_error(f"{prefix}{msg[:300]}")
# SDK may leave body empty while httpx has the payload. Redact: the body is attacker-influenced
# and may echo Authorization / x-api-key / request JSON.
# Redact before returning: the raw provider/proxy error body is attacker-influenced and may echo
# Authorization / x-api-key / request JSON, which would otherwise leak into final_response + logs
# (this path widens exposure vs the old empty-body "HTTP 400" string). See #36109.
response = getattr(error, "response", None)
if response is not None:
try:
snippet = (getattr(response, "text", None) or "").strip()
except Exception:
snippet = ""
if snippet:
try:
payload = json.loads(snippet)
except (json.JSONDecodeError, TypeError):
payload = None
if isinstance(payload, dict):
err = payload.get("error")
if isinstance(err, dict) and err.get("message"):
return redact_sensitive_text(f"{prefix}{str(err['message'])[:300]}")
if payload.get("message"):
return redact_sensitive_text(f"{prefix}{str(payload['message'])[:300]}")
return redact_sensitive_text(f"{prefix}{snippet[:300]}")
# Fallback: truncate the raw string but give more room than 200 chars
return ApiErrorSummaryMixin._decorate_xai_entitlement_error(f"{prefix}{raw[:500]}")
def _mask_api_key_for_logs(self, key: Any) -> Optional[str]:
# Azure Foundry Entra ID bearer providers are callables β never invoke them in log
# paths; identify the auth surface instead.
if callable(key) and not isinstance(key, str):
return "<entra-id-bearer>"
if not key:
return None
if len(key) <= 12:
return "***"
return f"{key[:8]}...{key[-4:]}"
def _clean_error_message(self, error_msg: str) -> str:
"""Clean up error messages for user display, removing HTML content and truncating."""
if not error_msg:
return "Unknown error"
# HTML content is common with CloudFlare and gateway error pages
if error_msg.strip().startswith('<!DOCTYPE html') or '<html' in error_msg:
return "Service temporarily unavailable (HTML error page returned)"
cleaned = ' '.join(error_msg.split())
if len(cleaned) > 150:
cleaned = cleaned[:150] + "..."
return cleaned
|