Spaces:
Sleeping
Sleeping
"""Central LLM client.
This is the ONLY module that talks to Anthropic, OpenAI, Groq, or Gemini
directly (Groq for text only). All other services route through
:func:`call_text_llm` or :func:`call_vision_llm` so:
* every API call is captured in :data:`st.session_state.api_usage_log`
with a stable ``task_name``,
* the UI can show a single "API Usage Status" panel that knows which
stages really used the API,
* the API key is never returned, logged, or echoed.
Return value
------------
Every call returns an :class:`LLMCallResult` with::
success: bool
task_name: str
provider: str | None
model: str | None
used_api: bool
response_text: str | None
response_json: dict | list | None
error_message: str | None
timestamp: str # ISO 8601, UTC
estimated_input_tokens: int | None
estimated_output_tokens: int | None
status: str # "ok" | "no_api" | "invalid_key" | "access_denied"
# | "quota" | "connection" | "model_missing"
# | "parse_error" | "unsupported_provider"
# | "skipped" | "exception"
request_id: str | None # provider-side call id (req_…), for support
http_status: int | None # exact HTTP status from the SDK exception
error_type: str | None # provider's machine-readable error code
``used_api`` is true only when bytes actually went over the wire (or were
attempted). Callers that fall back to local logic set it to ``False`` via
:func:`record_local_use`.
Every finished call is mirrored to the durable diagnostics log (see
:mod:`app.services.diagnostics`) so a failure survives the page reload that
usually follows it and can be handed to the app team as one redacted report.
"""
from __future__ import annotations
import base64
import json
import logging
import os
import re
import threading
import time
from dataclasses import asdict, dataclass, field
from datetime import datetime, timezone
from typing import Any, Iterable, Optional
from app.config import (
GROQ_VISION_UNSUPPORTED_MESSAGE,
INTERVIEW_PROVIDER,
LLM_TASK_NAMES,
Settings,
get_settings,
)
from app.utils.logging_utils import get_logger
from app.utils.redaction import (
API_KEY_PATTERNS as _API_KEY_PATTERNS,
ORG_ID_PATTERNS as _ORG_ID_PATTERNS,
redact as _redact,
)
_logger: logging.Logger = get_logger("upwork_strategist.llm_client")
# Stable status codes — surfaced verbatim by the UI / api_gate.
STATUS_OK = "ok"
STATUS_NO_API = "no_api"
STATUS_INVALID_KEY = "invalid_key"
# HTTP 403: the key works but may not use this model (or this service) — a
# plan, billing or organisation-verification gate, not a mistyped key.
STATUS_ACCESS_DENIED = "access_denied"
STATUS_QUOTA = "quota"
STATUS_CONTEXT_OVERFLOW = "context_overflow"
STATUS_CONNECTION = "connection"
STATUS_MODEL_MISSING = "model_missing"
STATUS_PARSE_ERROR = "parse_error"
STATUS_TRUNCATED = "truncated"
STATUS_UNSUPPORTED_PROVIDER = "unsupported_provider"
STATUS_SKIPPED = "skipped"
STATUS_EXCEPTION = "exception"
_SUPPORTED_PROVIDERS = ("anthropic", "openai", "groq", "gemini")
# Providers that can read images. Groq is text-only here, so it is excluded
# and handled with a dedicated, actionable message in ``call_vision_llm``.
_SUPPORTED_VISION_PROVIDERS = ("anthropic", "openai", "gemini")
# Per-request HTTP timeout (seconds) applied to every provider client so a
# hung socket cannot freeze the synchronous Streamlit call indefinitely.
# Overridable via the LLM_REQUEST_TIMEOUT env var.
def _request_timeout() -> float:
raw = os.getenv("LLM_REQUEST_TIMEOUT", "").strip()
try:
value = float(raw)
if value > 0:
return value
except (TypeError, ValueError):
pass
return 60.0
# Bounded transient-failure retry. Only STATUS_CONNECTION and genuine
# rate-limit (STATUS_QUOTA) are retried; auth/model/parse/context errors
# are never retried (retrying cannot help and would waste calls/money).
_MAX_ATTEMPTS = 3
_RETRYABLE_STATUSES = (STATUS_CONNECTION, STATUS_QUOTA)
# Backoff base in seconds (exponential: base, base*2, …). Kept small so the
# synchronous UI is never blocked for long.
_RETRY_BACKOFF_BASE = 0.5
# This loop is the ONLY retry layer: the SDK clients are built with
# max_retries=0 and Gemini calls pass retry=None, because the SDKs' own
# retries stacked under it turned one call into up to 9 HTTP requests (and
# minutes of a frozen screen on a hung provider). A retry is started only if
# it can finish within the request timeout plus this many seconds of the
# first attempt, so a hung provider costs one timeout, not three.
_RETRY_WINDOW = 30.0
# A provider's Retry-After is honoured up to this many seconds. Asked to wait
# longer, the call is not retried at all and fails at once.
_RETRY_AFTER_CAP = 10.0
# Finish/stop reasons that mean the model was cut off by the token budget.
_TRUNCATION_REASONS = frozenset(
{"length", "max_tokens", "max_output_tokens"}
)
# ---------------------------------------------------------------------------
# Result type
# ---------------------------------------------------------------------------
@dataclass
class LLMCallResult:
success: bool
task_name: str
provider: Optional[str] = None
model: Optional[str] = None
used_api: bool = False
response_text: Optional[str] = None
response_json: Any = None
error_message: Optional[str] = None
timestamp: str = field(
default_factory=lambda: datetime.now(timezone.utc).isoformat(timespec="seconds")
)
estimated_input_tokens: Optional[int] = None
estimated_output_tokens: Optional[int] = None
status: str = STATUS_OK
duration_seconds: Optional[float] = None
# Provider-side correlation fields. ``request_id`` is the single most
# useful value when escalating to a provider's support (Anthropic and
# OpenAI both return one per call); ``http_status`` and ``error_type``
# come straight from the SDK exception instead of being guessed from
# the message text, so "wrong key" (401) and "wrong model" (404) are
# distinguishable with certainty.
request_id: Optional[str] = None
http_status: Optional[int] = None
error_type: Optional[str] = None
def to_log_entry(self) -> dict:
"""Return a dict that is safe to display in the UI / usage log."""
return {
"timestamp": self.timestamp,
"task_name": self.task_name,
"provider": self.provider,
"model": self.model,
"used_api": self.used_api,
"status": self.status,
"error_message": self.error_message,
"duration_seconds": self.duration_seconds,
"estimated_input_tokens": self.estimated_input_tokens,
"estimated_output_tokens": self.estimated_output_tokens,
"request_id": self.request_id,
"http_status": self.http_status,
"error_type": self.error_type,
# response_text / response_json are intentionally NOT stored
# in the usage log — that log is metadata-only.
}
# ---------------------------------------------------------------------------
# Session-state usage log
# ---------------------------------------------------------------------------
# A worker thread (see :mod:`app.services.background_tasks`) has no Streamlit
# session: ``st.session_state`` there is a throwaway object, so whatever a call
# logs to it is lost. The worker installs a list of its own here instead; the
# list goes back to the main thread with the result, which adds it to the
# session's log with :func:`add_to_usage_log`.
_thread_log = threading.local()
def set_thread_usage_log(entries: Optional[list]) -> Optional[list]:
"""Log this thread's calls to ``entries`` instead of the session (``None`` stops).
Returns the list it replaces, so the caller can put it back.
"""
previous = getattr(_thread_log, "entries", None)
_thread_log.entries = entries
return previous
def _usage_log() -> Optional[list]:
"""The usage log this thread records to, or ``None`` when there is none.
The list a worker installed with :func:`set_thread_usage_log` if any,
otherwise ``st.session_state.api_usage_log`` (created when missing).
"""
entries = getattr(_thread_log, "entries", None)
if entries is not None:
return entries
try:
import streamlit as st # type: ignore
except Exception: # pragma: no cover - streamlit not always loaded
return None
try:
log = st.session_state.get("api_usage_log")
if log is None:
log = []
st.session_state["api_usage_log"] = log
return log
except Exception: # pragma: no cover - not running inside a session
return None
def _record_to_session(entry: dict) -> None:
"""Append ``entry`` to ``st.session_state.api_usage_log`` if available.
On a worker thread it goes to the worker's own list instead.
"""
log = _usage_log()
if log is not None:
log.append(entry)
def add_to_usage_log(entries: Iterable[dict]) -> None:
"""Add the entries a worker thread logged to this session's usage log.
Call it on the main thread, when the worker's result is collected.
"""
entries = list(entries or ())
if not entries:
return
log = _usage_log()
if log is not None:
log.extend(entries)
def _record_to_disk(entry: dict) -> None:
"""Mirror ``entry`` into the durable diagnostics log.
Imported lazily — same pattern as the streamlit import above — so this
module keeps working in a bare unit-test process where the diagnostics
service (and its config-dependent log directory) may not be importable.
"""
try:
from app.services import diagnostics
diagnostics.record_event(entry)
except Exception: # pragma: no cover - diagnostics must never break a call
return
def record_local_use(task_name: str, *, note: str = "") -> None:
"""Record a non-API stage so the UI can show LOCAL PLACEHOLDER clearly.
Callers that intentionally use deterministic local logic must call
this so the API Usage panel can flag the stage.
"""
entry = {
"timestamp": datetime.now(timezone.utc).isoformat(timespec="seconds"),
"task_name": task_name,
"provider": None,
"model": None,
"used_api": False,
"status": "local_placeholder",
"error_message": note or None,
"duration_seconds": None,
"estimated_input_tokens": None,
"estimated_output_tokens": None,
# Keep the record shape identical to to_log_entry() so a report
# renders one uniform table instead of a ragged one.
"request_id": None,
"http_status": None,
"error_type": None,
}
_record_to_session(entry)
# Persisted too: knowing a stage fell back to local logic is part of the
# sequence support needs, even though it is not itself an error.
_record_to_disk(entry)
_logger.info(
"stage=%s used_api=false status=local_placeholder note=%s",
task_name,
note or "-",
)
def get_usage_log() -> list[dict]:
"""Return the current session's usage log (or an empty list)."""
try:
import streamlit as st # type: ignore
except Exception: # pragma: no cover
return []
try:
return list(st.session_state.get("api_usage_log") or [])
except Exception: # pragma: no cover
return []
def reset_usage_log() -> None:
try:
import streamlit as st # type: ignore
except Exception: # pragma: no cover
return
try:
st.session_state["api_usage_log"] = []
except Exception: # pragma: no cover
return
# ---------------------------------------------------------------------------
# Helpers
# ---------------------------------------------------------------------------
def _validate_task_name(task_name: str) -> None:
if task_name not in LLM_TASK_NAMES:
_logger.warning("unknown llm task_name=%s", task_name)
def _estimate_tokens(text: str) -> int:
if not text:
return 0
# Rough heuristic: ~4 characters per token. Cheap, never exact.
return max(1, len(text) // 4)
def _resolve_key(settings: Settings, provider: Optional[str] = None) -> Optional[str]:
target = (provider or settings.llm_provider or "").strip().lower()
return {
"anthropic": settings.anthropic_api_key,
"openai": settings.openai_api_key,
"groq": settings.groq_api_key,
"gemini": settings.gemini_api_key,
}.get(target)
def _resolve_model(
settings: Settings, *, vision: bool = False, provider: Optional[str] = None
) -> Optional[str]:
"""Model for ``provider``, or for the configured provider when None.
``provider`` must be passed whenever the caller overrode the provider:
resolving the model from ``settings.llm_provider`` while the request
goes elsewhere sends one provider's model name to another provider's
endpoint, which comes back as a model-not-found error.
"""
if vision:
# Vision model resolution (VISION_MODEL, or the text model when it is
# itself vision-capable) lives on Settings so the rule is shared.
return settings.active_vision_model
target = (provider or settings.llm_provider or "").strip().lower()
return {
"anthropic": settings.anthropic_model,
"openai": settings.openai_model,
"groq": settings.groq_model,
"gemini": settings.gemini_model,
}.get(target)
def _resolve_provider(
settings: Settings, *, vision: bool = False, provider: Optional[str] = None
) -> str:
"""Provider for this call — the override when given, else the configured one.
Normalised with ``strip().lower()`` (matching ``Settings.active_provider``)
so an override carrying stray whitespace still matches
:data:`_SUPPORTED_PROVIDERS` instead of being reported as unconfigured.
"""
if vision:
return settings.active_vision_provider
return (provider or settings.llm_provider or "").strip().lower()
_BRACKET_RE = re.compile(r"[{\[]")
# An opening bracket that starts a JSON value, as opposed to a stray "{" or
# "[" in prose ("use the { key", "[see below").
_JSON_OPENER_RE = re.compile(r'\{\s*["}]|\[\s*(?:["{\[\]\-\d]|true\b|false\b|null\b)')
def _bracket_span_end(text: str, start: int) -> Optional[int]:
"""Index just past the bracket opened at ``start``, or None if it never closes.
Counts ``{``/``[`` against ``}``/``]`` and skips anything inside JSON
strings (honouring backslash escapes), but does not otherwise validate,
so it can also step over a bracketed bit of prose such as ``{name}``.
"""
depth = 0
in_string = False
escaped = False
for pos in range(start, len(text)):
ch = text[pos]
if in_string:
if escaped:
escaped = False
elif ch == "\\":
escaped = True
elif ch == '"':
in_string = False
elif ch == '"':
in_string = True
elif ch in "{[":
depth += 1
elif ch in "}]":
depth -= 1
if depth == 0:
return pos + 1
return None
def _find_json_span(text: str) -> Optional[Any]:
"""Return the outermost complete JSON value embedded in ``text``.
Only top-level bracketed spans are tried: the scanner steps over each
whole span, whether it decodes or not, so a value nested inside a broken
or cut-off object is never returned in its place. (Trying every brace
turned a reply cut off by the token limit into its first complete inner
object, reported as success — a cut "failed" fact-check read as
"passed".) JSON that opens but never closes means the reply was cut off
or is broken inside it, so the result is ``None``. Of the complete values
found, the largest wins, so a short bracket in prose such as
``evidence [1]`` cannot stand in for the payload. This replaces the old
greedy ``(\\{.*\\}|\\[.*\\])`` regex, which spanned from the first brace
to the LAST one, so prose containing a stray brace, or two objects,
would defeat it.
"""
best: Optional[Any] = None
best_len = 0
pos = 0
while True:
match = _BRACKET_RE.search(text, pos)
if match is None:
return best
start = match.start()
end = _bracket_span_end(text, start)
if end is None:
if _JSON_OPENER_RE.match(text, start):
return None
pos = start + 1 # a stray bracket in prose
continue
try:
value = json.loads(text[start:end])
except ValueError:
pass
else:
if end - start > best_len:
best, best_len = value, end - start
pos = end
def _parse_json(text: str) -> Optional[Any]:
if not text:
return None
candidate = text.strip()
candidate = re.sub(r"^```(?:json)?", "", candidate).strip()
candidate = re.sub(r"```$", "", candidate).strip()
try:
return json.loads(candidate)
except ValueError:
pass
# Fallback: locate the outermost complete JSON value in surrounding prose
# rather than greedily spanning to the last brace in the string.
return _find_json_span(candidate)
def _data_url(data: bytes, mime: str) -> str:
return f"data:{mime};base64," + base64.b64encode(data).decode("ascii")
# Message wording for the two "too big / too fast" conditions. Named so the
# HTTP-429 branch and the substring fallback in :func:`_classify_exception`
# read from one list each and cannot drift apart.
# Wording that says THIS REQUEST is itself too big to ever be accepted — as
# opposed to "you are sending them too quickly". Groq's most common free-tier
# 429 is of this kind: "Request too large for model ... on tokens per minute
# (TPM): Limit 12000, Requested 13000, please reduce your message size". The
# fix is to send less input, so it must never be classified as a retryable
# rate limit. Checked before the rate-limit wording because such a body also
# mentions "tokens per minute".
_OVERSIZED_REQUEST_PHRASES: tuple[str, ...] = (
"request too large",
"reduce your message size",
"maximum context length",
"context window",
)
_CONTEXT_OVERFLOW_PHRASES: tuple[str, ...] = _OVERSIZED_REQUEST_PHRASES + (
"context length",
"context_length",
"maximum context",
"too many tokens",
"token limit",
"tokens exceed",
"reduce the length",
)
# The bare status "429" is deliberately NOT in this tuple: SDKs prefix the
# message with "Error code: 429 - ...", so it would match every 429 body,
# including one that is really a context-window overflow. The fallback in
# :func:`_classify_exception` still checks it, for exceptions that carry no
# usable ``http_status``.
_RATE_LIMIT_PHRASES: tuple[str, ...] = (
"quota",
"rate limit",
"rate_limit",
"insufficient",
"tokens per minute",
"tokens-per-minute",
"tpm",
)
# Wording that says the account has no credit left. Anthropic answers a key
# with an empty balance with a plain HTTP 400 ("Your credit balance is too
# low to access the Anthropic API ... purchase credits"), so no status code
# marks it as billing. Also never retried: waiting does not add credit.
_NO_CREDIT_PHRASES: tuple[str, ...] = (
"credit balance",
"purchase credits",
"billing",
"insufficient_quota",
"exceeded your current quota",
)
# Rate-limit wording for an allowance that only resets the next day (Groq's
# "tokens per day (TPD)", Gemini's "...PerDay..." quota). Still a quota
# answer, but never retried: a wait of seconds cannot help.
_DAILY_LIMIT_PHRASES: tuple[str, ...] = ("per day", "perday", "daily")
# Wording in a 403 that means the key itself was shut off (Gemini answers a
# key flagged as leaked with PERMISSION_DENIED), rather than a working key
# that may not use this model.
_KEY_REVOKED_PHRASES: tuple[str, ...] = ("leaked", "suspended", "revoked")
def _matches_any(msg: str, phrases: tuple[str, ...]) -> bool:
return any(phrase in msg for phrase in phrases)
def _is_failed_generation(exc: BaseException) -> bool:
"""True for Groq's ``json_validate_failed`` reply.
In JSON mode Groq checks the model's output itself and, when it is not
valid JSON, answers HTTP 400 with ``code: json_validate_failed`` and the
model's text in ``failed_generation``. The OpenAI-style SDKs keep the
inner ``error`` object as ``exc.body`` and its code as ``exc.code``.
"""
if getattr(exc, "code", None) == "json_validate_failed":
return True
body = getattr(exc, "body", None)
if isinstance(body, dict):
err = body.get("error", body)
if isinstance(err, dict) and (
err.get("code") == "json_validate_failed" or "failed_generation" in err
):
return True
return "json_validate_failed" in str(exc).lower()
def _classify_exception(exc: BaseException) -> str:
"""Map provider SDK exceptions to one of the stable status codes.
Covers OpenAI/Anthropic/Groq (httpx-based SDKs) and Gemini
(``google.api_core`` exceptions such as ``PermissionDenied``,
``ResourceExhausted``, ``InvalidArgument``, ``NotFound``).
"""
name = exc.__class__.__name__.lower()
msg = str(exc).lower()
http_status = _error_metadata(exc).get("http_status")
# Groq's json_validate_failed is one unreadable reply, the same as a
# reply that does not parse. Tested first: its body echoes the model's
# own text, which can say anything ("not a model I use", "network",
# "authentication"), so no substring rule below may read it.
if _is_failed_generation(exc):
return STATUS_PARSE_ERROR
# The exact HTTP status wins when the SDK exposes one. A 429 is a
# rate-limit / quota reply on nearly every provider here, so it must
# not fall through to the substring checks below and be reported as a
# bad key — and Groq's OpenAI-compatible fallback client raises a
# different exception class from the groq SDK for the same condition,
# so the class name cannot be trusted either.
#
# But not every 429 is transient. Groq answers an oversized prompt with
# one too: "Request too large for model ... on tokens per minute (TPM):
# Limit 12000, Requested 13000, please reduce your message size". That
# body mentions "tokens per minute", so testing rate-limit wording first
# would call it STATUS_QUOTA — which is retryable — and the app would
# re-send the identical oversized request three times with sleeps on the
# Streamlit thread, failing the same way each time. The request being
# too big is the non-retryable size error, so that wording is tested
# FIRST; genuine rate limiting ("rate limit reached", "requests per
# minute", "quota") keeps STATUS_QUOTA and keeps its retry — unless the
# allowance is spent for the day or the credit is gone (_retry_delay).
if http_status == 429:
if _matches_any(msg, _OVERSIZED_REQUEST_PHRASES):
return STATUS_CONTEXT_OVERFLOW
if _matches_any(msg, _RATE_LIMIT_PHRASES):
return STATUS_QUOTA
if _matches_any(msg, _CONTEXT_OVERFLOW_PHRASES):
return STATUS_CONTEXT_OVERFLOW
return STATUS_QUOTA
# A server-side failure is transient whatever its body says: any 5xx,
# including Anthropic's 529 "overloaded_error" (OverloadedError, whose
# class name matches none of the rules below). "overloaded" also covers
# SDKs and wrappers that report no status ("The model is overloaded").
if (
(isinstance(http_status, int) and 500 <= http_status <= 599)
or "overloaded" in name
or "overloaded" in msg
):
return STATUS_CONNECTION
# 401 / 403 / 404 are three different problems that read alike in a
# message: a key that was not accepted, a working key that may not use
# this model, and a model name the provider does not know.
if http_status == 401:
return STATUS_INVALID_KEY
if http_status == 403:
if _matches_any(msg, _KEY_REVOKED_PHRASES):
return STATUS_INVALID_KEY
return STATUS_ACCESS_DENIED
if http_status == 404:
return STATUS_MODEL_MISSING
if _matches_any(msg, _NO_CREDIT_PHRASES):
return STATUS_QUOTA
if (
"auth" in name
or "permissiondenied" in name
or "unauthenticated" in name
or ("invalid" in name and "key" in msg)
):
return STATUS_INVALID_KEY
if (
"authentication" in msg
or "invalid api key" in msg
or "api key not valid" in msg
or "api_key_invalid" in msg
or "permission denied" in msg
or "unauthorized" in msg
):
return STATUS_INVALID_KEY
# Context-window / prompt-size overflow is a DISTINCT condition from a
# billing/rate-limit quota: the fix is "send less input / raise
# max_tokens", not "wait or buy credits". Classify it first so it is
# never mislabeled as STATUS_QUOTA (which would tell the user they are
# out of credits). Note the parenthesized and-clause — without parens
# Python's `and` binds tighter than `or`, which silently changed the
# intended grouping in the original code.
if _matches_any(msg, _CONTEXT_OVERFLOW_PHRASES) or (
"exceeds the" in msg and "token" in msg
):
return STATUS_CONTEXT_OVERFLOW
if (
"ratelimit" in name
or "resourceexhausted" in name
or "429" in msg
or _matches_any(msg, _RATE_LIMIT_PHRASES)
):
return STATUS_QUOTA
if (
"connect" in name
or "timeout" in name
or "deadlineexceeded" in name
or "serviceunavailable" in name
or "internalservererror" in name
or "network" in msg
or "connection" in msg
):
return STATUS_CONNECTION
# Narrow on purpose: "model" and "not" anywhere in a message ("the model
# did not answer") is not a missing model.
if (
"notfound" in name
or "model_not_found" in msg
or ("model" in msg and ("not found" in msg or "does not exist" in msg))
):
return STATUS_MODEL_MISSING
return STATUS_EXCEPTION
def _error_metadata(exc: BaseException) -> dict:
"""Pull structured correlation fields off a provider SDK exception.
Complements :func:`_classify_exception`, which has to guess from
substrings. These fields are exact:
* ``request_id`` — Anthropic (``req_…``) and OpenAI both attach one to
both responses and errors. It is what a provider's support team asks
for first, and without it a bug report can say "a call failed" but
not *which* call.
* ``http_status`` — 401 (bad key) vs 403 (no access to that model) vs
404 (model does not exist) vs 400 (bad parameter) are four different
user-facing problems that all read the same in a message string.
* ``error_type`` — the provider's own machine-readable code, e.g.
``authentication_error`` / ``not_found_error`` / ``invalid_request_error``.
Everything is read defensively: SDKs differ, and a missing attribute
must degrade to ``None`` rather than mask the original error.
"""
meta: dict = {"request_id": None, "http_status": None, "error_type": None}
try:
request_id = getattr(exc, "request_id", None)
if request_id:
meta["request_id"] = str(request_id)[:64]
status_code = getattr(exc, "status_code", None)
if isinstance(status_code, int):
meta["http_status"] = status_code
else:
# google-api-core exposes the numeric status as `code`.
code = getattr(exc, "code", None)
if isinstance(code, int):
meta["http_status"] = code
# Anthropic/OpenAI put a machine-readable code inside the parsed
# JSON body: {"error": {"type": "...", "message": "..."}}.
body = getattr(exc, "body", None)
if isinstance(body, dict):
err = body.get("error")
if isinstance(err, dict) and err.get("type"):
meta["error_type"] = str(err["type"])[:64]
if not meta["error_type"]:
# Fall back to the SDK's own exception class name, which is
# still more precise than the free-text message.
meta["error_type"] = exc.__class__.__name__[:64]
except Exception: # noqa: BLE001 - never let diagnostics mask the real error
pass
return meta
# Key/org-id patterns now live in app.utils.redaction (single source of
# truth, imported above as _API_KEY_PATTERNS / _ORG_ID_PATTERNS) so the
# sink-level logging Filter and this sanitizer can never drift apart.
def sanitize_error_message(
message: Optional[str], *, secrets: Iterable[Optional[str]] = ()
) -> Optional[str]:
"""Return ``message`` with API keys and provider organization IDs scrubbed.
The UI and the API usage log only ever show the sanitized form. Provider
SDK errors can echo the raw API key (an OpenAI-compatible endpoint emits
"Incorrect API key provided: sk-..."; Gemini emits "API key not valid.
Provided: AIza..."), so this redacts, in order:
* any exact ``secrets`` value the caller supplies (the configured key) —
catches even unanticipated formats,
* API-key-shaped tokens for every provider family,
* provider organization IDs and request slugs.
Idempotent — safe to call more than once — and never returns a raw key.
Delegates to :func:`app.utils.redaction.redact` (also used by the
logging Filter) so there is exactly one definition of "what is a secret".
"""
return _redact(message, secrets=secrets)
def extend_last_entry(task_name: str, extra: dict) -> None:
"""Merge ``extra`` into the most recent usage-log entry for ``task_name``.
Used by callers that need to attach task-specific metadata
(e.g. compact-context size) to the entry the central client wrote.
The merge is metadata-only — callers must not pass raw prompts,
proposal text, dossier content, or API keys.
If no entry yet exists for ``task_name`` (this happens in tests
that monkey-patch the provider call and bypass the central
finalize step), a fresh metadata-only entry is appended so the
caller's attached fields are still recorded.
On a worker thread this is the worker's own list (see
:func:`set_thread_usage_log`), which holds the entry its call wrote.
"""
log = _usage_log()
if log is None:
return
for entry in reversed(log):
if entry.get("task_name") == task_name:
entry.update(extra)
return
new_entry: dict = {
"timestamp": datetime.now(timezone.utc).isoformat(timespec="seconds"),
"task_name": task_name,
"provider": None,
"model": None,
"used_api": False,
"status": "metadata_only",
"error_message": None,
"duration_seconds": None,
"estimated_input_tokens": None,
"estimated_output_tokens": None,
}
new_entry.update(extra)
log.append(new_entry)
def _normalize_image(image: Any) -> Optional[tuple[bytes, str]]:
"""Return (raw_bytes, mime) for a Streamlit UploadedFile or bytes-like."""
if image is None:
return None
if isinstance(image, tuple) and len(image) == 2 and isinstance(image[0], (bytes, bytearray)):
return bytes(image[0]), str(image[1]) or "image/png"
if isinstance(image, (bytes, bytearray)):
return bytes(image), "image/png"
name = getattr(image, "name", "") or ""
mime = getattr(image, "type", "") or ""
suffix = name.lower().rsplit(".", 1)[-1] if "." in name else ""
if not mime:
mime = {
"png": "image/png",
"jpg": "image/jpeg",
"jpeg": "image/jpeg",
"webp": "image/webp",
}.get(suffix, "image/png")
data: Optional[bytes] = None
if hasattr(image, "getvalue"):
try:
data = image.getvalue()
except Exception: # noqa: BLE001
data = None
if data is None and hasattr(image, "read"):
try:
if hasattr(image, "seek"):
image.seek(0)
data = image.read()
except Exception: # noqa: BLE001
data = None
if data is None:
return None
return data, mime
# Target long-edge size (px) we upscale small screenshots to before sending to
# the vision model. Small pasted screenshots have tiny client-panel text the
# model can't resolve; enlarging the image to a comfortable resolution makes
# that fine print legible. ~1600 px matches what the major vision models
# downsample to internally, so we never send needlessly huge payloads.
_VISION_TARGET_LONG_EDGE = 1600
# Only upscale when the long edge is below this — images already large enough
# are left untouched (no quality loss, no wasted bytes).
_VISION_MIN_LONG_EDGE = 1100
# Hard cap so a pathological tiny image isn't blown up enormously.
_VISION_MAX_UPSCALE = 3.0
# Provider ceilings. Anthropic refuses an image over 8000 px on a side or
# over 5 MB once base64-encoded (about 3.75 MB of raw bytes), and Gemini a
# request over 20 MB in all. A full-page capture of a long job post or a
# 5K-display screenshot breaks them, and the provider then answers 400 on
# every try, so such an image is scaled down and re-encoded to fit.
_VISION_MAX_EDGE = 7680
_VISION_MAX_BYTES = 3_500_000
# Raw bytes all the images of one request may take together: room under
# Gemini's 20 MB for the prompt, and under Anthropic's 32 MB once base64
# adds its third.
_VISION_MAX_TOTAL_BYTES = 14_000_000
# Media type of the bytes Pillow actually found. A renamed file (JPEG bytes
# in "shot.png") must go out with its real type, or Anthropic refuses it.
_VISION_FORMAT_MIME = {
"PNG": "image/png",
"JPEG": "image/jpeg",
"MPO": "image/jpeg", # a JPEG with extra frames, as some phones save
"WEBP": "image/webp",
}
def _prepare_vision_image(
data: bytes, mime: str, *, max_bytes: int = _VISION_MAX_BYTES
) -> tuple[bytes, str]:
"""Fit a screenshot to what the vision model can read and accept.
A small screenshot is upscaled so the model can read fine print. One
over :data:`_VISION_MAX_EDGE` px on a side or over ``max_bytes`` is
scaled down and re-encoded until it fits (:func:`_shrink_vision_image`).
Anything else is sent unchanged, labelled with its real media type.
Best-effort: if Pillow is unavailable or the image can't be decoded, the
original bytes are returned unchanged. Upscaled output is re-encoded as
lossless PNG so small text stays crisp.
"""
try:
import io as _io
from PIL import Image # type: ignore
except Exception: # noqa: BLE001 - Pillow missing → send original
return data, mime
try:
with Image.open(_io.BytesIO(data)) as im:
im.load()
mime = _VISION_FORMAT_MIME.get(im.format, mime)
w, h = im.size
long_edge = max(w, h)
if long_edge > _VISION_MAX_EDGE or len(data) > max_bytes:
return _shrink_vision_image(im, max_bytes)
if long_edge <= 0 or long_edge >= _VISION_MIN_LONG_EDGE:
return data, mime # already big enough
scale = min(_VISION_TARGET_LONG_EDGE / long_edge, _VISION_MAX_UPSCALE)
if scale <= 1.0:
return data, mime
new_size = (max(1, round(w * scale)), max(1, round(h * scale)))
# Convert to a safe mode for PNG, then high-quality upscale.
if im.mode not in ("RGB", "RGBA", "L"):
im = im.convert("RGB")
resized = im.resize(new_size, Image.LANCZOS)
buf = _io.BytesIO()
resized.save(buf, format="PNG", optimize=True)
if buf.tell() > max_bytes:
return data, mime # the enlarged copy would not fit
return buf.getvalue(), "image/png"
except Exception: # noqa: BLE001 - any decode/resize failure → send original
return data, mime
def _shrink_vision_image(im: Any, max_bytes: int) -> tuple[bytes, str]:
"""Re-encode ``im`` within :data:`_VISION_MAX_EDGE` px and ``max_bytes``.
Lossless PNG is tried first, then JPEG at falling quality, and only then
a smaller size, so the screenshot keeps as much of its fine print as the
limits allow.
"""
import io as _io
from PIL import Image, ImageOps # type: ignore
im = ImageOps.exif_transpose(im) # re-encoding drops the EXIF rotation
if im.mode not in ("RGB", "L"):
# JPEG has no alpha channel: flatten a transparent capture onto white.
rgba = im.convert("RGBA")
im = Image.new("RGB", rgba.size, "white")
im.paste(rgba, mask=rgba.getchannel("A"))
w, h = im.size
scale = min(1.0, _VISION_MAX_EDGE / max(w, h))
for _ in range(8):
size = (max(1, int(w * scale)), max(1, int(h * scale)))
frame = im if size == im.size else im.resize(size, Image.LANCZOS)
for fmt, options in (
("PNG", {}),
("JPEG", {"quality": 90}),
("JPEG", {"quality": 80}),
("JPEG", {"quality": 70}),
):
buf = _io.BytesIO()
frame.save(buf, format=fmt, **options)
if buf.tell() <= max_bytes:
return buf.getvalue(), f"image/{fmt.lower()}"
scale *= 0.75
return buf.getvalue(), "image/jpeg"
# ---------------------------------------------------------------------------
# Provider adapters
# ---------------------------------------------------------------------------
def _response_meta(response: Any) -> dict:
"""Correlation fields from a *successful* provider response.
Both the Anthropic and OpenAI SDKs expose the HTTP request id as
``response._request_id`` — public despite the underscore, and the value
provider support asks for when investigating a specific call. Captured
on success as well as failure so a report can point at the exact call
that returned a bad-looking result, not only at calls that raised.
"""
try:
request_id = getattr(response, "_request_id", None)
if request_id:
return {"request_id": str(request_id)[:64]}
except Exception: # noqa: BLE001
pass
return {}
# Models that take no sampling parameters. Anthropic removed temperature
# from Claude Opus 4.7 and every model since (Opus 4.8, the 5-series, Fable,
# Mythos; Sonnet 5.5 and Haiku 5.5 refuse any non-default value), and
# OpenAI's reasoning models (the o-series and the gpt-5 family) take only
# the default temperature of 1. temperature=0 is a 400 on every call to
# them, so they are sent the provider's default instead.
_CLAUDE_VERSION_RE = re.compile(r"claude-(?:[a-z]+-)?(\d+)(?:-(\d{1,2})(?!\d))?")
_OPENAI_GPT_VERSION_RE = re.compile(r"gpt-(\d+)")
def _sampling_params(provider: str, model: str) -> dict:
"""``{"temperature": 0}`` for a model that accepts it, else ``{}``.
Claude ids carry their version (``claude-sonnet-4-6``, the older
``claude-3-5-sonnet-…``); a date suffix is not mistaken for a minor
version. A name that cannot be read keeps temperature=0, as before.
"""
name = (model or "").strip().lower()
if provider == "anthropic":
if "fable" in name or "mythos" in name:
return {}
match = _CLAUDE_VERSION_RE.search(name)
if match and (int(match.group(1)), int(match.group(2) or 0)) >= (4, 7):
return {}
elif provider == "openai":
name = name.removeprefix("ft:") # fine-tuned: ft:gpt-4o-mini:org::id
if re.match(r"o\d", name):
return {}
match = _OPENAI_GPT_VERSION_RE.match(name)
if match and int(match.group(1)) >= 5:
return {}
return {"temperature": 0}
def _anthropic_text(
*,
api_key: str,
model: str,
system_prompt: str,
user_prompt: str,
expected_json: bool,
max_tokens: int,
) -> tuple[str, Optional[int], Optional[int], Optional[str], dict]:
import anthropic # type: ignore
client = anthropic.Anthropic(
api_key=api_key, timeout=_request_timeout(), max_retries=0
)
response = client.messages.create(
model=model,
max_tokens=max_tokens,
system=system_prompt or "",
messages=[{"role": "user", "content": user_prompt}],
**_sampling_params("anthropic", model),
)
parts: list[str] = []
for block in getattr(response, "content", []) or []:
text = getattr(block, "text", None)
if text:
parts.append(text)
out_text = "\n".join(parts).strip()
usage = getattr(response, "usage", None)
in_tokens = getattr(usage, "input_tokens", None) if usage else None
out_tokens = getattr(usage, "output_tokens", None) if usage else None
finish = getattr(response, "stop_reason", None)
return out_text, in_tokens, out_tokens, finish, _response_meta(response)
def _openai_text(
*,
api_key: str,
model: str,
system_prompt: str,
user_prompt: str,
expected_json: bool,
max_tokens: int,
) -> tuple[str, Optional[int], Optional[int], Optional[str], dict]:
from openai import OpenAI # type: ignore
client = OpenAI(api_key=api_key, timeout=_request_timeout(), max_retries=0)
kwargs: dict[str, Any] = {
"model": model,
"messages": [
*([{"role": "system", "content": system_prompt}] if system_prompt else []),
{"role": "user", "content": user_prompt},
],
**_sampling_params("openai", model),
}
if expected_json:
kwargs["response_format"] = {"type": "json_object"}
response = client.chat.completions.create(**kwargs)
out_text = (response.choices[0].message.content or "").strip()
usage = getattr(response, "usage", None)
in_tokens = getattr(usage, "prompt_tokens", None) if usage else None
out_tokens = getattr(usage, "completion_tokens", None) if usage else None
finish = getattr(response.choices[0], "finish_reason", None)
return out_text, in_tokens, out_tokens, finish, _response_meta(response)
def _groq_text(
*,
api_key: str,
model: str,
system_prompt: str,
user_prompt: str,
expected_json: bool,
max_tokens: int,
) -> tuple[str, Optional[int], Optional[int], Optional[str], dict]:
"""Groq text completion (text-only — Groq has no image input).
Groq exposes an OpenAI-compatible Chat Completions API. We prefer the
official ``groq`` SDK when installed and otherwise fall back to the
already-bundled ``openai`` SDK pointed at Groq's compatibility endpoint,
so the provider works even when ``groq`` was not separately installed.
The API key is passed to the client only and never logged.
"""
messages = [
*([{"role": "system", "content": system_prompt}] if system_prompt else []),
{"role": "user", "content": user_prompt},
]
kwargs: dict[str, Any] = {"model": model, "messages": messages, "temperature": 0}
if expected_json:
kwargs["response_format"] = {"type": "json_object"}
# GPT-OSS models think before they answer, and on Groq's free plan that
# thinking counts against the same per-minute token allowance as the
# answer. Low effort leaves the room for the answer itself.
# Sent as extra_body so an older `groq` SDK, whose create() predates the
# parameter, passes it through instead of rejecting it.
if "gpt-oss" in model.lower():
kwargs["extra_body"] = {"reasoning_effort": "low"}
try:
from groq import Groq # type: ignore
client = Groq(api_key=api_key, timeout=_request_timeout(), max_retries=0)
except ModuleNotFoundError:
# OpenAI-compatible fallback (the openai SDK ships with this app).
from openai import OpenAI # type: ignore
client = OpenAI(
api_key=api_key,
base_url="https://api.groq.com/openai/v1",
timeout=_request_timeout(),
max_retries=0,
)
response = client.chat.completions.create(**kwargs)
out_text = (response.choices[0].message.content or "").strip()
usage = getattr(response, "usage", None)
in_tokens = getattr(usage, "prompt_tokens", None) if usage else None
out_tokens = getattr(usage, "completion_tokens", None) if usage else None
finish = getattr(response.choices[0], "finish_reason", None)
return out_text, in_tokens, out_tokens, finish, _response_meta(response)
def _gemini_response_text(response: Any) -> str:
"""Best-effort text extraction from a Gemini response.
``response.text`` raises when the only candidate was blocked or empty, so
we fall back to manually concatenating any text parts on the candidates.
"""
try:
text = getattr(response, "text", None)
if text:
return text.strip()
except Exception: # noqa: BLE001 - blocked/empty candidate; fall back below
pass
parts: list[str] = []
for candidate in getattr(response, "candidates", None) or []:
content = getattr(candidate, "content", None)
for part in getattr(content, "parts", None) or []:
piece = getattr(part, "text", None)
if piece:
parts.append(piece)
return "\n".join(parts).strip()
def _gemini_usage(response: Any) -> tuple[Optional[int], Optional[int]]:
usage = getattr(response, "usage_metadata", None)
in_tokens = getattr(usage, "prompt_token_count", None) if usage else None
out_tokens = getattr(usage, "candidates_token_count", None) if usage else None
return in_tokens, out_tokens
def _gemini_finish_reason(response: Any) -> Optional[str]:
"""Return the first candidate's finish reason as a lowercase string.
Gemini exposes ``finish_reason`` as an enum (e.g. ``MAX_TOKENS``); we
normalize it to a string so the truncation check can recognize it.
"""
for candidate in getattr(response, "candidates", None) or []:
reason = getattr(candidate, "finish_reason", None)
if reason is None:
continue
name = getattr(reason, "name", None)
return str(name if name is not None else reason).lower()
return None
# Gemini 2.5 and later think before they answer, and the thinking tokens are
# counted inside max_output_tokens. The google-generativeai SDK cannot set a
# thinking budget, so a small JSON budget (150-900 tokens) could be spent on
# thinking alone and come back empty or cut off. These models get this much
# extra room on top of the caller's budget for the answer; output is billed
# by what is generated, not by the cap.
_GEMINI_THINKING_HEADROOM = 8192
# Output ceiling of those models; the extra room never pushes a request past it.
_GEMINI_MAX_OUTPUT_TOKENS = 65536
_GEMINI_VERSION_RE = re.compile(r"gemini-(\d+)(?:\.(\d+))?")
def _gemini_thinks(model: str) -> bool:
"""True for Gemini models whose thinking shares the output budget (2.5+)."""
name = (model or "").strip().lower()
if "thinking" in name:
return True
match = _GEMINI_VERSION_RE.search(name)
if match is None:
# Unversioned aliases such as gemini-flash-latest track the current
# (thinking) models.
return name.endswith("-latest")
return (int(match.group(1)), int(match.group(2) or 0)) >= (2, 5)
def _gemini_output_budget(model: str, max_tokens: int) -> int:
"""max_output_tokens to send: the caller's budget plus room to think."""
if not _gemini_thinks(model):
return max_tokens
return max(
max_tokens,
min(max_tokens + _GEMINI_THINKING_HEADROOM, _GEMINI_MAX_OUTPUT_TOKENS),
)
def _gemini_model_obj(
*, api_key: str, model: str, system_prompt: str, expected_json: bool, max_tokens: int
):
"""Build a ``genai.GenerativeModel`` bound to ``api_key``. Key is never logged.
The key goes to a client made for this call alone, never to
``genai.configure``: that sets it for the whole process (every visitor),
and the model looks its client up only when the request is sent, so two
visitors calling Gemini at once could send on each other's key.
"""
import google.ai.generativelanguage as glm # type: ignore
import google.generativeai as genai # type: ignore
generation_config: dict[str, Any] = {
"temperature": 0,
"max_output_tokens": _gemini_output_budget(model, max_tokens),
}
if expected_json:
generation_config["response_mime_type"] = "application/json"
gen_model = genai.GenerativeModel(
model_name=model,
system_instruction=system_prompt or None,
generation_config=generation_config,
)
# GenerativeModel takes no client argument; it uses one already set here
# instead of fetching the process-wide default.
gen_model._client = glm.GenerativeServiceClient(
client_options={"api_key": api_key}
)
return gen_model
def _gemini_text(
*,
api_key: str,
model: str,
system_prompt: str,
user_prompt: str,
expected_json: bool,
max_tokens: int,
) -> tuple[str, Optional[int], Optional[int], Optional[str]]:
"""Gemini text completion via the google-generativeai SDK.
When ``expected_json`` is set we both request a JSON response MIME type and
reinforce strict-JSON output in the prompt.
"""
gen_model = _gemini_model_obj(
api_key=api_key,
model=model,
system_prompt=system_prompt,
expected_json=expected_json,
max_tokens=max_tokens,
)
prompt = user_prompt
if expected_json:
prompt = (
f"{user_prompt}\n\nReturn ONLY a single valid JSON object. "
"Do not include any prose or Markdown code fences."
)
# A per-request timeout is only honored when passed to generate_content
# via request_options — a configure-/model-level timeout is ignored.
# retry=None switches off google-api-core's own retry (ServiceUnavailable,
# for up to 600 s), so _invoke_with_retry stays the one retry layer.
response = gen_model.generate_content(
prompt, request_options={"timeout": _request_timeout(), "retry": None}
)
out_text = _gemini_response_text(response)
in_tokens, out_tokens = _gemini_usage(response)
finish = _gemini_finish_reason(response)
return out_text, in_tokens, out_tokens, finish
def _anthropic_vision(
*,
api_key: str,
model: str,
system_prompt: str,
user_prompt: str,
images: list[tuple[bytes, str]],
expected_json: bool,
max_tokens: int,
) -> tuple[str, Optional[int], Optional[int], Optional[str], dict]:
import anthropic # type: ignore
client = anthropic.Anthropic(
api_key=api_key, timeout=_request_timeout(), max_retries=0
)
content: list[dict[str, Any]] = []
for data, mime in images:
content.append(
{
"type": "image",
"source": {
"type": "base64",
"media_type": mime,
"data": base64.b64encode(data).decode("ascii"),
},
}
)
if user_prompt:
content.append({"type": "text", "text": user_prompt})
response = client.messages.create(
model=model,
max_tokens=max_tokens,
system=system_prompt or "",
messages=[{"role": "user", "content": content}],
**_sampling_params("anthropic", model),
)
parts: list[str] = []
for block in getattr(response, "content", []) or []:
text = getattr(block, "text", None)
if text:
parts.append(text)
out_text = "\n".join(parts).strip()
usage = getattr(response, "usage", None)
in_tokens = getattr(usage, "input_tokens", None) if usage else None
out_tokens = getattr(usage, "output_tokens", None) if usage else None
finish = getattr(response, "stop_reason", None)
return out_text, in_tokens, out_tokens, finish, _response_meta(response)
def _openai_vision(
*,
api_key: str,
model: str,
system_prompt: str,
user_prompt: str,
images: list[tuple[bytes, str]],
expected_json: bool,
max_tokens: int,
) -> tuple[str, Optional[int], Optional[int], Optional[str], dict]:
from openai import OpenAI # type: ignore
client = OpenAI(api_key=api_key, timeout=_request_timeout(), max_retries=0)
content: list[dict[str, Any]] = []
if user_prompt:
content.append({"type": "text", "text": user_prompt})
for data, mime in images:
# detail="high" makes OpenAI process the image at full resolution (tiled),
# which is essential for reading small text in a job-post screenshot.
content.append({
"type": "image_url",
"image_url": {"url": _data_url(data, mime), "detail": "high"},
})
kwargs: dict[str, Any] = {
"model": model,
"messages": [
*([{"role": "system", "content": system_prompt}] if system_prompt else []),
{"role": "user", "content": content},
],
**_sampling_params("openai", model),
}
if expected_json:
kwargs["response_format"] = {"type": "json_object"}
response = client.chat.completions.create(**kwargs)
out_text = (response.choices[0].message.content or "").strip()
usage = getattr(response, "usage", None)
in_tokens = getattr(usage, "prompt_tokens", None) if usage else None
out_tokens = getattr(usage, "completion_tokens", None) if usage else None
finish = getattr(response.choices[0], "finish_reason", None)
return out_text, in_tokens, out_tokens, finish, _response_meta(response)
def _gemini_vision(
*,
api_key: str,
model: str,
system_prompt: str,
user_prompt: str,
images: list[tuple[bytes, str]],
expected_json: bool,
max_tokens: int,
) -> tuple[str, Optional[int], Optional[int], Optional[str]]:
"""Gemini multimodal completion: prompt text plus inline image blobs."""
gen_model = _gemini_model_obj(
api_key=api_key,
model=model,
system_prompt=system_prompt,
expected_json=expected_json,
max_tokens=max_tokens,
)
content: list[Any] = []
if user_prompt:
content.append(user_prompt)
for data, mime in images:
# google-generativeai accepts inline blobs as {"mime_type", "data"}.
content.append({"mime_type": mime, "data": data})
response = gen_model.generate_content(
content, request_options={"timeout": _request_timeout(), "retry": None}
)
out_text = _gemini_response_text(response)
in_tokens, out_tokens = _gemini_usage(response)
finish = _gemini_finish_reason(response)
return out_text, in_tokens, out_tokens, finish
# ---------------------------------------------------------------------------
# Adapter invocation helpers (retry + tolerant unpack)
# ---------------------------------------------------------------------------
def _normalize_adapter_return(
ret: Any,
) -> tuple[str, Optional[int], Optional[int], Optional[str], dict]:
"""Accept a 3-, 4- or 5-tuple from a provider adapter.
Current adapters return ``(text, in_tokens, out_tokens, finish_reason,
meta)`` where ``meta`` carries provider correlation fields such as
``request_id``. Adapters that predate ``meta`` return the 4-tuple, and
older/monkeypatched fakes return the legacy 3-tuple; both keep working
and simply report no request id. Widening here rather than at each call
site is what lets a fake stay a three-liner.
"""
if isinstance(ret, tuple) and len(ret) == 5:
text, in_toks, out_toks, finish, meta = ret
return text, in_toks, out_toks, finish, meta if isinstance(meta, dict) else {}
if isinstance(ret, tuple) and len(ret) == 4:
text, in_toks, out_toks, finish = ret
return text, in_toks, out_toks, finish, {}
if isinstance(ret, tuple) and len(ret) == 3:
text, in_toks, out_toks = ret
return text, in_toks, out_toks, None, {}
# Unexpected shape — treat as an empty text response.
return str(ret or ""), None, None, None, {}
def _retry_after_seconds(exc: BaseException) -> Optional[float]:
"""The wait the provider asked for in its Retry-After header, if any."""
headers = getattr(getattr(exc, "response", None), "headers", None)
for header, scale in (("retry-after-ms", 1000.0), ("retry-after", 1.0)):
try:
raw = headers.get(header) if headers is not None else None
if raw:
return float(raw) / scale
except (AttributeError, TypeError, ValueError): # an HTTP date or junk
continue
return None
def _retry_delay(exc: BaseException, status: str, attempt: int) -> Optional[float]:
"""Seconds to wait before retrying ``exc``, or ``None`` not to retry it.
A spent daily allowance or an account out of credit is a quota answer
too, but no wait of seconds can fix it, so it fails at once instead of
being re-sent. The provider's Retry-After, when it sent one, wins over
the backoff, up to :data:`_RETRY_AFTER_CAP`.
"""
if status not in _RETRYABLE_STATUSES:
return None
if status == STATUS_QUOTA and _matches_any(
str(exc).lower(), _NO_CREDIT_PHRASES + _DAILY_LIMIT_PHRASES
):
return None
delay = _RETRY_BACKOFF_BASE * (2 ** (attempt - 1))
retry_after = _retry_after_seconds(exc)
if retry_after is not None:
if retry_after > _RETRY_AFTER_CAP:
return None
delay = max(delay, retry_after)
return delay
def _invoke_with_retry(adapter, *, api_key: str, **kwargs):
"""Call ``adapter`` with a bounded transient-failure retry.
Retries ONLY transient conditions (STATUS_CONNECTION and genuine
rate-limit STATUS_QUOTA) with exponential backoff; never retries
auth / model-missing / context-overflow / parse errors or a spent
quota (a retry cannot help and would waste calls and money), and never
starts an attempt that could run past the call's deadline (see
:data:`_RETRY_WINDOW`). Re-raises the final exception so the caller's
existing except-block classifies it.
"""
deadline = time.monotonic() + _request_timeout() + _RETRY_WINDOW
attempt = 0
while True:
attempt += 1
try:
return _normalize_adapter_return(
adapter(api_key=api_key, **kwargs)
)
except ModuleNotFoundError:
raise # missing SDK — surfaced verbatim, never retried
except Exception as exc: # noqa: BLE001
status = _classify_exception(exc)
delay = (
_retry_delay(exc, status, attempt)
if attempt < _MAX_ATTEMPTS
else None
)
if (
delay is not None
and time.monotonic() + delay + _request_timeout() <= deadline
):
_logger.info(
"transient llm failure status=%s attempt=%s/%s — retrying",
status,
attempt,
_MAX_ATTEMPTS,
)
time.sleep(delay)
continue
raise
def _is_truncated(finish_reason: Optional[str]) -> bool:
if not finish_reason:
return False
return str(finish_reason).strip().lower() in _TRUNCATION_REASONS
# ---------------------------------------------------------------------------
# Public API
# ---------------------------------------------------------------------------
def interview_route(
settings: Optional[Settings] = None,
*,
prefer: Optional[str] = None,
) -> Optional[tuple[str, Optional[str]]]:
"""Which provider, and which model, the guided interview runs on.
By default the interview uses the provider set up on the Setup screen
— the same key and model as the rest of the app — so it is intelligent
for anyone who has passed Setup. A Groq key, when one is added, wins:
it is the free option a user adds on purpose to keep the interview's
calls off their main API.
``prefer="main"`` skips the Groq key and returns the Setup provider — for
when the Groq check failed and the interview falls back to the main API.
Returns ``(provider, model)``, where ``model`` is ``None`` to mean "that
provider's own configured model", or ``None`` when no provider with a
key is available. The dedicated interview model applies only on Groq:
its default names a Groq model, which no other provider would accept.
Never returns or logs a key.
"""
settings = settings or get_settings()
main = _resolve_provider(settings)
if prefer != "main" and _resolve_key(settings, INTERVIEW_PROVIDER):
model = (getattr(settings, "interview_model", "") or "").strip() or None
return INTERVIEW_PROVIDER, model
if main in _SUPPORTED_PROVIDERS and _resolve_key(settings, main):
return main, None
return None
def interview_available(settings: Optional[Settings] = None) -> bool:
"""True when the guided interview has a provider it can reach.
See :func:`interview_route`: a Groq key if one is set, otherwise the
provider configured on the Setup screen. Returns a bool only.
"""
return interview_route(settings) is not None
def call_text_llm(
task_name: str,
system_prompt: str,
user_prompt: str,
*,
expected_json: bool = False,
max_tokens: int = 1024,
settings: Optional[Settings] = None,
provider: Optional[str] = None,
model: Optional[str] = None,
) -> LLMCallResult:
"""Send a single text completion to the configured provider.
``provider`` routes this one call to a named provider instead of the
configured one; that provider's key and model are used, so a stage
that must run elsewhere (the guided interview, which always runs on
:data:`app.config.INTERVIEW_PROVIDER`) does not disturb the rest of
the flow. ``model`` overrides the model name for this call only.
Both default to ``None``, which keeps resolution exactly as it was,
so every existing caller is unchanged. The override is recorded in
the usage log like any other call, because the result already carries
the resolved provider and model.
A provider named here with no key configured fails cleanly with
``STATUS_NO_API``; there is no silent fallback to the configured
provider.
"""
_validate_task_name(task_name)
settings = settings or get_settings()
provider = _resolve_provider(settings, provider=provider)
model = (model or "").strip() or _resolve_model(settings, provider=provider)
api_key = _resolve_key(settings, provider)
started = time.monotonic()
if provider not in _SUPPORTED_PROVIDERS:
result = LLMCallResult(
success=False,
task_name=task_name,
provider=provider or None,
model=model,
used_api=False,
error_message="Provider not configured",
status=STATUS_UNSUPPORTED_PROVIDER,
)
_finalize(result, started)
return result
if not api_key:
result = LLMCallResult(
success=False,
task_name=task_name,
provider=provider,
model=model,
used_api=False,
error_message="No API key configured for provider",
status=STATUS_NO_API,
)
_finalize(result, started)
return result
if not model:
result = LLMCallResult(
success=False,
task_name=task_name,
provider=provider,
model=None,
used_api=False,
error_message="Model not configured",
status=STATUS_MODEL_MISSING,
)
_finalize(result, started)
return result
try:
_text_adapter = {
"anthropic": _anthropic_text,
"openai": _openai_text,
"groq": _groq_text,
"gemini": _gemini_text,
}[provider]
text, in_toks, out_toks, finish_reason, call_meta = _invoke_with_retry(
_text_adapter,
api_key=api_key,
model=model,
system_prompt=system_prompt,
user_prompt=user_prompt,
expected_json=expected_json,
max_tokens=max_tokens,
)
except ModuleNotFoundError as exc:
result = LLMCallResult(
success=False,
task_name=task_name,
provider=provider,
model=model,
used_api=True,
error_message=f"Provider SDK missing: {exc.name}",
status=STATUS_CONNECTION,
)
_finalize(result, started)
return result
except Exception as exc: # noqa: BLE001
status = _classify_exception(exc)
err_meta = _error_metadata(exc)
result = LLMCallResult(
success=False,
task_name=task_name,
provider=provider,
model=model,
used_api=True,
error_message=sanitize_error_message(
f"{exc.__class__.__name__}: {exc}", secrets=(api_key,)
)[:200],
status=status,
request_id=err_meta.get("request_id"),
http_status=err_meta.get("http_status"),
error_type=err_meta.get("error_type"),
)
_finalize(result, started)
return result
response_json: Any = None
# An empty plain-text reply cut off by the token budget (a thinking model
# can spend all of it before writing a word) is no answer either: it
# parses to None and takes the same failure path as unusable JSON.
if expected_json or (not text and _is_truncated(finish_reason)):
response_json = _parse_json(text)
if response_json is None:
# Distinguish an output cut off by the token budget (recoverable
# by raising max_tokens) from genuinely malformed JSON.
truncated = _is_truncated(finish_reason)
result = LLMCallResult(
success=False,
task_name=task_name,
provider=provider,
model=model,
used_api=True,
response_text=text,
error_message=(
"Provider response was truncated by the output token "
"limit; raise max_tokens and retry."
if truncated
else "Provider response was not valid JSON"
),
estimated_input_tokens=in_toks if in_toks is not None
else _estimate_tokens(system_prompt + user_prompt),
estimated_output_tokens=out_toks if out_toks is not None
else _estimate_tokens(text),
status=STATUS_TRUNCATED if truncated else STATUS_PARSE_ERROR,
request_id=call_meta.get("request_id"),
)
_finalize(result, started)
return result
result = LLMCallResult(
success=True,
task_name=task_name,
provider=provider,
model=model,
used_api=True,
response_text=text,
response_json=response_json,
estimated_input_tokens=in_toks if in_toks is not None
else _estimate_tokens(system_prompt + user_prompt),
estimated_output_tokens=out_toks if out_toks is not None
else _estimate_tokens(text),
status=STATUS_OK,
request_id=call_meta.get("request_id"),
)
_finalize(result, started)
return result
def call_vision_llm(
task_name: str,
system_prompt: str,
image_inputs: Iterable,
user_prompt: str = "",
*,
expected_json: bool = False,
max_tokens: int = 2048,
settings: Optional[Settings] = None,
) -> LLMCallResult:
"""Send a vision request to the configured provider."""
_validate_task_name(task_name)
settings = settings or get_settings()
provider = _resolve_provider(settings, vision=True)
model = _resolve_model(settings, vision=True)
api_key = _resolve_key(settings, provider)
started = time.monotonic()
normalized = [
norm for norm in map(_normalize_image, image_inputs or []) if norm is not None
]
# Each image gets an equal share of what one request may carry.
max_bytes = min(
_VISION_MAX_BYTES, _VISION_MAX_TOTAL_BYTES // max(1, len(normalized))
)
images: list[tuple[bytes, str]] = []
for data, mime in normalized:
# Upscale small screenshots so tiny client-panel text is legible, and
# shrink oversized ones to what the provider accepts.
images.append(_prepare_vision_image(data, mime, max_bytes=max_bytes))
if not images:
result = LLMCallResult(
success=False,
task_name=task_name,
provider=provider or None,
model=model,
used_api=False,
error_message="No image inputs supplied",
status=STATUS_SKIPPED,
)
_finalize(result, started)
return result
# Groq is text-only here — surface the dedicated, actionable message
# rather than a generic "provider not configured".
if provider == "groq":
result = LLMCallResult(
success=False,
task_name=task_name,
provider=provider,
model=model,
used_api=False,
error_message=GROQ_VISION_UNSUPPORTED_MESSAGE,
status=STATUS_UNSUPPORTED_PROVIDER,
)
_finalize(result, started)
return result
if provider not in _SUPPORTED_VISION_PROVIDERS:
result = LLMCallResult(
success=False,
task_name=task_name,
provider=provider or None,
model=model,
used_api=False,
error_message="Provider not configured",
status=STATUS_UNSUPPORTED_PROVIDER,
)
_finalize(result, started)
return result
if not api_key:
result = LLMCallResult(
success=False,
task_name=task_name,
provider=provider,
model=model,
used_api=False,
error_message="No API key configured for provider",
status=STATUS_NO_API,
)
_finalize(result, started)
return result
if not model:
result = LLMCallResult(
success=False,
task_name=task_name,
provider=provider,
model=None,
used_api=False,
error_message="Model not configured",
status=STATUS_MODEL_MISSING,
)
_finalize(result, started)
return result
try:
_vision_adapter = {
"anthropic": _anthropic_vision,
"openai": _openai_vision,
"gemini": _gemini_vision,
}[provider]
text, in_toks, out_toks, finish_reason, call_meta = _invoke_with_retry(
_vision_adapter,
api_key=api_key,
model=model,
system_prompt=system_prompt,
user_prompt=user_prompt,
images=images,
expected_json=expected_json,
max_tokens=max_tokens,
)
except ModuleNotFoundError as exc:
result = LLMCallResult(
success=False,
task_name=task_name,
provider=provider,
model=model,
used_api=True,
error_message=f"Provider SDK missing: {exc.name}",
status=STATUS_CONNECTION,
)
_finalize(result, started)
return result
except Exception as exc: # noqa: BLE001
status = _classify_exception(exc)
err_meta = _error_metadata(exc)
result = LLMCallResult(
success=False,
task_name=task_name,
provider=provider,
model=model,
used_api=True,
error_message=sanitize_error_message(
f"{exc.__class__.__name__}: {exc}", secrets=(api_key,)
)[:200],
status=status,
request_id=err_meta.get("request_id"),
http_status=err_meta.get("http_status"),
error_type=err_meta.get("error_type"),
)
_finalize(result, started)
return result
response_json: Any = None
# Same rule as call_text_llm: an empty reply cut off by the token budget
# is a failure even when no JSON was asked for.
if expected_json or (not text and _is_truncated(finish_reason)):
response_json = _parse_json(text)
if response_json is None:
truncated = _is_truncated(finish_reason)
result = LLMCallResult(
success=False,
task_name=task_name,
provider=provider,
model=model,
used_api=True,
response_text=text,
error_message=(
"Provider response was truncated by the output token "
"limit; raise max_tokens and retry."
if truncated
else "Provider response was not valid JSON"
),
estimated_input_tokens=in_toks if in_toks is not None
else _estimate_tokens(system_prompt + user_prompt),
estimated_output_tokens=out_toks if out_toks is not None
else _estimate_tokens(text),
status=STATUS_TRUNCATED if truncated else STATUS_PARSE_ERROR,
request_id=call_meta.get("request_id"),
)
_finalize(result, started)
return result
result = LLMCallResult(
success=True,
task_name=task_name,
provider=provider,
model=model,
used_api=True,
response_text=text,
response_json=response_json,
estimated_input_tokens=in_toks if in_toks is not None
else _estimate_tokens(system_prompt + user_prompt),
estimated_output_tokens=out_toks if out_toks is not None
else _estimate_tokens(text),
status=STATUS_OK,
request_id=call_meta.get("request_id"),
)
_finalize(result, started)
return result
# ---------------------------------------------------------------------------
# Finalization
# ---------------------------------------------------------------------------
def _finalize(result: LLMCallResult, started_monotonic: float) -> None:
result.duration_seconds = round(time.monotonic() - started_monotonic, 3)
# Strip provider organization IDs before the message touches the
# session log so the UI never displays raw provider metadata.
if result.error_message:
result.error_message = sanitize_error_message(result.error_message)
entry = result.to_log_entry()
_record_to_session(entry)
# Durable copy, written the instant the call finishes rather than when
# the user clicks "share". A browser reload is the usual first reaction
# to a failure, and it wipes session state — so a report assembled only
# at share time would routinely arrive empty. Fail-safe by contract:
# `record_event` swallows its own errors.
_record_to_disk(entry)
_logger.info(
"stage=%s used_api=%s status=%s provider=%s model=%s in_tokens=%s "
"out_tokens=%s http_status=%s error_type=%s request_id=%s",
result.task_name,
str(result.used_api).lower(),
result.status,
result.provider or "-",
result.model or "-",
result.estimated_input_tokens if result.estimated_input_tokens is not None else "-",
result.estimated_output_tokens if result.estimated_output_tokens is not None else "-",
result.http_status if result.http_status is not None else "-",
result.error_type or "-",
result.request_id or "-",
)
__all__ = [
"LLMCallResult",
"call_text_llm",
"interview_available",
"interview_route",
"call_vision_llm",
"record_local_use",
"extend_last_entry",
"sanitize_error_message",
"get_usage_log",
"reset_usage_log",
"set_thread_usage_log",
"add_to_usage_log",
"STATUS_OK",
"STATUS_NO_API",
"STATUS_INVALID_KEY",
"STATUS_ACCESS_DENIED",
"STATUS_QUOTA",
"STATUS_CONTEXT_OVERFLOW",
"STATUS_CONNECTION",
"STATUS_MODEL_MISSING",
"STATUS_PARSE_ERROR",
"STATUS_TRUNCATED",
"STATUS_UNSUPPORTED_PROVIDER",
"STATUS_SKIPPED",
"STATUS_EXCEPTION",
]
"""Central LLM client.
This is the ONLY module that talks to Anthropic, OpenAI, Groq, or Gemini
directly (Groq for text only). All other services route through
:func:`call_text_llm` or :func:`call_vision_llm` so:
* every API call is captured in :data:`st.session_state.api_usage_log`
with a stable ``task_name``,
* the UI can show a single "API Usage Status" panel that knows which
stages really used the API,
* the API key is never returned, logged, or echoed.
Return value
------------
Every call returns an :class:`LLMCallResult` with::
success: bool
task_name: str
provider: str | None
model: str | None
used_api: bool
response_text: str | None
response_json: dict | list | None
error_message: str | None
timestamp: str # ISO 8601, UTC
estimated_input_tokens: int | None
estimated_output_tokens: int | None
status: str # "ok" | "no_api" | "invalid_key" | "access_denied"
# | "quota" | "connection" | "model_missing"
# | "parse_error" | "unsupported_provider"
# | "skipped" | "exception"
request_id: str | None # provider-side call id (req_…), for support
http_status: int | None # exact HTTP status from the SDK exception
error_type: str | None # provider's machine-readable error code
``used_api`` is true only when bytes actually went over the wire (or were
attempted). Callers that fall back to local logic set it to ``False`` via
:func:`record_local_use`.
Every finished call is mirrored to the durable diagnostics log (see
:mod:`app.services.diagnostics`) so a failure survives the page reload that
usually follows it and can be handed to the app team as one redacted report.
"""
from __future__ import annotations
import base64
import json
import logging
import os
import re
import threading
import time
from dataclasses import asdict, dataclass, field
from datetime import datetime, timezone
from typing import Any, Iterable, Optional
from app.config import (
GROQ_VISION_UNSUPPORTED_MESSAGE,
INTERVIEW_PROVIDER,
LLM_TASK_NAMES,
Settings,
get_settings,
)
from app.utils.logging_utils import get_logger
from app.utils.redaction import (
API_KEY_PATTERNS as _API_KEY_PATTERNS,
ORG_ID_PATTERNS as _ORG_ID_PATTERNS,
redact as _redact,
)
_logger: logging.Logger = get_logger("upwork_strategist.llm_client")
# Stable status codes — surfaced verbatim by the UI / api_gate.
STATUS_OK = "ok"
STATUS_NO_API = "no_api"
STATUS_INVALID_KEY = "invalid_key"
# HTTP 403: the key works but may not use this model (or this service) — a
# plan, billing or organisation-verification gate, not a mistyped key.
STATUS_ACCESS_DENIED = "access_denied"
STATUS_QUOTA = "quota"
STATUS_CONTEXT_OVERFLOW = "context_overflow"
STATUS_CONNECTION = "connection"
STATUS_MODEL_MISSING = "model_missing"
STATUS_PARSE_ERROR = "parse_error"
STATUS_TRUNCATED = "truncated"
STATUS_UNSUPPORTED_PROVIDER = "unsupported_provider"
STATUS_SKIPPED = "skipped"
STATUS_EXCEPTION = "exception"
_SUPPORTED_PROVIDERS = ("anthropic", "openai", "groq", "gemini")
# Providers that can read images. Groq is text-only here, so it is excluded
# and handled with a dedicated, actionable message in ``call_vision_llm``.
_SUPPORTED_VISION_PROVIDERS = ("anthropic", "openai", "gemini")