"""Detect rate-limit / transient API errors for key rotation and backoff.""" from __future__ import annotations import random import time from typing import Callable, TypeVar T = TypeVar("T") def is_rate_limit_error(exc: BaseException) -> bool: """True for HTTP 429, OpenAI-style rate limits, and common Zep wording.""" try: code = getattr(exc, "status_code", None) if code is None and hasattr(exc, "response"): code = getattr(exc.response, "status_code", None) if code == 429: return True except Exception: pass name = type(exc).__name__.lower() if "ratelimit" in name or "rate_limit" in name: return True msg = str(exc).lower() if "429" in msg: return True if "rate limit" in msg or "too many requests" in msg: return True if "quota" in msg and ("exceed" in msg or "exceeded" in msg): return True return False def is_transient_server_error(exc: BaseException) -> bool: try: code = getattr(exc, "status_code", None) if code is None and hasattr(exc, "response"): code = getattr(exc.response, "status_code", None) if code in (500, 502, 503, 504): return True except Exception: pass return False def sleep_backoff(attempt: float, cap: float = 60.0) -> None: """Exponential backoff with jitter (seconds).""" base = min(cap, (2**min(attempt, 6)) + random.uniform(0, 1.5)) time.sleep(base) def run_with_key_rotation( keys: list[str], fn_per_key: Callable[[str], T], *, per_key_attempts: int = 3, op_name: str = "api", ) -> T: """ For each key, retry fn_per_key(key) with backoff on rate limits / 5xx. Then try the next key. Re-raises the last error if nothing succeeds. """ from ..utils.logger import get_logger log = get_logger("mirofish.api_retry") keys = [k.strip() for k in keys if k and str(k).strip()] if not keys: raise ValueError("No API keys configured") last: BaseException | None = None for ki, key in enumerate(keys): for attempt in range(per_key_attempts): try: return fn_per_key(key) except Exception as e: last = e retryable = is_rate_limit_error(e) or is_transient_server_error(e) if not retryable: raise log.warning( "%s retryable error (key %s/%s, attempt %s/%s): %s", op_name, ki + 1, len(keys), attempt + 1, per_key_attempts, e, ) if attempt < per_key_attempts - 1: sleep_backoff(float(attempt)) continue if ki < len(keys) - 1: log.warning("%s: switching to next API key after %s attempts", op_name, per_key_attempts) break raise if last: raise last raise RuntimeError(f"{op_name}: failed without exception")