Spaces:
Runtime error
Runtime error
| """ | |
| DocuMint AI - Resilient HF Inference Client. | |
| Wraps huggingface_hub.InferenceClient with: | |
| - Per-request timeout (env: HF_TIMEOUT_SECONDS, default 120) | |
| - Exponential-backoff retry on transient errors (503, 429, timeouts) | |
| - Configurable max retries (env: HF_MAX_RETRIES, default 2) | |
| Usage: | |
| from hf_client import create_inference_client, robust_chat_completion | |
| client = create_inference_client(api_key=token) | |
| response = robust_chat_completion(client, model=model_id, messages=msgs, max_tokens=2048) | |
| """ | |
| import os | |
| import time | |
| import logging | |
| from typing import Optional | |
| from huggingface_hub import InferenceClient | |
| logger = logging.getLogger(__name__) | |
| # --------------------------------------------------------------------------- | |
| # Configuration | |
| # --------------------------------------------------------------------------- | |
| HF_TIMEOUT_SECONDS = int(os.environ.get("HF_TIMEOUT_SECONDS", "120")) | |
| HF_MAX_RETRIES = int(os.environ.get("HF_MAX_RETRIES", "2")) | |
| HF_RETRY_BASE_DELAY = float(os.environ.get("HF_RETRY_BASE_DELAY", "2.0")) | |
| # Transient error indicators (substring match on exception message) | |
| _TRANSIENT_PATTERNS = [ | |
| "503", | |
| "429", | |
| "rate limit", | |
| "service unavailable", | |
| "temporarily unavailable", | |
| "overloaded", | |
| "timeout", | |
| "timed out", | |
| "connection", | |
| "loading", | |
| "model is currently loading", | |
| ] | |
| # --------------------------------------------------------------------------- | |
| # Client factory | |
| # --------------------------------------------------------------------------- | |
| def create_inference_client( | |
| api_key: str, | |
| timeout: Optional[int] = None, | |
| **kwargs, | |
| ) -> InferenceClient: | |
| """Create an InferenceClient with timeout configured. | |
| Reuse this across calls within a request to share the connection. | |
| """ | |
| t = timeout or HF_TIMEOUT_SECONDS | |
| return InferenceClient( | |
| api_key=api_key, | |
| timeout=t, | |
| **kwargs, | |
| ) | |
| # --------------------------------------------------------------------------- | |
| # Retry wrapper | |
| # --------------------------------------------------------------------------- | |
| def _is_transient(exc: Exception) -> bool: | |
| """Check whether an exception looks transient and worth retrying.""" | |
| msg = str(exc).lower() | |
| exc_type = type(exc).__name__.lower() | |
| return any(p in msg or p in exc_type for p in _TRANSIENT_PATTERNS) | |
| def robust_chat_completion( | |
| client: InferenceClient, | |
| *, | |
| max_retries: Optional[int] = None, | |
| base_delay: Optional[float] = None, | |
| **kwargs, | |
| ): | |
| """Call client.chat_completion with exponential-backoff retry. | |
| kwargs are passed directly to chat_completion (model, messages, etc.). | |
| Only retries on transient errors (503, 429, timeouts). | |
| Non-transient errors (400, 401, 422) raise immediately. | |
| Returns the ChatCompletion response on success. | |
| """ | |
| retries = max_retries if max_retries is not None else HF_MAX_RETRIES | |
| delay = base_delay or HF_RETRY_BASE_DELAY | |
| model_id = kwargs.get("model", "unknown") | |
| last_exc = None | |
| for attempt in range(1 + retries): | |
| try: | |
| return client.chat_completion(**kwargs) | |
| except Exception as e: | |
| last_exc = e | |
| if attempt < retries and _is_transient(e): | |
| wait = delay * (2 ** attempt) | |
| logger.warning( | |
| f"Transient error on {model_id} (attempt {attempt + 1}/{1 + retries}): " | |
| f"{type(e).__name__}: {str(e)[:120]}. Retrying in {wait:.1f}s..." | |
| ) | |
| time.sleep(wait) | |
| else: | |
| # Non-transient or last attempt — let it propagate | |
| if attempt > 0: | |
| logger.warning( | |
| f"Giving up on {model_id} after {attempt + 1} attempts: " | |
| f"{type(e).__name__}: {str(e)[:120]}" | |
| ) | |
| raise | |
| # Should not reach here, but just in case | |
| raise last_exc # type: ignore[misc] | |