Download core/utils/retry.py from Kyrlos11/nniback: direct link, hf CLI and curl.
- Browser
- Download file 8.18 kB
-
https://huggingface.co/spaces/Kyrlos11/nniback/resolve/main/core/utils/retry.py
- Command line
-
hf download hf://spaces/Kyrlos11/nniback/core/utils/retry.py
-
curl -L -o retry.py https://huggingface.co/spaces/Kyrlos11/nniback/resolve/main/core/utils/retry.py
8.18 kB
| import asyncio | |
| import os | |
| import random | |
| from typing import TypeVar, Callable, Awaitable, Optional, Tuple, Type | |
| import httpx | |
| from core.utils.logger import logger | |
| from core.services.supabase import DBConnection | |
| T = TypeVar("T") | |
| DB_DEFAULT_MAX_RETRIES = 6 | |
| DB_RETRY_INITIAL_DELAY = 0.5 | |
| DB_RETRY_MAX_DELAY = 10.0 | |
| DB_RETRY_JITTER_FACTOR = 0.3 | |
| DB_RETRYABLE_EXCEPTIONS: Tuple[Type[Exception], ...] = ( | |
| httpx.ConnectTimeout, | |
| httpx.ReadTimeout, | |
| httpx.PoolTimeout, | |
| httpx.ConnectError, | |
| httpx.NetworkError, | |
| ConnectionError, | |
| TimeoutError, | |
| ) | |
| def _add_jitter(delay: float, jitter_factor: float = DB_RETRY_JITTER_FACTOR) -> float: | |
| """Add random jitter to delay to prevent thundering herd.""" | |
| jitter = delay * jitter_factor * random.random() | |
| return delay + jitter | |
| async def retry( | |
| fn: Callable[[], Awaitable[T]], | |
| max_attempts: int = 3, | |
| delay_seconds: int = 1, | |
| backoff_factor: float = 2.0, | |
| max_delay: Optional[float] = None, | |
| retryable_exceptions: Optional[Tuple[Type[Exception], ...]] = None, | |
| ) -> T: | |
| """ | |
| Retry an async function with exponential backoff. | |
| Args: | |
| fn: The async function to retry | |
| max_attempts: Maximum number of attempts | |
| delay_seconds: Initial delay between attempts in seconds | |
| backoff_factor: Multiplier for exponential backoff (default: 2.0) | |
| max_delay: Maximum delay between retries (None = no limit) | |
| retryable_exceptions: Tuple of exception types to retry on (None = retry on all exceptions) | |
| Returns: | |
| The result of the function call | |
| Raises: | |
| The last exception if all attempts fail, or immediately for non-retryable exceptions | |
| Example: | |
| ```python | |
| async def fetch_data(): | |
| # Some operation that might fail | |
| return await api_call() | |
| try: | |
| result = await retry(fetch_data, max_attempts=3, delay_seconds=2) | |
| print(f"Success: {result}") | |
| except Exception as e: | |
| print(f"Failed after all retries: {e}") | |
| ``` | |
| """ | |
| if max_attempts <= 0: | |
| raise ValueError("max_attempts must be greater than zero") | |
| last_error: Optional[Exception] = None | |
| retryable = retryable_exceptions if retryable_exceptions is not None else (Exception,) | |
| for attempt in range(1, max_attempts + 1): | |
| try: | |
| return await fn() | |
| except retryable as error: | |
| last_error = error | |
| if attempt == max_attempts: | |
| break | |
| # Calculate delay with exponential backoff | |
| delay = delay_seconds * (backoff_factor ** (attempt - 1)) | |
| if max_delay is not None: | |
| delay = min(delay, max_delay) | |
| logger.debug( | |
| f"Retry attempt {attempt}/{max_attempts} failed: {type(error).__name__}. " | |
| f"Retrying in {delay:.1f}s..." | |
| ) | |
| await asyncio.sleep(delay) | |
| except Exception as error: | |
| # Non-retryable exception - raise immediately | |
| logger.debug(f"Non-retryable error: {type(error).__name__}: {str(error)}") | |
| raise | |
| if last_error: | |
| raise last_error | |
| raise RuntimeError("Unexpected: last_error is None") | |
| async def retry_db_operation( | |
| operation: Callable[[], Awaitable[T]], | |
| operation_name: Optional[str] = None, | |
| max_retries: Optional[int] = None, | |
| initial_delay: Optional[float] = None, | |
| max_delay: Optional[float] = None, | |
| backoff_factor: float = 2.0, | |
| reset_connection_on_error: bool = True, | |
| reset_on_pool_timeout: bool = True, | |
| ) -> T: | |
| """ | |
| Retry a database operation with exponential backoff, jitter, and connection reset. | |
| Designed for high-concurrency production workloads with: | |
| - Configurable retry limits (default: 5 attempts via DB_DEFAULT_MAX_RETRIES) | |
| - Jitter to prevent thundering herd on retries | |
| - Automatic connection pool reset on timeouts | |
| - Handles ConnectTimeout, ReadTimeout, PoolTimeout, network errors, | |
| client-closed errors, and route-not-found errors | |
| Args: | |
| operation: The async database operation to retry | |
| operation_name: Name for logging purposes (optional) | |
| max_retries: Maximum retry attempts (default: DB_DEFAULT_MAX_RETRIES=5) | |
| initial_delay: Initial delay in seconds (default: DB_RETRY_INITIAL_DELAY=0.5) | |
| max_delay: Maximum delay between retries (default: DB_RETRY_MAX_DELAY=15.0) | |
| backoff_factor: Multiplier for exponential backoff (default: 2.0) | |
| reset_connection_on_error: Reset DB connection on connection errors (default: True) | |
| reset_on_pool_timeout: Reset connection specifically on PoolTimeout (default: True) | |
| Returns: | |
| The result of the operation | |
| Raises: | |
| The last exception if all retries are exhausted, or immediately for non-retryable errors | |
| """ | |
| if max_retries is None: | |
| max_retries = DB_DEFAULT_MAX_RETRIES | |
| if initial_delay is None: | |
| initial_delay = DB_RETRY_INITIAL_DELAY | |
| if max_delay is None: | |
| max_delay = DB_RETRY_MAX_DELAY | |
| last_exception: Optional[Exception] = None | |
| # Use singleton - already initialized at startup | |
| db = DBConnection() | |
| op_name = operation_name or "Database operation" | |
| for attempt in range(max_retries): | |
| try: | |
| return await operation() | |
| except DB_RETRYABLE_EXCEPTIONS as e: | |
| last_exception = e | |
| is_pool_timeout = isinstance(e, httpx.PoolTimeout) | |
| is_connect_timeout = isinstance(e, httpx.ConnectTimeout) | |
| if attempt < max_retries - 1: | |
| should_reset = reset_connection_on_error or (reset_on_pool_timeout and is_pool_timeout) | |
| if should_reset: | |
| try: | |
| await db.force_reconnect() | |
| logger.debug(f"🔄 Reconnected DB after {type(e).__name__}") | |
| except Exception as reset_error: | |
| logger.warning(f"Failed to reconnect: {reset_error}") | |
| base_delay = min(initial_delay * (backoff_factor ** attempt), max_delay) | |
| delay = _add_jitter(base_delay) | |
| if is_pool_timeout or is_connect_timeout: | |
| delay = min(delay * 1.5, max_delay) | |
| logger.warning( | |
| f"{op_name} failed (attempt {attempt + 1}/{max_retries}): {type(e).__name__}. " | |
| f"Retrying in {delay:.2f}s..." | |
| ) | |
| await asyncio.sleep(delay) | |
| else: | |
| logger.error( | |
| f"{op_name} failed after {max_retries} attempts: {type(e).__name__}. " | |
| f"Consider increasing DB_DEFAULT_MAX_RETRIES or SUPABASE_MAX_CONNECTIONS." | |
| ) | |
| except Exception as e: | |
| # Check if this is a recoverable connection error (client-closed, route-not-found) | |
| if DBConnection.is_recoverable_connection_error(e): | |
| last_exception = e | |
| if attempt < max_retries - 1: | |
| error_type = "client-closed" if DBConnection.is_client_closed_error(e) else "route-not-found" | |
| logger.warning(f"🔄 Recoverable {error_type} error in {op_name} (attempt {attempt + 1}/{max_retries}), reconnecting...") | |
| try: | |
| await db.force_reconnect() | |
| except Exception as reconnect_err: | |
| logger.warning(f"Failed to reconnect: {reconnect_err}") | |
| base_delay = min(initial_delay * (backoff_factor ** attempt), max_delay) | |
| delay = _add_jitter(base_delay) | |
| await asyncio.sleep(delay) | |
| else: | |
| logger.error(f"{op_name} failed after {max_retries} attempts with {error_type} error") | |
| else: | |
| logger.error(f"{op_name} failed with non-retryable error: {type(e).__name__}: {str(e)}") | |
| raise | |
| if last_exception: | |
| raise last_exception | |
| raise RuntimeError("Unexpected: retry loop completed without exception") | |