Download core/utils/distributed_lock.py from Kyrlos11/nniback: direct link, hf CLI and curl.
- Browser
- Download file 15.5 kB
-
https://huggingface.co/spaces/Kyrlos11/nniback/resolve/main/core/utils/distributed_lock.py
- Command line
-
hf download hf://spaces/Kyrlos11/nniback/core/utils/distributed_lock.py
-
curl -L -o distributed_lock.py https://huggingface.co/spaces/Kyrlos11/nniback/resolve/main/core/utils/distributed_lock.py
15.5 kB
| import asyncio | |
| import uuid | |
| from typing import Optional | |
| from datetime import datetime, timezone, timedelta | |
| from contextlib import asynccontextmanager | |
| from core.utils.logger import logger | |
| from core.services.supabase import DBConnection | |
| class DistributedLock: | |
| def __init__(self, lock_key: str, timeout_seconds: int = 300, holder_id: Optional[str] = None): | |
| self.lock_key = lock_key | |
| self.timeout_seconds = timeout_seconds | |
| self.holder_id = holder_id or f"{uuid.uuid4()}" | |
| self.db = DBConnection() | |
| self._acquired = False | |
| async def acquire(self, wait: bool = False, wait_timeout: int = 30) -> bool: | |
| client = await self.db.client | |
| start_time = datetime.now(timezone.utc) | |
| while True: | |
| try: | |
| result = await client.rpc('acquire_distributed_lock', { | |
| 'p_lock_key': self.lock_key, | |
| 'p_holder_id': self.holder_id, | |
| 'p_timeout_seconds': self.timeout_seconds | |
| }).execute() | |
| if result.data: | |
| self._acquired = True | |
| logger.info(f"[LOCK] Acquired lock: {self.lock_key} by {self.holder_id}") | |
| return True | |
| if not wait: | |
| logger.warning(f"[LOCK] Failed to acquire lock (no wait): {self.lock_key}") | |
| return False | |
| elapsed = (datetime.now(timezone.utc) - start_time).total_seconds() | |
| if elapsed >= wait_timeout: | |
| logger.warning(f"[LOCK] Lock acquisition timeout after {elapsed}s: {self.lock_key}") | |
| return False | |
| await asyncio.sleep(0.5) | |
| except Exception as e: | |
| logger.error(f"[LOCK] Error acquiring lock {self.lock_key}: {e}") | |
| if not wait: | |
| return False | |
| await asyncio.sleep(1) | |
| async def release(self) -> bool: | |
| if not self._acquired: | |
| return True | |
| try: | |
| client = await self.db.client | |
| result = await client.rpc('release_distributed_lock', { | |
| 'p_lock_key': self.lock_key, | |
| 'p_holder_id': self.holder_id | |
| }).execute() | |
| self._acquired = False | |
| logger.info(f"[LOCK] Released lock: {self.lock_key} by {self.holder_id}") | |
| return result.data if result.data is not None else False | |
| except Exception as e: | |
| logger.error(f"[LOCK] Error releasing lock {self.lock_key}: {e}") | |
| return False | |
| async def __aenter__(self): | |
| acquired = await self.acquire(wait=True, wait_timeout=30) | |
| if not acquired: | |
| raise RuntimeError(f"Failed to acquire lock: {self.lock_key}") | |
| return self | |
| async def __aexit__(self, exc_type, exc_val, exc_tb): | |
| await self.release() | |
| async def distributed_lock(lock_key: str, timeout_seconds: int = 300, wait: bool = True): | |
| lock = DistributedLock(lock_key, timeout_seconds) | |
| try: | |
| acquired = await lock.acquire(wait=wait, wait_timeout=30) | |
| if not acquired: | |
| raise RuntimeError(f"Failed to acquire lock: {lock_key}") | |
| yield lock | |
| finally: | |
| await lock.release() | |
| class RenewalLock: | |
| async def lock_renewal_processing(account_id: str, period_start: int) -> DistributedLock: | |
| lock_key = f"renewal:{account_id}:{period_start}" | |
| return DistributedLock(lock_key, timeout_seconds=300) | |
| async def check_and_mark_renewal_processed( | |
| account_id: str, | |
| period_start: int, | |
| period_end: int, | |
| subscription_id: str, | |
| credits_granted: float, | |
| processed_by: str, | |
| stripe_event_id: Optional[str] = None | |
| ) -> bool: | |
| # Use singleton - already initialized at startup | |
| db = DBConnection() | |
| client = await db.client | |
| existing = await client.from_('renewal_processing').select('id, processed_by, credits_granted').eq( | |
| 'account_id', account_id | |
| ).eq('period_start', period_start).execute() | |
| if existing.data: | |
| logger.warning( | |
| f"[RENEWAL BLOCK] Period {period_start} for account {account_id} " | |
| f"already processed by {existing.data[0]['processed_by']} " | |
| f"(granted ${existing.data[0]['credits_granted']})" | |
| ) | |
| return False | |
| try: | |
| await client.from_('renewal_processing').insert({ | |
| 'account_id': account_id, | |
| 'period_start': period_start, | |
| 'period_end': period_end, | |
| 'subscription_id': subscription_id, | |
| 'processed_by': processed_by, | |
| 'credits_granted': credits_granted, | |
| 'stripe_event_id': stripe_event_id | |
| }).execute() | |
| logger.info( | |
| f"[RENEWAL TRACK] Marked period {period_start} as processed by {processed_by} " | |
| f"for account {account_id} (${credits_granted} credits)" | |
| ) | |
| return True | |
| except Exception as e: | |
| logger.error(f"[RENEWAL TRACK] Failed to mark renewal processed: {e}") | |
| return False | |
| class WebhookLock: | |
| async def check_and_mark_webhook_processing( | |
| event_id: str, | |
| event_type: str, | |
| payload: dict = None, | |
| force_reprocess: bool = False | |
| ) -> tuple[bool, Optional[str]]: | |
| # Use singleton - already initialized at startup | |
| db = DBConnection() | |
| client = await db.client | |
| existing = await client.from_('webhook_events').select('id, status, processed_at, processing_started_at, retry_count').eq( | |
| 'event_id', event_id | |
| ).execute() | |
| if existing.data: | |
| event = existing.data[0] | |
| if event['status'] == 'completed': | |
| if force_reprocess: | |
| logger.warning(f"[WEBHOOK] Event {event_id} marked as completed but force_reprocess=True - resetting to processing") | |
| await client.from_('webhook_events').update({ | |
| 'status': 'processing', | |
| 'processing_started_at': datetime.now(timezone.utc).isoformat(), | |
| 'retry_count': event.get('retry_count', 0) + 1, | |
| 'error_message': None # Clear any previous error | |
| }).eq('id', event['id']).execute() | |
| return True, None | |
| logger.info(f"[WEBHOOK] Event {event_id} already completed at {event['processed_at']}") | |
| return False, 'already_completed' | |
| elif event['status'] == 'processing': | |
| # Check if webhook is stuck (processing for more than 5 minutes) | |
| processing_started = event.get('processing_started_at') | |
| if processing_started: | |
| try: | |
| started_at = datetime.fromisoformat(processing_started.replace('Z', '+00:00')) | |
| timeout_threshold = datetime.now(timezone.utc) - timedelta(minutes=5) | |
| if started_at < timeout_threshold: | |
| logger.warning( | |
| f"[WEBHOOK] Event {event_id} stuck in processing since {processing_started}, " | |
| f"resetting to allow retry" | |
| ) | |
| await client.from_('webhook_events').update({ | |
| 'status': 'failed', | |
| 'error_message': 'Webhook processing timeout - stuck in processing for >5 minutes', | |
| 'retry_count': event.get('retry_count', 0) + 1 | |
| }).eq('id', event['id']).execute() | |
| # Now retry it | |
| await client.from_('webhook_events').update({ | |
| 'status': 'processing', | |
| 'processing_started_at': datetime.now(timezone.utc).isoformat(), | |
| 'retry_count': event.get('retry_count', 0) + 1 | |
| }).eq('id', event['id']).execute() | |
| return True, None | |
| except Exception as e: | |
| logger.error(f"[WEBHOOK] Error checking timeout for event {event_id}: {e}") | |
| logger.warning(f"[WEBHOOK] Event {event_id} is currently being processed") | |
| return False, 'in_progress' | |
| elif event['status'] == 'failed': | |
| logger.info(f"[WEBHOOK] Retrying previously failed event {event_id}") | |
| await client.from_('webhook_events').update({ | |
| 'status': 'processing', | |
| 'processing_started_at': datetime.now(timezone.utc).isoformat(), | |
| 'retry_count': event.get('retry_count', 0) + 1 | |
| }).eq('id', event['id']).execute() | |
| return True, None | |
| else: | |
| try: | |
| await client.from_('webhook_events').insert({ | |
| 'event_id': event_id, | |
| 'event_type': event_type, | |
| 'status': 'processing', | |
| 'processing_started_at': datetime.now(timezone.utc).isoformat(), | |
| 'payload': payload | |
| }).execute() | |
| logger.info(f"[WEBHOOK] Started processing new event {event_id}") | |
| return True, None | |
| except Exception as e: | |
| if 'duplicate key' in str(e).lower() or 'unique' in str(e).lower(): | |
| logger.warning(f"[WEBHOOK] Race condition detected for event {event_id}, checking if existing event is stuck...") | |
| # Race condition - another process inserted first, check if it's stuck | |
| existing_after_race = await client.from_('webhook_events').select( | |
| 'id, status, processing_started_at, retry_count' | |
| ).eq('event_id', event_id).execute() | |
| if existing_after_race.data: | |
| race_event = existing_after_race.data[0] | |
| # If the winner already completed, we're done | |
| if race_event['status'] == 'completed': | |
| logger.info(f"[WEBHOOK] Race condition resolved - event {event_id} already completed") | |
| return False, 'already_completed' | |
| # If the winner already failed, we can retry | |
| if race_event['status'] == 'failed': | |
| logger.info(f"[WEBHOOK] Race condition resolved - event {event_id} failed, retrying") | |
| await client.from_('webhook_events').update({ | |
| 'status': 'processing', | |
| 'processing_started_at': datetime.now(timezone.utc).isoformat(), | |
| 'retry_count': race_event.get('retry_count', 0) + 1 | |
| }).eq('id', race_event['id']).execute() | |
| return True, None | |
| # Winner is still processing - wait briefly and check again | |
| if race_event['status'] == 'processing': | |
| logger.info(f"[WEBHOOK] Race condition - event {event_id} being processed, waiting to verify...") | |
| await asyncio.sleep(3) # Wait 3 seconds for the other process | |
| # Re-check status | |
| recheck = await client.from_('webhook_events').select( | |
| 'id, status, processing_started_at, retry_count' | |
| ).eq('event_id', event_id).execute() | |
| if recheck.data: | |
| final_status = recheck.data[0]['status'] | |
| if final_status == 'completed': | |
| logger.info(f"[WEBHOOK] Race condition resolved - event {event_id} completed while waiting") | |
| return False, 'already_completed' | |
| elif final_status == 'failed': | |
| logger.info(f"[WEBHOOK] Race condition resolved - event {event_id} failed, retrying") | |
| await client.from_('webhook_events').update({ | |
| 'status': 'processing', | |
| 'processing_started_at': datetime.now(timezone.utc).isoformat(), | |
| 'retry_count': recheck.data[0].get('retry_count', 0) + 1 | |
| }).eq('id', recheck.data[0]['id']).execute() | |
| return True, None | |
| # Still processing after wait - tell caller to signal retry | |
| logger.warning(f"[WEBHOOK] Event {event_id} still processing after wait, signaling retry needed") | |
| return False, 'processing_retry_later' | |
| logger.warning(f"[WEBHOOK] Race condition detected for event {event_id}, signaling retry needed") | |
| return False, 'processing_retry_later' | |
| raise | |
| return True, None | |
| async def mark_webhook_completed(event_id: str): | |
| # Use singleton - already initialized at startup | |
| db = DBConnection() | |
| client = await db.client | |
| await client.from_('webhook_events').update({ | |
| 'status': 'completed', | |
| 'processed_at': datetime.now(timezone.utc).isoformat() | |
| }).eq('event_id', event_id).execute() | |
| logger.info(f"[WEBHOOK] Marked event {event_id} as completed") | |
| async def mark_webhook_failed(event_id: str, error_message: str): | |
| # Use singleton - already initialized at startup | |
| db = DBConnection() | |
| client = await db.client | |
| try: | |
| safe_error_message = str(error_message)[:2000] | |
| import re | |
| safe_error_message = re.sub(r'[^\x00-\x7F]+', ' ', safe_error_message) | |
| safe_error_message = safe_error_message.replace('\x00', '') | |
| except Exception: | |
| safe_error_message = "Error message could not be serialized" | |
| try: | |
| await client.from_('webhook_events').update({ | |
| 'status': 'failed', | |
| 'error_message': safe_error_message, | |
| 'processed_at': datetime.now(timezone.utc).isoformat() | |
| }).eq('event_id', event_id).execute() | |
| logger.error(f"[WEBHOOK] Marked event {event_id} as failed: {safe_error_message}") | |
| except Exception as db_error: | |
| logger.error(f"[WEBHOOK] Failed to mark event {event_id} as failed in database: {db_error}") | |
| logger.error(f"[WEBHOOK] Original error was: {safe_error_message}") | |