thio / core /utils /distributed_lock.py
kyrlos adel
cva1
7cfdc4c
Raw History Blame Contribute Delete
15.5 kB
import asyncio
import uuid
from typing import Optional
from datetime import datetime, timezone, timedelta
from contextlib import asynccontextmanager
from core.utils.logger import logger
from core.services.supabase import DBConnection
class DistributedLock:
def __init__(self, lock_key: str, timeout_seconds: int = 300, holder_id: Optional[str] = None):
self.lock_key = lock_key
self.timeout_seconds = timeout_seconds
self.holder_id = holder_id or f"{uuid.uuid4()}"
self.db = DBConnection()
self._acquired = False
async def acquire(self, wait: bool = False, wait_timeout: int = 30) -> bool:
client = await self.db.client
start_time = datetime.now(timezone.utc)
while True:
try:
result = await client.rpc('acquire_distributed_lock', {
'p_lock_key': self.lock_key,
'p_holder_id': self.holder_id,
'p_timeout_seconds': self.timeout_seconds
}).execute()
if result.data:
self._acquired = True
logger.info(f"[LOCK] Acquired lock: {self.lock_key} by {self.holder_id}")
return True
if not wait:
logger.warning(f"[LOCK] Failed to acquire lock (no wait): {self.lock_key}")
return False
elapsed = (datetime.now(timezone.utc) - start_time).total_seconds()
if elapsed >= wait_timeout:
logger.warning(f"[LOCK] Lock acquisition timeout after {elapsed}s: {self.lock_key}")
return False
await asyncio.sleep(0.5)
except Exception as e:
logger.error(f"[LOCK] Error acquiring lock {self.lock_key}: {e}")
if not wait:
return False
await asyncio.sleep(1)
async def release(self) -> bool:
if not self._acquired:
return True
try:
client = await self.db.client
result = await client.rpc('release_distributed_lock', {
'p_lock_key': self.lock_key,
'p_holder_id': self.holder_id
}).execute()
self._acquired = False
logger.info(f"[LOCK] Released lock: {self.lock_key} by {self.holder_id}")
return result.data if result.data is not None else False
except Exception as e:
logger.error(f"[LOCK] Error releasing lock {self.lock_key}: {e}")
return False
async def __aenter__(self):
acquired = await self.acquire(wait=True, wait_timeout=30)
if not acquired:
raise RuntimeError(f"Failed to acquire lock: {self.lock_key}")
return self
async def __aexit__(self, exc_type, exc_val, exc_tb):
await self.release()
@asynccontextmanager
async def distributed_lock(lock_key: str, timeout_seconds: int = 300, wait: bool = True):
lock = DistributedLock(lock_key, timeout_seconds)
try:
acquired = await lock.acquire(wait=wait, wait_timeout=30)
if not acquired:
raise RuntimeError(f"Failed to acquire lock: {lock_key}")
yield lock
finally:
await lock.release()
class RenewalLock:
@staticmethod
async def lock_renewal_processing(account_id: str, period_start: int) -> DistributedLock:
lock_key = f"renewal:{account_id}:{period_start}"
return DistributedLock(lock_key, timeout_seconds=300)
@staticmethod
async def check_and_mark_renewal_processed(
account_id: str,
period_start: int,
period_end: int,
subscription_id: str,
credits_granted: float,
processed_by: str,
stripe_event_id: Optional[str] = None
) -> bool:
# Use singleton - already initialized at startup
db = DBConnection()
client = await db.client
existing = await client.from_('renewal_processing').select('id, processed_by, credits_granted').eq(
'account_id', account_id
).eq('period_start', period_start).execute()
if existing.data:
logger.warning(
f"[RENEWAL BLOCK] Period {period_start} for account {account_id} "
f"already processed by {existing.data[0]['processed_by']} "
f"(granted ${existing.data[0]['credits_granted']})"
)
return False
try:
await client.from_('renewal_processing').insert({
'account_id': account_id,
'period_start': period_start,
'period_end': period_end,
'subscription_id': subscription_id,
'processed_by': processed_by,
'credits_granted': credits_granted,
'stripe_event_id': stripe_event_id
}).execute()
logger.info(
f"[RENEWAL TRACK] Marked period {period_start} as processed by {processed_by} "
f"for account {account_id} (${credits_granted} credits)"
)
return True
except Exception as e:
logger.error(f"[RENEWAL TRACK] Failed to mark renewal processed: {e}")
return False
class WebhookLock:
@staticmethod
async def check_and_mark_webhook_processing(
event_id: str,
event_type: str,
payload: dict = None,
force_reprocess: bool = False
) -> tuple[bool, Optional[str]]:
# Use singleton - already initialized at startup
db = DBConnection()
client = await db.client
existing = await client.from_('webhook_events').select('id, status, processed_at, processing_started_at, retry_count').eq(
'event_id', event_id
).execute()
if existing.data:
event = existing.data[0]
if event['status'] == 'completed':
if force_reprocess:
logger.warning(f"[WEBHOOK] Event {event_id} marked as completed but force_reprocess=True - resetting to processing")
await client.from_('webhook_events').update({
'status': 'processing',
'processing_started_at': datetime.now(timezone.utc).isoformat(),
'retry_count': event.get('retry_count', 0) + 1,
'error_message': None # Clear any previous error
}).eq('id', event['id']).execute()
return True, None
logger.info(f"[WEBHOOK] Event {event_id} already completed at {event['processed_at']}")
return False, 'already_completed'
elif event['status'] == 'processing':
# Check if webhook is stuck (processing for more than 5 minutes)
processing_started = event.get('processing_started_at')
if processing_started:
try:
started_at = datetime.fromisoformat(processing_started.replace('Z', '+00:00'))
timeout_threshold = datetime.now(timezone.utc) - timedelta(minutes=5)
if started_at < timeout_threshold:
logger.warning(
f"[WEBHOOK] Event {event_id} stuck in processing since {processing_started}, "
f"resetting to allow retry"
)
await client.from_('webhook_events').update({
'status': 'failed',
'error_message': 'Webhook processing timeout - stuck in processing for >5 minutes',
'retry_count': event.get('retry_count', 0) + 1
}).eq('id', event['id']).execute()
# Now retry it
await client.from_('webhook_events').update({
'status': 'processing',
'processing_started_at': datetime.now(timezone.utc).isoformat(),
'retry_count': event.get('retry_count', 0) + 1
}).eq('id', event['id']).execute()
return True, None
except Exception as e:
logger.error(f"[WEBHOOK] Error checking timeout for event {event_id}: {e}")
logger.warning(f"[WEBHOOK] Event {event_id} is currently being processed")
return False, 'in_progress'
elif event['status'] == 'failed':
logger.info(f"[WEBHOOK] Retrying previously failed event {event_id}")
await client.from_('webhook_events').update({
'status': 'processing',
'processing_started_at': datetime.now(timezone.utc).isoformat(),
'retry_count': event.get('retry_count', 0) + 1
}).eq('id', event['id']).execute()
return True, None
else:
try:
await client.from_('webhook_events').insert({
'event_id': event_id,
'event_type': event_type,
'status': 'processing',
'processing_started_at': datetime.now(timezone.utc).isoformat(),
'payload': payload
}).execute()
logger.info(f"[WEBHOOK] Started processing new event {event_id}")
return True, None
except Exception as e:
if 'duplicate key' in str(e).lower() or 'unique' in str(e).lower():
logger.warning(f"[WEBHOOK] Race condition detected for event {event_id}, checking if existing event is stuck...")
# Race condition - another process inserted first, check if it's stuck
existing_after_race = await client.from_('webhook_events').select(
'id, status, processing_started_at, retry_count'
).eq('event_id', event_id).execute()
if existing_after_race.data:
race_event = existing_after_race.data[0]
# If the winner already completed, we're done
if race_event['status'] == 'completed':
logger.info(f"[WEBHOOK] Race condition resolved - event {event_id} already completed")
return False, 'already_completed'
# If the winner already failed, we can retry
if race_event['status'] == 'failed':
logger.info(f"[WEBHOOK] Race condition resolved - event {event_id} failed, retrying")
await client.from_('webhook_events').update({
'status': 'processing',
'processing_started_at': datetime.now(timezone.utc).isoformat(),
'retry_count': race_event.get('retry_count', 0) + 1
}).eq('id', race_event['id']).execute()
return True, None
# Winner is still processing - wait briefly and check again
if race_event['status'] == 'processing':
logger.info(f"[WEBHOOK] Race condition - event {event_id} being processed, waiting to verify...")
await asyncio.sleep(3) # Wait 3 seconds for the other process
# Re-check status
recheck = await client.from_('webhook_events').select(
'id, status, processing_started_at, retry_count'
).eq('event_id', event_id).execute()
if recheck.data:
final_status = recheck.data[0]['status']
if final_status == 'completed':
logger.info(f"[WEBHOOK] Race condition resolved - event {event_id} completed while waiting")
return False, 'already_completed'
elif final_status == 'failed':
logger.info(f"[WEBHOOK] Race condition resolved - event {event_id} failed, retrying")
await client.from_('webhook_events').update({
'status': 'processing',
'processing_started_at': datetime.now(timezone.utc).isoformat(),
'retry_count': recheck.data[0].get('retry_count', 0) + 1
}).eq('id', recheck.data[0]['id']).execute()
return True, None
# Still processing after wait - tell caller to signal retry
logger.warning(f"[WEBHOOK] Event {event_id} still processing after wait, signaling retry needed")
return False, 'processing_retry_later'
logger.warning(f"[WEBHOOK] Race condition detected for event {event_id}, signaling retry needed")
return False, 'processing_retry_later'
raise
return True, None
@staticmethod
async def mark_webhook_completed(event_id: str):
# Use singleton - already initialized at startup
db = DBConnection()
client = await db.client
await client.from_('webhook_events').update({
'status': 'completed',
'processed_at': datetime.now(timezone.utc).isoformat()
}).eq('event_id', event_id).execute()
logger.info(f"[WEBHOOK] Marked event {event_id} as completed")
@staticmethod
async def mark_webhook_failed(event_id: str, error_message: str):
# Use singleton - already initialized at startup
db = DBConnection()
client = await db.client
try:
safe_error_message = str(error_message)[:2000]
import re
safe_error_message = re.sub(r'[^\x00-\x7F]+', ' ', safe_error_message)
safe_error_message = safe_error_message.replace('\x00', '')
except Exception:
safe_error_message = "Error message could not be serialized"
try:
await client.from_('webhook_events').update({
'status': 'failed',
'error_message': safe_error_message,
'processed_at': datetime.now(timezone.utc).isoformat()
}).eq('event_id', event_id).execute()
logger.error(f"[WEBHOOK] Marked event {event_id} as failed: {safe_error_message}")
except Exception as db_error:
logger.error(f"[WEBHOOK] Failed to mark event {event_id} as failed in database: {db_error}")
logger.error(f"[WEBHOOK] Original error was: {safe_error_message}")