Download api.py from Kyrlos11/DTTH: direct link, hf CLI and curl.
- Browser
- Download file 28.6 kB
-
https://huggingface.co/spaces/Kyrlos11/DTTH/resolve/main/api.py
- Command line
-
hf download hf://spaces/Kyrlos11/DTTH/api.py
-
curl -L -o api.py https://huggingface.co/spaces/Kyrlos11/DTTH/resolve/main/api.py
28.6 kB
| from dotenv import load_dotenv | |
| load_dotenv() | |
| from fastapi import FastAPI, Request, HTTPException, Response, Depends, APIRouter, Query | |
| from fastapi.middleware.cors import CORSMiddleware | |
| from fastapi.responses import JSONResponse, StreamingResponse | |
| from core.services import redis | |
| from core.utils.openapi_config import configure_openapi | |
| from contextlib import asynccontextmanager | |
| from core.agentpress.thread_manager import ThreadManager | |
| from core.services.supabase import DBConnection | |
| from datetime import datetime, timezone | |
| from core.utils.config import config, EnvMode | |
| import asyncio | |
| from core.utils.logger import logger, structlog | |
| import time | |
| from collections import OrderedDict | |
| import os | |
| import psutil | |
| from pydantic import BaseModel | |
| import uuid | |
| from core.versioning.api import router as versioning_router | |
| from core.agents.api import router as agent_runs_router | |
| from core.agents.agent_crud import router as agent_crud_router | |
| from core.agents.agent_tools import router as agent_tools_router | |
| from core.agents.agent_json import router as agent_json_router | |
| from core.agents.agent_setup import router as agent_setup_router | |
| from core.threads.api import router as threads_router | |
| from core.categorization.api import router as categorization_router | |
| from core.endpoints import router as endpoints_router | |
| from core.sandbox import api as sandbox_api | |
| from core.billing.api import router as billing_router | |
| from core.setup import router as setup_router, webhook_router | |
| from core.admin.admin_api import router as admin_router | |
| from core.admin.billing_admin_api import router as billing_admin_router | |
| from core.admin.feedback_admin_api import router as feedback_admin_router | |
| from core.admin.notification_admin_api import router as notification_admin_router | |
| from core.admin.analytics_admin_api import router as analytics_admin_router | |
| from core.admin.stress_test_admin_api import router as stress_test_admin_router | |
| from core.admin.system_status_admin_api import router as system_status_admin_router | |
| from core.admin.sandbox_pool_admin_api import router as sandbox_pool_admin_router | |
| from core.endpoints.system_status_api import router as system_status_router | |
| from core.services import transcription as transcription_api | |
| import sys | |
| from core.triggers import api as triggers_api | |
| from core.services import api_keys_api | |
| from core.notifications import api as notifications_api | |
| from core.services.orphan_cleanup import cleanup_orphaned_agent_runs | |
| from auth import api as auth_api | |
| from core.utils.auth_utils import verify_and_get_user_id_from_jwt | |
| if sys.platform == "win32": | |
| # Use SelectorEventLoop on Windows for psycopg async compatibility | |
| asyncio.set_event_loop_policy(asyncio.WindowsSelectorEventLoopPolicy()) | |
| db = DBConnection() | |
| # Use shared instance ID for distributed deployments | |
| from core.utils.instance import get_instance_id, INSTANCE_ID | |
| instance_id = INSTANCE_ID # Keep backward compatibility | |
| # Rate limiter state | |
| ip_tracker = OrderedDict() | |
| MAX_CONCURRENT_IPS = 25 | |
| # Background task handles | |
| _worker_metrics_task = None | |
| _memory_watchdog_task = None | |
| _stream_cleanup_task = None | |
| _metrics_logger_task = None | |
| # Graceful shutdown flag for health checks | |
| # When True, health check will return unhealthy to stop receiving traffic | |
| _is_shutting_down = False | |
| async def lifespan(app: FastAPI): | |
| global _worker_metrics_task, _memory_watchdog_task, _stream_cleanup_task, _metrics_logger_task, _is_shutting_down | |
| env_mode = config.ENV_MODE.value if config.ENV_MODE else "unknown" | |
| logger.debug(f"Starting up FastAPI application with instance ID: {instance_id} in {env_mode} mode") | |
| try: | |
| await db.initialize() | |
| from core.services.db import init_db | |
| await init_db() | |
| try: | |
| from core.services.db import execute_one | |
| await execute_one("SELECT 1", {}) | |
| logger.debug("Database connection pool warmed up") | |
| except Exception as e: | |
| logger.warning(f"Failed to warm up database connection pool: {e}") | |
| # Pre-load tool classes and schemas to avoid first-request delay | |
| from core.utils.tool_discovery import warm_up_tools_cache | |
| warm_up_tools_cache() | |
| # Pre-load static Suna config for fast path in API requests | |
| from core.cache.runtime_cache import load_static_suna_config | |
| load_static_suna_config() | |
| sandbox_api.initialize(db) | |
| from core.services import redis | |
| try: | |
| await redis.initialize_async() | |
| logger.debug("Redis connection initialized successfully") | |
| try: | |
| tier_keys = await redis.scan_keys("tier_info:*") | |
| sub_keys = await redis.scan_keys("subscription_tier:*") | |
| all_keys = tier_keys + sub_keys | |
| if all_keys: | |
| for key in all_keys: | |
| await redis.delete(key) | |
| logger.info(f"[STARTUP] Cleared {len(all_keys)} tier caches to pick up config changes") | |
| except Exception as e: | |
| logger.warning(f"[STARTUP] Failed to clear tier caches: {e}") | |
| except Exception as e: | |
| logger.error(f"Failed to initialize Redis connection: {e}") | |
| try: | |
| client = await db.client | |
| await cleanup_orphaned_agent_runs(client) | |
| except Exception as e: | |
| logger.error(f"Failed to cleanup orphaned agent runs on startup: {e}") | |
| if config.ACTIVATE_MCPS_TRIG: | |
| triggers_api.initialize(db) | |
| credentials_api.initialize(db) | |
| composio_api.initialize(db) | |
| template_api.initialize(db) | |
| # Start CloudWatch worker metrics publisher (production only) | |
| if config.ENV_MODE == EnvMode.PRODUCTION: | |
| from core.services import worker_metrics | |
| _worker_metrics_task = asyncio.create_task(worker_metrics.start_cloudwatch_publisher()) | |
| # Start Redis stream cleanup task (catches orphaned streams with no TTL) | |
| from core.services import worker_metrics | |
| _stream_cleanup_task = asyncio.create_task(worker_metrics.start_stream_cleanup_task()) | |
| # Start memory watchdog for observability | |
| _memory_watchdog_task = asyncio.create_task(_memory_watchdog()) | |
| # Log system metrics to CloudWatch every 5 minutes (logger ships to CloudWatch) | |
| _metrics_logger_task = asyncio.create_task(_metrics_logger_loop()) | |
| # Start sandbox pool service (maintains pre-warmed sandboxes) | |
| from core.sandbox.pool_background import start_pool_service | |
| asyncio.create_task(start_pool_service()) | |
| # Start conversation analytics worker | |
| from core.analytics.conversation_analytics_worker import start_analytics_worker | |
| asyncio.create_task(start_analytics_worker()) | |
| # Initialize stateless pipeline | |
| from core.agents.pipeline.stateless import lifecycle | |
| await lifecycle.initialize() | |
| logger.info("[STARTUP] Stateless pipeline initialized") | |
| yield | |
| _is_shutting_down = True | |
| logger.info(f"Starting graceful shutdown for instance {instance_id}") | |
| # Give K8s readiness probe time to detect unhealthy state | |
| # This ensures no new traffic is routed to this pod | |
| await asyncio.sleep(2) | |
| # ===== CRITICAL: Stop all running agent runs on this instance ===== | |
| from core.agents.api import _cancellation_events | |
| from core.agents.runner import update_agent_run_status | |
| active_run_ids = list(_cancellation_events.keys()) | |
| if active_run_ids: | |
| logger.warning(f"🛑 Stopping {len(active_run_ids)} active agent runs on shutdown: {active_run_ids}") | |
| # Set cancellation events for all running runs | |
| for agent_run_id in active_run_ids: | |
| try: | |
| event = _cancellation_events.get(agent_run_id) | |
| if event: | |
| event.set() | |
| logger.info(f"Set cancellation event for {agent_run_id}") | |
| except Exception as e: | |
| logger.error(f"Failed to set cancellation event for {agent_run_id}: {e}") | |
| # Give tasks a moment to handle cancellation gracefully | |
| await asyncio.sleep(1) | |
| # Force update DB status for any runs that didn't clean up | |
| for agent_run_id in active_run_ids: | |
| try: | |
| # Update status to stopped with shutdown message | |
| await update_agent_run_status( | |
| agent_run_id, | |
| "stopped", | |
| error=f"Instance shutdown: {instance_id}" | |
| ) | |
| logger.info(f"✅ Marked agent run {agent_run_id} as stopped (instance shutdown)") | |
| # Also set Redis stop signal for any reconnecting clients | |
| try: | |
| await redis.set_stop_signal(agent_run_id) | |
| except Exception: | |
| pass | |
| except Exception as e: | |
| logger.error(f"Failed to update agent run {agent_run_id} on shutdown: {e}") | |
| else: | |
| logger.info("No active agent runs to stop on shutdown") | |
| # Shutdown stateless pipeline | |
| from core.agents.pipeline.stateless import lifecycle | |
| await lifecycle.shutdown() | |
| logger.info("[SHUTDOWN] Stateless pipeline shutdown complete") | |
| logger.debug("Cleaning up resources") | |
| # Stop CloudWatch worker metrics task | |
| if _worker_metrics_task is not None: | |
| _worker_metrics_task.cancel() | |
| try: | |
| await _worker_metrics_task | |
| except asyncio.CancelledError: | |
| pass | |
| # Stop memory watchdog task | |
| if _memory_watchdog_task is not None: | |
| _memory_watchdog_task.cancel() | |
| try: | |
| await _memory_watchdog_task | |
| except asyncio.CancelledError: | |
| pass | |
| # Stop metrics logger task | |
| if _metrics_logger_task is not None: | |
| _metrics_logger_task.cancel() | |
| try: | |
| await _metrics_logger_task | |
| except asyncio.CancelledError: | |
| pass | |
| # Stop sandbox pool service | |
| from core.sandbox.pool_background import stop_pool_service | |
| await stop_pool_service() | |
| # Stop conversation analytics worker | |
| from core.analytics.conversation_analytics_worker import stop_analytics_worker | |
| await stop_analytics_worker() | |
| try: | |
| logger.debug("Closing Redis connection") | |
| await redis.close() | |
| logger.debug("Redis connection closed successfully") | |
| except Exception as e: | |
| logger.error(f"Error closing Redis connection: {e}") | |
| logger.debug("Disconnecting from database") | |
| await db.disconnect() | |
| # Close direct Postgres connection pool | |
| from core.services.db import close_db | |
| await close_db() | |
| except Exception as e: | |
| logger.error(f"Error during application startup: {e}") | |
| raise | |
| app = FastAPI( | |
| lifespan=lifespan, | |
| swagger_ui_parameters={ | |
| "persistAuthorization": True, # Keep auth between page refreshes | |
| }, | |
| ) | |
| # Configure OpenAPI docs with API Key and Bearer token auth | |
| configure_openapi(app) | |
| async def log_requests_middleware(request: Request, call_next): | |
| structlog.contextvars.clear_contextvars() | |
| request_id = str(uuid.uuid4()) | |
| start_time = time.time() | |
| client_ip = request.client.host if request.client else "unknown" | |
| method = request.method | |
| path = request.url.path | |
| query_params = str(request.query_params) | |
| structlog.contextvars.bind_contextvars( | |
| request_id=request_id, | |
| client_ip=client_ip, | |
| method=method, | |
| path=path, | |
| query_params=query_params | |
| ) | |
| # Log the incoming request | |
| logger.debug(f"Request started: {method} {path} from {client_ip} | Query: {query_params}") | |
| try: | |
| response = await call_next(request) | |
| process_time = time.time() - start_time | |
| logger.debug(f"Request completed: {method} {path} | Status: {response.status_code} | Time: {process_time:.2f}s") | |
| return response | |
| except Exception as e: | |
| process_time = time.time() - start_time | |
| try: | |
| error_str = str(e) | |
| except Exception: | |
| error_str = f"Error of type {type(e).__name__}" | |
| logger.error(f"Request failed: {method} {path} | Error: {error_str} | Time: {process_time:.2f}s") | |
| raise | |
| # Define allowed origins based on environment | |
| allowed_origins = [ | |
| "https://www.kortix.com", | |
| "https://kortix.com", | |
| "https://dev.kortix.com", | |
| "https://staging.kortix.com", | |
| "https://prod-test.kortix.com", | |
| ] | |
| # Allow all *.kortix.com subdomains and Vercel preview deployments | |
| allow_origin_regex = r"https://([a-z0-9-]+\.)?kortix\.com|https://.*-kortixai\.vercel\.app" | |
| # Add local origins for development | |
| if config.ENV_MODE == EnvMode.LOCAL: | |
| allowed_origins.append("http://localhost:3000") | |
| allowed_origins.append("http://127.0.0.1:3000") | |
| # Add staging-specific origins | |
| if config.ENV_MODE == EnvMode.STAGING: | |
| allowed_origins.append("https://staging.kortix.com") | |
| allowed_origins.append("http://localhost:3000") | |
| app.add_middleware( | |
| CORSMiddleware, | |
| allow_origins=allowed_origins, | |
| allow_origin_regex=allow_origin_regex, | |
| allow_credentials=True, | |
| allow_methods=["GET", "POST", "PUT", "PATCH", "DELETE", "OPTIONS"], | |
| allow_headers=["Content-Type", "Authorization", "X-Project-Id", "X-MCP-URL", "X-MCP-Type", "X-MCP-Headers", "X-API-Key"], | |
| ) | |
| # Create a main API router | |
| api_router = APIRouter() | |
| # Include all API routers without individual prefixes | |
| # Core routers | |
| api_router.include_router(versioning_router) | |
| api_router.include_router(agent_runs_router) | |
| api_router.include_router(agent_crud_router) | |
| api_router.include_router(agent_tools_router) | |
| api_router.include_router(agent_json_router) | |
| api_router.include_router(agent_setup_router) | |
| api_router.include_router(threads_router) | |
| api_router.include_router(categorization_router) | |
| api_router.include_router(endpoints_router) | |
| api_router.include_router(sandbox_api.router) | |
| api_router.include_router(billing_router) | |
| api_router.include_router(setup_router) | |
| api_router.include_router(webhook_router) # Webhooks at /api/webhooks/* | |
| api_router.include_router(api_keys_api.router) | |
| api_router.include_router(billing_admin_router) | |
| api_router.include_router(admin_router) | |
| api_router.include_router(feedback_admin_router) | |
| api_router.include_router(notification_admin_router) | |
| api_router.include_router(analytics_admin_router) | |
| api_router.include_router(stress_test_admin_router) | |
| api_router.include_router(system_status_admin_router) | |
| api_router.include_router(sandbox_pool_admin_router) | |
| api_router.include_router(system_status_router) | |
| from core.mcp_module import api as mcp_api | |
| from core.credentials import api as credentials_api | |
| from core.templates import api as template_api | |
| from core.templates import presentations_api | |
| if config.ACTIVATE_MCPS_TRIG: | |
| api_router.include_router(mcp_api.router) | |
| api_router.include_router(credentials_api.router, prefix="/secure-mcp") | |
| api_router.include_router(template_api.router, prefix="/templates") | |
| api_router.include_router(presentations_api.router, prefix="/presentation-templates") | |
| api_router.include_router(transcription_api.router) | |
| from core.services import voice_generation as voice_api | |
| api_router.include_router(voice_api.router) | |
| from core.knowledge_base import api as knowledge_base_api | |
| api_router.include_router(knowledge_base_api.router) | |
| if config.ACTIVATE_MCPS_TRIG: | |
| api_router.include_router(triggers_api.router) | |
| api_router.include_router(notifications_api.router) | |
| from core.notifications import presence_api | |
| api_router.include_router(presence_api.router) | |
| from core.composio_integration import api as composio_api | |
| if config.ACTIVATE_MCPS_TRIG: | |
| api_router.include_router(composio_api.router) | |
| from core.google.google_slides_api import router as google_slides_router | |
| api_router.include_router(google_slides_router) | |
| from core.google.google_docs_api import router as google_docs_router | |
| api_router.include_router(google_docs_router) | |
| from core.referrals import router as referrals_router | |
| from core.memory.api import router as memory_router | |
| api_router.include_router(referrals_router) | |
| api_router.include_router(memory_router) | |
| from core.test_harness.api import router as test_harness_router, e2e_router | |
| api_router.include_router(test_harness_router) | |
| api_router.include_router(e2e_router) | |
| from core.sandbox.canvas_ai_api import router as canvas_ai_router | |
| api_router.include_router(canvas_ai_router) | |
| from core.admin.stateless_admin_api import router as stateless_admin_router | |
| api_router.include_router(stateless_admin_router) | |
| # Auth OTP endpoint for expired magic links | |
| api_router.include_router(auth_api.router) | |
| async def health_check(): | |
| logger.debug("Health check endpoint called") | |
| # During shutdown, return unhealthy status | |
| # This causes K8s readinessProbe to fail and removes pod from service endpoints | |
| if _is_shutting_down: | |
| logger.debug(f"Health check returning unhealthy (shutting down) for instance {instance_id}") | |
| raise HTTPException( | |
| status_code=503, | |
| detail={ | |
| "status": "shutting_down", | |
| "timestamp": datetime.now(timezone.utc).isoformat(), | |
| "instance_id": instance_id | |
| } | |
| ) | |
| return { | |
| "status": "ok", | |
| "timestamp": datetime.now(timezone.utc).isoformat(), | |
| "instance_id": instance_id, | |
| } | |
| async def prewarm_user_caches(user_id: str = Depends(verify_and_get_user_id_from_jwt)): | |
| async def _do_prewarm(): | |
| try: | |
| from core.cache.runtime_cache import prewarm_user_agents | |
| await prewarm_user_agents(user_id) | |
| except Exception as e: | |
| logger.warning(f"[PREWARM] Background prewarm failed for {user_id[:8]}...: {e}") | |
| asyncio.create_task(_do_prewarm()) | |
| return {"status": "accepted", "message": "Prewarming started in background"} | |
| async def metrics_endpoint(): | |
| """ | |
| Get API instance metrics for monitoring. | |
| Returns: | |
| - active_agent_runs: Total active runs across all instances (from DB) | |
| - active_redis_streams: Active Redis stream keys | |
| - orphaned_streams: Streams without DB records (should be 0) | |
| """ | |
| from core.services import worker_metrics | |
| try: | |
| return await worker_metrics.get_worker_metrics() | |
| except Exception as e: | |
| logger.error(f"Failed to get metrics: {e}") | |
| raise HTTPException(status_code=500, detail=f"Failed to get metrics: {str(e)}") | |
| async def debug_endpoint(): | |
| """Get basic debug information for troubleshooting.""" | |
| from core.agents.api import _cancellation_events | |
| return { | |
| "instance_id": instance_id, | |
| "active_runs_on_instance": len(_cancellation_events), | |
| "is_shutting_down": _is_shutting_down, | |
| "timestamp": datetime.now(timezone.utc).isoformat() | |
| } | |
| async def redis_health_endpoint(): | |
| """ | |
| Get detailed Redis health and pool diagnostics. | |
| Returns: | |
| - status: healthy, degraded, or unhealthy | |
| - latency_ms: ping latency in milliseconds | |
| - pool: connection pool statistics | |
| - timeouts: configured timeout values | |
| """ | |
| try: | |
| health_data = await redis.health_check() | |
| # Add instance info | |
| health_data["instance_id"] = instance_id | |
| health_data["timestamp"] = datetime.now(timezone.utc).isoformat() | |
| # Return appropriate status code | |
| if health_data.get("status") == "unhealthy": | |
| return JSONResponse(status_code=503, content=health_data) | |
| elif health_data.get("status") == "degraded": | |
| return JSONResponse(status_code=200, content=health_data) | |
| else: | |
| return health_data | |
| except Exception as e: | |
| logger.error(f"Redis health check failed: {e}") | |
| return JSONResponse( | |
| status_code=503, | |
| content={ | |
| "status": "unhealthy", | |
| "error": str(e), | |
| "instance_id": instance_id, | |
| "timestamp": datetime.now(timezone.utc).isoformat() | |
| } | |
| ) | |
| def _get_health_metrics() -> dict: | |
| from core.services.system_metrics import get_system_metrics | |
| metrics = get_system_metrics() | |
| metrics["instance_id"] = instance_id | |
| metrics["timestamp"] = datetime.now(timezone.utc).isoformat() | |
| try: | |
| from core.agents.api import _cancellation_events | |
| metrics["active_runs_on_instance"] = len(_cancellation_events) | |
| except Exception: | |
| metrics["active_runs_on_instance"] = None | |
| try: | |
| from core.utils.lifecycle_tracker import get_active_runs | |
| metrics["lifecycle_active_runs"] = len(get_active_runs()) | |
| except Exception: | |
| metrics["lifecycle_active_runs"] = None | |
| return metrics | |
| async def health_check_docker(): | |
| logger.debug("Health docker check endpoint called") | |
| try: | |
| client = await redis.get_client() | |
| await client.ping() | |
| db_client = await db.client | |
| await db_client.table("threads").select("thread_id").limit(1).execute() | |
| logger.debug("Health docker check complete") | |
| metrics = _get_health_metrics() | |
| return { | |
| "status": "ok", | |
| "timestamp": metrics["timestamp"], | |
| "instance_id": instance_id, | |
| "metrics": {k: v for k, v in metrics.items() if k not in ("timestamp", "instance_id")}, | |
| } | |
| except Exception as e: | |
| logger.error(f"Failed health docker check: {e}") | |
| raise HTTPException(status_code=500, detail="Health check failed") | |
| app.include_router(api_router, prefix="/v1") | |
| METRICS_LOG_INTERVAL_SECONDS = int(os.getenv("METRICS_LOG_INTERVAL_SECONDS", "300")) # 5 min | |
| async def _metrics_logger_loop(): | |
| import json | |
| await asyncio.sleep(60) | |
| logger.info("[METRICS_LOG] Started, interval=%ds", METRICS_LOG_INTERVAL_SECONDS) | |
| try: | |
| while True: | |
| try: | |
| metrics = _get_health_metrics() | |
| log_payload = {"event": "system_metrics", **metrics} | |
| logger.info("system_metrics %s", json.dumps(log_payload)) | |
| except Exception as e: | |
| logger.debug("metrics_logger error: %s", e) | |
| await asyncio.sleep(METRICS_LOG_INTERVAL_SECONDS) | |
| except asyncio.CancelledError: | |
| logger.debug("Metrics logger cancelled") | |
| async def _memory_watchdog(): | |
| import time as time_module | |
| workers = int(os.getenv("WORKERS", "16")) | |
| total_ram_mb = psutil.virtual_memory().total / 1024 / 1024 | |
| per_worker_limit_mb = (total_ram_mb * 0.8) / workers | |
| critical_threshold_mb = per_worker_limit_mb * 0.87 | |
| warning_threshold_mb = per_worker_limit_mb * 0.80 | |
| info_threshold_mb = per_worker_limit_mb * 0.67 | |
| logger.info( | |
| f"Memory watchdog started: {total_ram_mb/1024:.1f}GB total, " | |
| f"{per_worker_limit_mb/1024:.1f}GB per worker ({workers} workers)" | |
| ) | |
| try: | |
| while True: | |
| try: | |
| process = psutil.Process() | |
| mem_info = process.memory_info() | |
| mem_mb = mem_info.rss / 1024 / 1024 | |
| mem_percent = (mem_mb / per_worker_limit_mb) * 100 | |
| # === NEW: Cleanup state tracking === | |
| from core.agents.api import _cancellation_events | |
| try: | |
| from core.utils.lifecycle_tracker import get_active_runs, evict_stale_runs | |
| active_runs = get_active_runs() | |
| stale_runs = [ | |
| rid for rid, start in active_runs.items() | |
| if (time_module.time() - start) > 3600 # > 1 hour | |
| ] | |
| # Actively evict zombie entries older than 2 hours | |
| evicted = evict_stale_runs(max_age_seconds=7200) | |
| if evicted: | |
| logger.warning(f"[WATCHDOG] Evicted {evicted} zombie entries from _active_runs") | |
| except ImportError: | |
| active_runs = {} | |
| stale_runs = [] | |
| # Evict orphaned cancellation events (run finished but event wasn't popped) | |
| stale_event_ids = [ | |
| rid for rid in _cancellation_events | |
| if rid not in active_runs | |
| ] | |
| for rid in stale_event_ids: | |
| _cancellation_events.pop(rid, None) | |
| if stale_event_ids: | |
| logger.warning(f"[WATCHDOG] Evicted {len(stale_event_ids)} orphaned cancellation events") | |
| cancellation_count = len(_cancellation_events) | |
| active_count = len(active_runs) | |
| stale_count = len(stale_runs) | |
| if stale_count > 0 or cancellation_count > 10: | |
| logger.warning( | |
| f"[WATCHDOG] mem={mem_mb:.0f}MB " | |
| f"cancellation_events={cancellation_count} " | |
| f"active_runs={active_count} " | |
| f"stale_runs={stale_count} " | |
| f"instance={instance_id}" | |
| ) | |
| if stale_runs: | |
| logger.warning(f"[WATCHDOG] stale_run_ids={stale_runs[:5]}") | |
| # === END NEW === | |
| # Existing memory threshold logging | |
| if mem_mb > critical_threshold_mb: | |
| logger.error( | |
| f"🚨 CRITICAL: Worker memory {mem_mb:.0f}MB ({mem_percent:.1f}%) " | |
| f"cancellation_events={cancellation_count} " | |
| f"active_runs={active_count} " | |
| f"instance={instance_id}" | |
| ) | |
| import gc | |
| gc.collect() | |
| elif mem_mb > warning_threshold_mb: | |
| logger.warning( | |
| f"⚠️ Worker memory high: {mem_mb:.0f}MB ({mem_percent:.1f}%) " | |
| f"cancellation_events={cancellation_count} " | |
| f"instance={instance_id}" | |
| ) | |
| elif mem_mb > info_threshold_mb: | |
| logger.info( | |
| f"Worker memory: {mem_mb:.0f}MB ({mem_percent:.1f}%) " | |
| f"cancellation_events={cancellation_count} " | |
| f"instance={instance_id}" | |
| ) | |
| except Exception as e: | |
| logger.debug(f"Memory watchdog error: {e}") | |
| await asyncio.sleep(60) | |
| except asyncio.CancelledError: | |
| logger.debug("Memory watchdog cancelled") | |
| except Exception as e: | |
| logger.error(f"Memory watchdog failed: {e}") | |
| if __name__ == "__main__": | |
| import uvicorn | |
| if sys.platform == "win32": | |
| asyncio.set_event_loop_policy(asyncio.WindowsProactorEventLoopPolicy()) | |
| # Enable reload mode for local and staging environments | |
| is_dev_env = config.ENV_MODE in [EnvMode.LOCAL, EnvMode.STAGING] | |
| workers = 1 if is_dev_env else 4 | |
| reload = is_dev_env | |
| logger.debug(f"Starting server on 0.0.0.0:8000 with {workers} workers (reload={reload})") | |
| uvicorn.run( | |
| "api:app", | |
| host="0.0.0.0", | |
| port=8000, | |
| workers=workers, | |
| loop="asyncio", | |
| reload=False if is_dev_env else False | |
| ) |