from dotenv import load_dotenv load_dotenv() from fastapi import FastAPI, Request, HTTPException, Response, Depends, APIRouter, Query from fastapi.middleware.cors import CORSMiddleware from fastapi.responses import JSONResponse, StreamingResponse from core.services import redis from core.utils.openapi_config import configure_openapi from contextlib import asynccontextmanager from core.agentpress.thread_manager import ThreadManager from core.services.supabase import DBConnection from datetime import datetime, timezone from core.utils.config import config, EnvMode import asyncio from core.utils.logger import logger, structlog import time from collections import OrderedDict import os import psutil from pydantic import BaseModel import uuid from core.versioning.api import router as versioning_router from core.agents.api import router as agent_runs_router from core.agents.agent_crud import router as agent_crud_router from core.agents.agent_tools import router as agent_tools_router from core.agents.agent_json import router as agent_json_router from core.agents.agent_setup import router as agent_setup_router from core.threads.api import router as threads_router from core.categorization.api import router as categorization_router from core.endpoints import router as endpoints_router from core.sandbox import api as sandbox_api from core.billing.api import router as billing_router from core.setup import router as setup_router, webhook_router from core.admin.admin_api import router as admin_router from core.admin.billing_admin_api import router as billing_admin_router from core.admin.feedback_admin_api import router as feedback_admin_router from core.admin.notification_admin_api import router as notification_admin_router from core.admin.analytics_admin_api import router as analytics_admin_router from core.admin.stress_test_admin_api import router as stress_test_admin_router from core.admin.system_status_admin_api import router as system_status_admin_router from core.admin.sandbox_pool_admin_api import router as sandbox_pool_admin_router from core.endpoints.system_status_api import router as system_status_router from core.services import transcription as transcription_api import sys from core.triggers import api as triggers_api from core.services import api_keys_api from core.notifications import api as notifications_api from core.services.orphan_cleanup import cleanup_orphaned_agent_runs from auth import api as auth_api from core.utils.auth_utils import verify_and_get_user_id_from_jwt if sys.platform == "win32": # Use SelectorEventLoop on Windows for psycopg async compatibility asyncio.set_event_loop_policy(asyncio.WindowsSelectorEventLoopPolicy()) db = DBConnection() # Use shared instance ID for distributed deployments from core.utils.instance import get_instance_id, INSTANCE_ID instance_id = INSTANCE_ID # Keep backward compatibility # Rate limiter state ip_tracker = OrderedDict() MAX_CONCURRENT_IPS = 25 # Background task handles _worker_metrics_task = None _memory_watchdog_task = None _stream_cleanup_task = None _metrics_logger_task = None # Graceful shutdown flag for health checks # When True, health check will return unhealthy to stop receiving traffic _is_shutting_down = False @asynccontextmanager async def lifespan(app: FastAPI): global _worker_metrics_task, _memory_watchdog_task, _stream_cleanup_task, _metrics_logger_task, _is_shutting_down env_mode = config.ENV_MODE.value if config.ENV_MODE else "unknown" logger.debug(f"Starting up FastAPI application with instance ID: {instance_id} in {env_mode} mode") try: await db.initialize() from core.services.db import init_db await init_db() try: from core.services.db import execute_one await execute_one("SELECT 1", {}) logger.debug("Database connection pool warmed up") except Exception as e: logger.warning(f"Failed to warm up database connection pool: {e}") # Pre-load tool classes and schemas to avoid first-request delay from core.utils.tool_discovery import warm_up_tools_cache warm_up_tools_cache() # Pre-load static Suna config for fast path in API requests from core.cache.runtime_cache import load_static_suna_config load_static_suna_config() sandbox_api.initialize(db) from core.services import redis try: await redis.initialize_async() logger.debug("Redis connection initialized successfully") try: tier_keys = await redis.scan_keys("tier_info:*") sub_keys = await redis.scan_keys("subscription_tier:*") all_keys = tier_keys + sub_keys if all_keys: for key in all_keys: await redis.delete(key) logger.info(f"[STARTUP] Cleared {len(all_keys)} tier caches to pick up config changes") except Exception as e: logger.warning(f"[STARTUP] Failed to clear tier caches: {e}") except Exception as e: logger.error(f"Failed to initialize Redis connection: {e}") try: client = await db.client await cleanup_orphaned_agent_runs(client) except Exception as e: logger.error(f"Failed to cleanup orphaned agent runs on startup: {e}") if config.ACTIVATE_MCPS_TRIG: triggers_api.initialize(db) credentials_api.initialize(db) composio_api.initialize(db) template_api.initialize(db) # Start CloudWatch worker metrics publisher (production only) if config.ENV_MODE == EnvMode.PRODUCTION: from core.services import worker_metrics _worker_metrics_task = asyncio.create_task(worker_metrics.start_cloudwatch_publisher()) # Start Redis stream cleanup task (catches orphaned streams with no TTL) from core.services import worker_metrics _stream_cleanup_task = asyncio.create_task(worker_metrics.start_stream_cleanup_task()) # Start memory watchdog for observability _memory_watchdog_task = asyncio.create_task(_memory_watchdog()) # Log system metrics to CloudWatch every 5 minutes (logger ships to CloudWatch) _metrics_logger_task = asyncio.create_task(_metrics_logger_loop()) # Start sandbox pool service (maintains pre-warmed sandboxes) from core.sandbox.pool_background import start_pool_service asyncio.create_task(start_pool_service()) # Start conversation analytics worker from core.analytics.conversation_analytics_worker import start_analytics_worker asyncio.create_task(start_analytics_worker()) # Initialize stateless pipeline from core.agents.pipeline.stateless import lifecycle await lifecycle.initialize() logger.info("[STARTUP] Stateless pipeline initialized") yield _is_shutting_down = True logger.info(f"Starting graceful shutdown for instance {instance_id}") # Give K8s readiness probe time to detect unhealthy state # This ensures no new traffic is routed to this pod await asyncio.sleep(2) # ===== CRITICAL: Stop all running agent runs on this instance ===== from core.agents.api import _cancellation_events from core.agents.runner import update_agent_run_status active_run_ids = list(_cancellation_events.keys()) if active_run_ids: logger.warning(f"🛑 Stopping {len(active_run_ids)} active agent runs on shutdown: {active_run_ids}") # Set cancellation events for all running runs for agent_run_id in active_run_ids: try: event = _cancellation_events.get(agent_run_id) if event: event.set() logger.info(f"Set cancellation event for {agent_run_id}") except Exception as e: logger.error(f"Failed to set cancellation event for {agent_run_id}: {e}") # Give tasks a moment to handle cancellation gracefully await asyncio.sleep(1) # Force update DB status for any runs that didn't clean up for agent_run_id in active_run_ids: try: # Update status to stopped with shutdown message await update_agent_run_status( agent_run_id, "stopped", error=f"Instance shutdown: {instance_id}" ) logger.info(f"✅ Marked agent run {agent_run_id} as stopped (instance shutdown)") # Also set Redis stop signal for any reconnecting clients try: await redis.set_stop_signal(agent_run_id) except Exception: pass except Exception as e: logger.error(f"Failed to update agent run {agent_run_id} on shutdown: {e}") else: logger.info("No active agent runs to stop on shutdown") # Shutdown stateless pipeline from core.agents.pipeline.stateless import lifecycle await lifecycle.shutdown() logger.info("[SHUTDOWN] Stateless pipeline shutdown complete") logger.debug("Cleaning up resources") # Stop CloudWatch worker metrics task if _worker_metrics_task is not None: _worker_metrics_task.cancel() try: await _worker_metrics_task except asyncio.CancelledError: pass # Stop memory watchdog task if _memory_watchdog_task is not None: _memory_watchdog_task.cancel() try: await _memory_watchdog_task except asyncio.CancelledError: pass # Stop metrics logger task if _metrics_logger_task is not None: _metrics_logger_task.cancel() try: await _metrics_logger_task except asyncio.CancelledError: pass # Stop sandbox pool service from core.sandbox.pool_background import stop_pool_service await stop_pool_service() # Stop conversation analytics worker from core.analytics.conversation_analytics_worker import stop_analytics_worker await stop_analytics_worker() try: logger.debug("Closing Redis connection") await redis.close() logger.debug("Redis connection closed successfully") except Exception as e: logger.error(f"Error closing Redis connection: {e}") logger.debug("Disconnecting from database") await db.disconnect() # Close direct Postgres connection pool from core.services.db import close_db await close_db() except Exception as e: logger.error(f"Error during application startup: {e}") raise app = FastAPI( lifespan=lifespan, swagger_ui_parameters={ "persistAuthorization": True, # Keep auth between page refreshes }, ) # Configure OpenAPI docs with API Key and Bearer token auth configure_openapi(app) @app.middleware("http") async def log_requests_middleware(request: Request, call_next): structlog.contextvars.clear_contextvars() request_id = str(uuid.uuid4()) start_time = time.time() client_ip = request.client.host if request.client else "unknown" method = request.method path = request.url.path query_params = str(request.query_params) structlog.contextvars.bind_contextvars( request_id=request_id, client_ip=client_ip, method=method, path=path, query_params=query_params ) # Log the incoming request logger.debug(f"Request started: {method} {path} from {client_ip} | Query: {query_params}") try: response = await call_next(request) process_time = time.time() - start_time logger.debug(f"Request completed: {method} {path} | Status: {response.status_code} | Time: {process_time:.2f}s") return response except Exception as e: process_time = time.time() - start_time try: error_str = str(e) except Exception: error_str = f"Error of type {type(e).__name__}" logger.error(f"Request failed: {method} {path} | Error: {error_str} | Time: {process_time:.2f}s") raise # Define allowed origins based on environment allowed_origins = [ "https://www.kortix.com", "https://kortix.com", "https://dev.kortix.com", "https://staging.kortix.com", "https://prod-test.kortix.com", ] # Allow all *.kortix.com subdomains and Vercel preview deployments allow_origin_regex = r"https://([a-z0-9-]+\.)?kortix\.com|https://.*-kortixai\.vercel\.app" # Add local origins for development if config.ENV_MODE == EnvMode.LOCAL: allowed_origins.append("http://localhost:3000") allowed_origins.append("http://127.0.0.1:3000") # Add staging-specific origins if config.ENV_MODE == EnvMode.STAGING: allowed_origins.append("https://staging.kortix.com") allowed_origins.append("http://localhost:3000") app.add_middleware( CORSMiddleware, allow_origins=allowed_origins, allow_origin_regex=allow_origin_regex, allow_credentials=True, allow_methods=["GET", "POST", "PUT", "PATCH", "DELETE", "OPTIONS"], allow_headers=["Content-Type", "Authorization", "X-Project-Id", "X-MCP-URL", "X-MCP-Type", "X-MCP-Headers", "X-API-Key"], ) # Create a main API router api_router = APIRouter() # Include all API routers without individual prefixes # Core routers api_router.include_router(versioning_router) api_router.include_router(agent_runs_router) api_router.include_router(agent_crud_router) api_router.include_router(agent_tools_router) api_router.include_router(agent_json_router) api_router.include_router(agent_setup_router) api_router.include_router(threads_router) api_router.include_router(categorization_router) api_router.include_router(endpoints_router) api_router.include_router(sandbox_api.router) api_router.include_router(billing_router) api_router.include_router(setup_router) api_router.include_router(webhook_router) # Webhooks at /api/webhooks/* api_router.include_router(api_keys_api.router) api_router.include_router(billing_admin_router) api_router.include_router(admin_router) api_router.include_router(feedback_admin_router) api_router.include_router(notification_admin_router) api_router.include_router(analytics_admin_router) api_router.include_router(stress_test_admin_router) api_router.include_router(system_status_admin_router) api_router.include_router(sandbox_pool_admin_router) api_router.include_router(system_status_router) from core.mcp_module import api as mcp_api from core.credentials import api as credentials_api from core.templates import api as template_api from core.templates import presentations_api if config.ACTIVATE_MCPS_TRIG: api_router.include_router(mcp_api.router) api_router.include_router(credentials_api.router, prefix="/secure-mcp") api_router.include_router(template_api.router, prefix="/templates") api_router.include_router(presentations_api.router, prefix="/presentation-templates") api_router.include_router(transcription_api.router) from core.services import voice_generation as voice_api api_router.include_router(voice_api.router) from core.knowledge_base import api as knowledge_base_api api_router.include_router(knowledge_base_api.router) if config.ACTIVATE_MCPS_TRIG: api_router.include_router(triggers_api.router) api_router.include_router(notifications_api.router) from core.notifications import presence_api api_router.include_router(presence_api.router) from core.composio_integration import api as composio_api if config.ACTIVATE_MCPS_TRIG: api_router.include_router(composio_api.router) from core.google.google_slides_api import router as google_slides_router api_router.include_router(google_slides_router) from core.google.google_docs_api import router as google_docs_router api_router.include_router(google_docs_router) from core.referrals import router as referrals_router from core.memory.api import router as memory_router api_router.include_router(referrals_router) api_router.include_router(memory_router) from core.test_harness.api import router as test_harness_router, e2e_router api_router.include_router(test_harness_router) api_router.include_router(e2e_router) from core.sandbox.canvas_ai_api import router as canvas_ai_router api_router.include_router(canvas_ai_router) from core.admin.stateless_admin_api import router as stateless_admin_router api_router.include_router(stateless_admin_router) # Auth OTP endpoint for expired magic links api_router.include_router(auth_api.router) @api_router.get("/health", summary="Health Check", operation_id="health_check", tags=["system"]) async def health_check(): logger.debug("Health check endpoint called") # During shutdown, return unhealthy status # This causes K8s readinessProbe to fail and removes pod from service endpoints if _is_shutting_down: logger.debug(f"Health check returning unhealthy (shutting down) for instance {instance_id}") raise HTTPException( status_code=503, detail={ "status": "shutting_down", "timestamp": datetime.now(timezone.utc).isoformat(), "instance_id": instance_id } ) return { "status": "ok", "timestamp": datetime.now(timezone.utc).isoformat(), "instance_id": instance_id, } @api_router.post("/prewarm", summary="Prewarm User Caches", operation_id="prewarm_user", tags=["system"]) async def prewarm_user_caches(user_id: str = Depends(verify_and_get_user_id_from_jwt)): async def _do_prewarm(): try: from core.cache.runtime_cache import prewarm_user_agents await prewarm_user_agents(user_id) except Exception as e: logger.warning(f"[PREWARM] Background prewarm failed for {user_id[:8]}...: {e}") asyncio.create_task(_do_prewarm()) return {"status": "accepted", "message": "Prewarming started in background"} @api_router.get("/metrics", summary="System Metrics", operation_id="metrics", tags=["system"]) async def metrics_endpoint(): """ Get API instance metrics for monitoring. Returns: - active_agent_runs: Total active runs across all instances (from DB) - active_redis_streams: Active Redis stream keys - orphaned_streams: Streams without DB records (should be 0) """ from core.services import worker_metrics try: return await worker_metrics.get_worker_metrics() except Exception as e: logger.error(f"Failed to get metrics: {e}") raise HTTPException(status_code=500, detail=f"Failed to get metrics: {str(e)}") @api_router.get("/debug", summary="Debug Information", operation_id="debug", tags=["system"]) async def debug_endpoint(): """Get basic debug information for troubleshooting.""" from core.agents.api import _cancellation_events return { "instance_id": instance_id, "active_runs_on_instance": len(_cancellation_events), "is_shutting_down": _is_shutting_down, "timestamp": datetime.now(timezone.utc).isoformat() } @api_router.get("/debug/redis", summary="Redis Health & Diagnostics", operation_id="redis_health", tags=["system"]) async def redis_health_endpoint(): """ Get detailed Redis health and pool diagnostics. Returns: - status: healthy, degraded, or unhealthy - latency_ms: ping latency in milliseconds - pool: connection pool statistics - timeouts: configured timeout values """ try: health_data = await redis.health_check() # Add instance info health_data["instance_id"] = instance_id health_data["timestamp"] = datetime.now(timezone.utc).isoformat() # Return appropriate status code if health_data.get("status") == "unhealthy": return JSONResponse(status_code=503, content=health_data) elif health_data.get("status") == "degraded": return JSONResponse(status_code=200, content=health_data) else: return health_data except Exception as e: logger.error(f"Redis health check failed: {e}") return JSONResponse( status_code=503, content={ "status": "unhealthy", "error": str(e), "instance_id": instance_id, "timestamp": datetime.now(timezone.utc).isoformat() } ) def _get_health_metrics() -> dict: from core.services.system_metrics import get_system_metrics metrics = get_system_metrics() metrics["instance_id"] = instance_id metrics["timestamp"] = datetime.now(timezone.utc).isoformat() try: from core.agents.api import _cancellation_events metrics["active_runs_on_instance"] = len(_cancellation_events) except Exception: metrics["active_runs_on_instance"] = None try: from core.utils.lifecycle_tracker import get_active_runs metrics["lifecycle_active_runs"] = len(get_active_runs()) except Exception: metrics["lifecycle_active_runs"] = None return metrics @api_router.get("/health-docker", summary="Docker Health Check", operation_id="health_check_docker", tags=["system"]) async def health_check_docker(): logger.debug("Health docker check endpoint called") try: client = await redis.get_client() await client.ping() db_client = await db.client await db_client.table("threads").select("thread_id").limit(1).execute() logger.debug("Health docker check complete") metrics = _get_health_metrics() return { "status": "ok", "timestamp": metrics["timestamp"], "instance_id": instance_id, "metrics": {k: v for k, v in metrics.items() if k not in ("timestamp", "instance_id")}, } except Exception as e: logger.error(f"Failed health docker check: {e}") raise HTTPException(status_code=500, detail="Health check failed") app.include_router(api_router, prefix="/v1") METRICS_LOG_INTERVAL_SECONDS = int(os.getenv("METRICS_LOG_INTERVAL_SECONDS", "300")) # 5 min async def _metrics_logger_loop(): import json await asyncio.sleep(60) logger.info("[METRICS_LOG] Started, interval=%ds", METRICS_LOG_INTERVAL_SECONDS) try: while True: try: metrics = _get_health_metrics() log_payload = {"event": "system_metrics", **metrics} logger.info("system_metrics %s", json.dumps(log_payload)) except Exception as e: logger.debug("metrics_logger error: %s", e) await asyncio.sleep(METRICS_LOG_INTERVAL_SECONDS) except asyncio.CancelledError: logger.debug("Metrics logger cancelled") async def _memory_watchdog(): import time as time_module workers = int(os.getenv("WORKERS", "16")) total_ram_mb = psutil.virtual_memory().total / 1024 / 1024 per_worker_limit_mb = (total_ram_mb * 0.8) / workers critical_threshold_mb = per_worker_limit_mb * 0.87 warning_threshold_mb = per_worker_limit_mb * 0.80 info_threshold_mb = per_worker_limit_mb * 0.67 logger.info( f"Memory watchdog started: {total_ram_mb/1024:.1f}GB total, " f"{per_worker_limit_mb/1024:.1f}GB per worker ({workers} workers)" ) try: while True: try: process = psutil.Process() mem_info = process.memory_info() mem_mb = mem_info.rss / 1024 / 1024 mem_percent = (mem_mb / per_worker_limit_mb) * 100 # === NEW: Cleanup state tracking === from core.agents.api import _cancellation_events try: from core.utils.lifecycle_tracker import get_active_runs, evict_stale_runs active_runs = get_active_runs() stale_runs = [ rid for rid, start in active_runs.items() if (time_module.time() - start) > 3600 # > 1 hour ] # Actively evict zombie entries older than 2 hours evicted = evict_stale_runs(max_age_seconds=7200) if evicted: logger.warning(f"[WATCHDOG] Evicted {evicted} zombie entries from _active_runs") except ImportError: active_runs = {} stale_runs = [] # Evict orphaned cancellation events (run finished but event wasn't popped) stale_event_ids = [ rid for rid in _cancellation_events if rid not in active_runs ] for rid in stale_event_ids: _cancellation_events.pop(rid, None) if stale_event_ids: logger.warning(f"[WATCHDOG] Evicted {len(stale_event_ids)} orphaned cancellation events") cancellation_count = len(_cancellation_events) active_count = len(active_runs) stale_count = len(stale_runs) if stale_count > 0 or cancellation_count > 10: logger.warning( f"[WATCHDOG] mem={mem_mb:.0f}MB " f"cancellation_events={cancellation_count} " f"active_runs={active_count} " f"stale_runs={stale_count} " f"instance={instance_id}" ) if stale_runs: logger.warning(f"[WATCHDOG] stale_run_ids={stale_runs[:5]}") # === END NEW === # Existing memory threshold logging if mem_mb > critical_threshold_mb: logger.error( f"🚨 CRITICAL: Worker memory {mem_mb:.0f}MB ({mem_percent:.1f}%) " f"cancellation_events={cancellation_count} " f"active_runs={active_count} " f"instance={instance_id}" ) import gc gc.collect() elif mem_mb > warning_threshold_mb: logger.warning( f"⚠️ Worker memory high: {mem_mb:.0f}MB ({mem_percent:.1f}%) " f"cancellation_events={cancellation_count} " f"instance={instance_id}" ) elif mem_mb > info_threshold_mb: logger.info( f"Worker memory: {mem_mb:.0f}MB ({mem_percent:.1f}%) " f"cancellation_events={cancellation_count} " f"instance={instance_id}" ) except Exception as e: logger.debug(f"Memory watchdog error: {e}") await asyncio.sleep(60) except asyncio.CancelledError: logger.debug("Memory watchdog cancelled") except Exception as e: logger.error(f"Memory watchdog failed: {e}") if __name__ == "__main__": import uvicorn if sys.platform == "win32": asyncio.set_event_loop_policy(asyncio.WindowsProactorEventLoopPolicy()) # Enable reload mode for local and staging environments is_dev_env = config.ENV_MODE in [EnvMode.LOCAL, EnvMode.STAGING] workers = 1 if is_dev_env else 4 reload = is_dev_env logger.debug(f"Starting server on 0.0.0.0:8000 with {workers} workers (reload={reload})") uvicorn.run( "api:app", host="0.0.0.0", port=8000, workers=workers, loop="asyncio", reload=False if is_dev_env else False )