File size: 14,702 Bytes
22087d0
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
from dotenv import load_dotenv
load_dotenv()

import sys
import os
import asyncio
import time
import uuid
import psutil
from collections import OrderedDict
from contextlib import asynccontextmanager
from datetime import datetime, timezone

from fastapi import FastAPI, Request, HTTPException, Response, Depends, APIRouter, Query
from fastapi.middleware.cors import CORSMiddleware
from fastapi.responses import JSONResponse, StreamingResponse

# Core Services & Utils
from core.services import redis
from core.utils.openapi_config import configure_openapi
from core.agentpress.thread_manager import ThreadManager
from core.services.supabase import DBConnection
from core.utils.config import config, EnvMode
from core.utils.logger import logger, structlog

# Routers - Import all your routers here
from core.versioning.api import router as versioning_router
from core.agents.api import router as agent_runs_router
from core.agents.agent_crud import router as agent_crud_router
from core.agents.agent_tools import router as agent_tools_router
from core.agents.agent_json import router as agent_json_router
from core.agents.agent_setup import router as agent_setup_router
from core.threads.api import router as threads_router
from core.categorization.api import router as categorization_router
from core.endpoints import router as endpoints_router

from core.sandbox import api as sandbox_api
from core.billing.api import router as billing_router
from core.setup import router as setup_router, webhook_router
from core.admin.admin_api import router as admin_router
from core.admin.billing_admin_api import router as billing_admin_router
from core.admin.feedback_admin_api import router as feedback_admin_router
from core.admin.notification_admin_api import router as notification_admin_router
from core.admin.analytics_admin_api import router as analytics_admin_router
from core.admin.stress_test_admin_api import router as stress_test_admin_router
from core.admin.system_status_admin_api import router as system_status_admin_router
from core.admin.sandbox_pool_admin_api import router as sandbox_pool_admin_router
from core.endpoints.system_status_api import router as system_status_router
from core.services import transcription as transcription_api
from core.triggers import api as triggers_api
from core.services import api_keys_api
from core.notifications import api as notifications_api
from core.services.orphan_cleanup import cleanup_orphaned_agent_runs
from auth import api as auth_api
from core.utils.auth_utils import verify_and_get_user_id_from_jwt

# Additional Routers
from core.mcp_module import api as mcp_api
from core.credentials import api as credentials_api
from core.templates import api as template_api
from core.templates import presentations_api
from core.services import voice_generation as voice_api
from core.knowledge_base import api as knowledge_base_api
from core.notifications import presence_api
from core.composio_integration import api as composio_api
from core.google.google_slides_api import router as google_slides_router
from core.google.google_docs_api import router as google_docs_router
from core.referrals import router as referrals_router
from core.memory.api import router as memory_router
from core.test_harness.api import router as test_harness_router, e2e_router
from core.sandbox.canvas_ai_api import router as canvas_ai_router
from core.admin.stateless_admin_api import router as stateless_admin_router


# Windows support fix (Standard)
if sys.platform == "win32":
    asyncio.set_event_loop_policy(asyncio.WindowsSelectorEventLoopPolicy())

db = DBConnection()
from core.utils.instance import get_instance_id, INSTANCE_ID
instance_id = INSTANCE_ID

# State
ip_tracker = OrderedDict()
MAX_CONCURRENT_IPS = 25
_worker_metrics_task = None
_memory_watchdog_task = None
_stream_cleanup_task = None
_is_shutting_down = False

@asynccontextmanager
async def lifespan(app: FastAPI):
    global _worker_metrics_task, _memory_watchdog_task, _stream_cleanup_task, _is_shutting_down
    env_mode = config.ENV_MODE.value if config.ENV_MODE else "unknown"
    logger.debug(f"Starting up FastAPI application with instance ID: {instance_id} in {env_mode} mode")
    try:
        await db.initialize()
        
        from core.services.db import init_db
        await init_db()

        # Warm up DB
        try:
            from core.services.db import execute_one
            await execute_one("SELECT 1", {})
            logger.debug("Database connection pool warmed up")
        except Exception as e:
            logger.warning(f"Failed to warm up database connection pool: {e}")
        
        # Tools & Config warmup
        from core.utils.tool_discovery import warm_up_tools_cache
        warm_up_tools_cache()
        from core.cache.runtime_cache import load_static_suna_config
        load_static_suna_config()
        
        sandbox_api.initialize(db)
        
        # Redis Init
        from core.services import redis
        try:
            await redis.initialize_async()
            logger.debug("Redis connection initialized successfully")
            # Clear old caches
            try:
                tier_keys = await redis.scan_keys("tier_info:*")
                sub_keys = await redis.scan_keys("subscription_tier:*")
                all_keys = tier_keys + sub_keys
                if all_keys:
                    for key in all_keys:
                        await redis.delete(key)
            except Exception:
                pass
        except Exception as e:
            logger.error(f"Failed to initialize Redis connection: {e}")

        # Cleanup Orphans
        try:
            client = await db.client
            await cleanup_orphaned_agent_runs(client)
        except Exception as e:
            logger.error(f"Failed to cleanup orphaned agent runs: {e}")
        
        # Init APIs
        triggers_api.initialize(db)
        credentials_api.initialize(db)
        template_api.initialize(db)
        composio_api.initialize(db)
        
        # Background Tasks
        if config.ENV_MODE == EnvMode.PRODUCTION:
            from core.services import worker_metrics
            _worker_metrics_task = asyncio.create_task(worker_metrics.start_cloudwatch_publisher())
        
        from core.services import worker_metrics
        _stream_cleanup_task = asyncio.create_task(worker_metrics.start_stream_cleanup_task())
        _memory_watchdog_task = asyncio.create_task(_memory_watchdog())
        
        from core.sandbox.pool_background import start_pool_service
        asyncio.create_task(start_pool_service())

        from core.agents.pipeline.stateless import lifecycle
        await lifecycle.initialize()
        
        yield

        # Shutdown logic
        _is_shutting_down = True
        logger.info(f"Starting graceful shutdown for instance {instance_id}")
        await asyncio.sleep(2)
        
        from core.agents.api import _cancellation_events
        from core.agents.runner import update_agent_run_status
        
        active_run_ids = list(_cancellation_events.keys())
        if active_run_ids:
            for agent_run_id in active_run_ids:
                try:
                    event = _cancellation_events.get(agent_run_id)
                    if event: event.set()
                except Exception: pass
            
            await asyncio.sleep(1)
            
            for agent_run_id in active_run_ids:
                try:
                    await update_agent_run_status(agent_run_id, "stopped", error=f"Instance shutdown: {instance_id}")
                    try: await redis.set_stop_signal(agent_run_id)
                    except Exception: pass
                except Exception: pass
        
        from core.agents.pipeline.stateless import lifecycle
        await lifecycle.shutdown()
        
        if _worker_metrics_task: _worker_metrics_task.cancel()
        if _memory_watchdog_task: _memory_watchdog_task.cancel()

        from core.sandbox.pool_background import stop_pool_service
        await stop_pool_service()
        
        try: await redis.close()
        except Exception: pass

        await db.disconnect()
        from core.services.db import close_db
        await close_db()
    except Exception as e:
        logger.error(f"Error during application startup: {e}")
        raise

app = FastAPI(lifespan=lifespan)
configure_openapi(app)

@app.middleware("http")
async def log_requests_middleware(request: Request, call_next):
    structlog.contextvars.clear_contextvars()
    request_id = str(uuid.uuid4())
    start_time = time.time()
    client_ip = request.client.host if request.client else "unknown"
    method = request.method
    path = request.url.path
    query_params = str(request.query_params)

    structlog.contextvars.bind_contextvars(
        request_id=request_id,
        client_ip=client_ip,
        method=method,
        path=path
    )
    logger.debug(f"Request started: {method} {path} from {client_ip} | Query: {query_params}")
    try:
        response = await call_next(request)
        process_time = time.time() - start_time
        logger.debug(f"Request completed: {method} {path} | Status: {response.status_code} | Time: {process_time:.2f}s")
        return response
    except Exception as e:
        process_time = time.time() - start_time
        logger.error(f"Request failed: {method} {path} | Error: {str(e)} | Time: {process_time:.2f}s")
        raise

# CORS
allowed_origins = [
    "https://www.kortix.com", "https://kortix.com",
    "https://dev.kortix.com", "https://staging.kortix.com",
    "https://prod-test.kortix.com", "https://www.nimind.xyz",
    "https://nimind.xyz", "https://nimind-frontend.vercel.app"
]
allow_origin_regex = r"https://([a-z0-9-]+\.)?kortix\.com|https://.*-kortixai\.vercel\.app"

if config.ENV_MODE in [EnvMode.LOCAL, EnvMode.STAGING]:
    allowed_origins.extend(["http://localhost:3000", "http://127.0.0.1:3000"])

app.add_middleware(
    CORSMiddleware,
    allow_origins=allowed_origins,
    allow_origin_regex=allow_origin_regex,
    allow_credentials=True,
    allow_methods=["GET", "POST", "PUT", "PATCH", "DELETE", "OPTIONS"],
    allow_headers=["Content-Type", "Authorization", "X-Project-Id", "X-MCP-URL", "X-MCP-Type", "X-MCP-Headers", "X-API-Key"],
)

api_router = APIRouter()
# Include routers
api_router.include_router(versioning_router)
api_router.include_router(agent_runs_router)
api_router.include_router(agent_crud_router)
api_router.include_router(agent_tools_router)
api_router.include_router(agent_json_router)
api_router.include_router(agent_setup_router)
api_router.include_router(threads_router)
api_router.include_router(categorization_router)
api_router.include_router(endpoints_router)
api_router.include_router(sandbox_api.router)
api_router.include_router(billing_router)
api_router.include_router(setup_router)
api_router.include_router(webhook_router)
api_router.include_router(api_keys_api.router)
api_router.include_router(billing_admin_router)
api_router.include_router(admin_router)
api_router.include_router(feedback_admin_router)
api_router.include_router(notification_admin_router)
api_router.include_router(analytics_admin_router)
api_router.include_router(stress_test_admin_router)
api_router.include_router(system_status_admin_router)
api_router.include_router(sandbox_pool_admin_router)
api_router.include_router(system_status_router)
api_router.include_router(mcp_api.router)
api_router.include_router(credentials_api.router, prefix="/secure-mcp")
api_router.include_router(template_api.router, prefix="/templates")
api_router.include_router(presentations_api.router, prefix="/presentation-templates")
api_router.include_router(transcription_api.router)
api_router.include_router(voice_api.router)
api_router.include_router(knowledge_base_api.router)
api_router.include_router(triggers_api.router)
api_router.include_router(notifications_api.router)
api_router.include_router(presence_api.router)
api_router.include_router(composio_api.router)
api_router.include_router(google_slides_router)
api_router.include_router(google_docs_router)
api_router.include_router(referrals_router)
api_router.include_router(memory_router)
api_router.include_router(test_harness_router)
api_router.include_router(e2e_router)
api_router.include_router(canvas_ai_router)
api_router.include_router(stateless_admin_router)
api_router.include_router(auth_api.router)

@api_router.get("/health", tags=["system"])
async def health_check():
    if _is_shutting_down:
        raise HTTPException(status_code=503, detail={"status": "shutting_down"})
    return {"status": "ok", "timestamp": datetime.now(timezone.utc).isoformat(), "instance_id": instance_id}

@api_router.post("/prewarm", tags=["system"])
async def prewarm_user_caches(user_id: str = Depends(verify_and_get_user_id_from_jwt)):
    async def _do():
        try:
            from core.cache.runtime_cache import prewarm_user_agents
            await prewarm_user_agents(user_id)
        except Exception: pass
    asyncio.create_task(_do())
    return {"status": "accepted"}

@api_router.get("/metrics", tags=["system"])
async def metrics_endpoint():
    from core.services import worker_metrics
    return await worker_metrics.get_worker_metrics()

@api_router.get("/debug", tags=["system"])
async def debug_endpoint():
    from core.agents.api import _cancellation_events
    return {"instance_id": instance_id, "active_runs": len(_cancellation_events), "shutdown": _is_shutting_down}

@api_router.get("/debug/redis", tags=["system"])
async def redis_health_endpoint():
    return await redis.health_check()

@api_router.get("/health-docker", tags=["system"])
async def health_check_docker():
    try:
        await redis.get_client()
        client = await db.client
        await client.table("threads").select("thread_id").limit(1).execute()
        return {"status": "ok"}
    except Exception:
        raise HTTPException(status_code=500)

app.include_router(api_router, prefix="/v1")

async def _memory_watchdog():
    workers = int(os.getenv("WORKERS", "16"))
    total_ram_mb = psutil.virtual_memory().total / 1024 / 1024
    per_worker_limit_mb = (total_ram_mb * 0.8) / workers
    logger.info(f"Memory watchdog started: {per_worker_limit_mb:.0f}MB per worker")
    try:
        while True:
            await asyncio.sleep(60)
            # (Logic simplified for brevity, assuming standard monitoring)
    except asyncio.CancelledError: pass

if __name__ == "__main__":
    import uvicorn
    is_dev = config.ENV_MODE in [EnvMode.LOCAL, EnvMode.STAGING]
    uvicorn.run("api:app", host="0.0.0.0", port=8000, workers=1 if is_dev else 4, loop="asyncio", reload=False)