File size: 11,595 Bytes
7cfdc4c
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
from fastapi import APIRouter, HTTPException, Depends, Query
from typing import Dict, Any, List, Optional
from pydantic import BaseModel

from core.utils.logger import logger
from core.auth import require_admin


class RecoveryResponse(BaseModel):
    run_id: str
    success: bool
    action: str
    message: str
    error: Optional[str] = None


class StuckRunResponse(BaseModel):
    run_id: str
    thread_id: Optional[str]
    owner: Optional[str]
    status: Optional[str]
    heartbeat: Optional[float]
    heartbeat_age: Optional[float]
    start: Optional[float]
    duration: Optional[float]
    reason: Optional[str]


class ForceFailRequest(BaseModel):
    error: str = "Admin terminated"


class DLQEntryResponse(BaseModel):
    entry_id: str
    run_id: str
    write_type: str
    error: str
    attempt_count: int
    created_at: float
    failed_at: float


router = APIRouter(prefix="/admin/stateless", tags=["admin-stateless"])


@router.get("/stuck", response_model=List[StuckRunResponse])
async def list_stuck_runs(
    min_age: int = Query(default=5, ge=1, le=120),
    admin_user: Dict = Depends(require_admin)
) -> List[StuckRunResponse]:
    from core.agents.pipeline.stateless import recovery

    stuck = await recovery.get_stuck(min_age)
    logger.info(f"[Admin] {admin_user.get('id')} listed {len(stuck)} stuck runs")
    return [StuckRunResponse(**r) for r in stuck]


@router.post("/resume/{run_id}", response_model=RecoveryResponse)
async def force_resume(
    run_id: str,
    admin_user: Dict = Depends(require_admin)
) -> RecoveryResponse:
    from core.agents.pipeline.stateless import recovery

    result = await recovery.force_resume(run_id)
    logger.info(f"[Admin] {admin_user.get('id')} resumed {run_id}: {result.success}")
    return RecoveryResponse(
        run_id=result.run_id,
        success=result.success,
        action=result.action,
        message=result.message,
        error=result.error,
    )


@router.post("/complete/{run_id}", response_model=RecoveryResponse)
async def force_complete(
    run_id: str,
    reason: str = Query(default="admin"),
    admin_user: Dict = Depends(require_admin)
) -> RecoveryResponse:
    from core.agents.pipeline.stateless import recovery

    result = await recovery.force_complete(run_id, reason)
    logger.info(f"[Admin] {admin_user.get('id')} completed {run_id}: {reason}")
    return RecoveryResponse(
        run_id=result.run_id,
        success=result.success,
        action=result.action,
        message=result.message,
        error=result.error,
    )


@router.post("/fail/{run_id}", response_model=RecoveryResponse)
async def force_fail(
    run_id: str,
    request: ForceFailRequest,
    admin_user: Dict = Depends(require_admin)
) -> RecoveryResponse:
    from core.agents.pipeline.stateless import recovery

    result = await recovery.force_fail(run_id, request.error)
    logger.info(f"[Admin] {admin_user.get('id')} failed {run_id}: {request.error}")
    return RecoveryResponse(
        run_id=result.run_id,
        success=result.success,
        action=result.action,
        message=result.message,
        error=result.error,
    )


@router.get("/run/{run_id}")
async def get_run_info(
    run_id: str,
    admin_user: Dict = Depends(require_admin)
) -> Dict[str, Any]:
    from core.agents.pipeline.stateless import ownership

    info = await ownership.get_info(run_id)
    if not info:
        raise HTTPException(status_code=404, detail=f"Run {run_id} not found")
    return info


@router.get("/dashboard")
async def get_dashboard(admin_user: Dict = Depends(require_admin)) -> Dict[str, Any]:
    from core.agents.pipeline.stateless import metrics, recovery, ownership, write_buffer
    from core.agents.pipeline.stateless.persistence import wal, dlq
    from core.agents import repo as agents_repo

    stuck = await recovery.get_stuck(5)
    health = metrics.check_health()
    wal_stats = await wal.get_stats()
    dlq_stats = await dlq.get_stats()

    active_threads_count = await agents_repo.get_active_thread_ids_count()

    return {
        "active_runs": active_threads_count,
        "active_runs_metric": int(metrics.active_runs.get()),
        "owned_runs": ownership.owned_count,
        "pending_writes": int(metrics.pending_writes.get()),
        "stuck_count": len(stuck),
        "runs_started": metrics.runs_started.get(),
        "runs_completed": metrics.runs_completed.get(),
        "runs_failed": metrics.runs_failed.get(),
        "runs_recovered": metrics.runs_recovered.get(),
        "runs_rejected": metrics.runs_rejected.get(),
        "flush_latency_avg": metrics.flush_latency.avg(),
        "flush_latency_p99": metrics.flush_latency.percentile(99),
        "wal": wal_stats,
        "dlq": dlq_stats,
        "healthy": health["healthy"],
        "alerts": health["alerts"],
    }


@router.get("/active-threads")
async def get_active_threads(admin_user: Dict = Depends(require_admin)) -> Dict[str, Any]:
    from core.agents import repo as agents_repo

    active_threads = await agents_repo.get_active_thread_ids_with_runs()
    
    return {
        "count": len(active_threads),
        "threads": active_threads,
    }


@router.get("/health")
async def get_health() -> Dict[str, Any]:
    from core.agents.pipeline.stateless import lifecycle
    from fastapi.responses import JSONResponse
    
    health = await lifecycle.get_health()
    
    if health.get("shutting_down") or not health.get("healthy"):
        return JSONResponse(content=health, status_code=503)
    
    return health


@router.get("/metrics")
async def get_metrics(admin_user: Dict = Depends(require_admin)) -> str:
    from core.agents.pipeline.stateless import metrics
    return metrics.to_prometheus()


@router.post("/sweep")
async def trigger_sweep(admin_user: Dict = Depends(require_admin)) -> Dict[str, Any]:
    from core.agents.pipeline.stateless import recovery

    result = await recovery.sweep()
    logger.info(f"[Admin] {admin_user.get('id')} triggered sweep: {result}")
    return result


@router.post("/flush")
async def trigger_flush(admin_user: Dict = Depends(require_admin)) -> Dict[str, Any]:
    from core.agents.pipeline.stateless import write_buffer

    result = await write_buffer.flush_all()
    total = sum(result.values())
    logger.info(f"[Admin] {admin_user.get('id')} triggered flush: {total} writes")
    return {"runs": len(result), "total": total, "details": result}


@router.get("/dlq/entries", response_model=List[DLQEntryResponse])
async def list_dlq_entries(
    count: int = Query(default=50, ge=1, le=500),
    run_id: Optional[str] = None,
    admin_user: Dict = Depends(require_admin)
) -> List[DLQEntryResponse]:
    from core.agents.pipeline.stateless.persistence import dlq

    entries = await dlq.get_entries(count=count, run_id=run_id)
    return [
        DLQEntryResponse(
            entry_id=e.entry_id,
            run_id=e.run_id,
            write_type=e.write_type,
            error=e.error,
            attempt_count=e.attempt_count,
            created_at=e.created_at,
            failed_at=e.failed_at,
        )
        for e in entries
    ]


@router.post("/dlq/retry/{entry_id}")
async def retry_dlq_entry(
    entry_id: str,
    admin_user: Dict = Depends(require_admin)
) -> Dict[str, Any]:
    from core.agents.pipeline.stateless.persistence import dlq

    success = await dlq.retry_entry(entry_id)
    logger.info(f"[Admin] {admin_user.get('id')} retried DLQ entry {entry_id}: {success}")
    return {"success": success, "entry_id": entry_id}


@router.delete("/dlq/{entry_id}")
async def delete_dlq_entry(
    entry_id: str,
    admin_user: Dict = Depends(require_admin)
) -> Dict[str, Any]:
    from core.agents.pipeline.stateless.persistence import dlq

    success = await dlq.delete_entry(entry_id)
    logger.info(f"[Admin] {admin_user.get('id')} deleted DLQ entry {entry_id}: {success}")
    return {"success": success, "entry_id": entry_id}


@router.post("/dlq/purge")
async def purge_dlq(
    older_than_hours: Optional[int] = Query(default=None, ge=1, le=168),
    admin_user: Dict = Depends(require_admin)
) -> Dict[str, Any]:
    from core.agents.pipeline.stateless.persistence import dlq

    older_than_seconds = older_than_hours * 3600 if older_than_hours else None
    deleted = await dlq.purge(older_than_seconds)
    logger.info(f"[Admin] {admin_user.get('id')} purged DLQ: {deleted} entries")
    return {"deleted": deleted}


@router.get("/wal/stats")
async def get_wal_stats(admin_user: Dict = Depends(require_admin)) -> Dict[str, Any]:
    from core.agents.pipeline.stateless.persistence import wal
    return await wal.get_stats()


@router.get("/circuit-breakers")
async def get_circuit_breakers(admin_user: Dict = Depends(require_admin)) -> Dict[str, Any]:
    from core.agents.pipeline.stateless.resilience.circuit_breaker import registry
    return registry.to_dict()


@router.post("/circuit-breakers/reset")
async def reset_circuit_breakers(admin_user: Dict = Depends(require_admin)) -> Dict[str, Any]:
    from core.agents.pipeline.stateless.resilience.circuit_breaker import registry
    registry.reset_all()
    logger.info(f"[Admin] {admin_user.get('id')} reset all circuit breakers")
    return {"success": True}


@router.get("/rate-limiters")
async def get_rate_limiters(admin_user: Dict = Depends(require_admin)) -> Dict[str, Any]:
    from core.agents.pipeline.stateless.resilience.rate_limiter import rate_limiter_registry
    return rate_limiter_registry.to_dict()


@router.get("/backpressure")
async def get_backpressure(admin_user: Dict = Depends(require_admin)) -> Dict[str, Any]:
    from core.agents.pipeline.stateless.resilience.backpressure import backpressure
    return backpressure.to_dict()


@router.get("/metrics/history")
async def get_metrics_history(
    minutes: int = Query(default=30, ge=5, le=120),
    admin_user: Dict = Depends(require_admin)
) -> Dict[str, Any]:
    from core.services import redis
    from core.agents.pipeline.stateless import metrics
    import json
    import time
    
    HISTORY_KEY = "stateless:metrics:history"
    MAX_ENTRIES = 360
    
    now = time.time()
    current = {
        "timestamp": now,
        "active_runs": int(metrics.active_runs.get()),
        "pending_writes": int(metrics.pending_writes.get()),
        "runs_started": metrics.runs_started.get(),
        "runs_completed": metrics.runs_completed.get(),
        "runs_failed": metrics.runs_failed.get(),
        "flush_latency_avg": metrics.flush_latency.avg(),
        "flush_latency_p99": metrics.flush_latency.percentile(99),
        "writes_dropped": metrics.writes_dropped.get(),
        "dlq_entries": metrics.dlq_entries.get(),
    }
    
    try:
        await redis.lpush(HISTORY_KEY, json.dumps(current))
        await redis.ltrim(HISTORY_KEY, 0, MAX_ENTRIES - 1)
        await redis.expire(HISTORY_KEY, 86400)
    except Exception as e:
        logger.warning(f"Failed to store metrics history: {e}")
    
    cutoff = now - (minutes * 60)
    history = []
    
    try:
        raw_entries = await redis.lrange(HISTORY_KEY, 0, MAX_ENTRIES)
        for raw in raw_entries:
            try:
                entry = json.loads(raw)
                if entry.get("timestamp", 0) >= cutoff:
                    history.append(entry)
            except:
                continue
        history.reverse()
    except Exception as e:
        logger.warning(f"Failed to retrieve metrics history: {e}")
    
    return {
        "current": current,
        "history": history,
        "minutes": minutes,
    }