thio / core /admin /stateless_admin_api.py
kyrlos adel
cva1
7cfdc4c
Raw History Blame Contribute Delete
11.6 kB
from fastapi import APIRouter, HTTPException, Depends, Query
from typing import Dict, Any, List, Optional
from pydantic import BaseModel
from core.utils.logger import logger
from core.auth import require_admin
class RecoveryResponse(BaseModel):
run_id: str
success: bool
action: str
message: str
error: Optional[str] = None
class StuckRunResponse(BaseModel):
run_id: str
thread_id: Optional[str]
owner: Optional[str]
status: Optional[str]
heartbeat: Optional[float]
heartbeat_age: Optional[float]
start: Optional[float]
duration: Optional[float]
reason: Optional[str]
class ForceFailRequest(BaseModel):
error: str = "Admin terminated"
class DLQEntryResponse(BaseModel):
entry_id: str
run_id: str
write_type: str
error: str
attempt_count: int
created_at: float
failed_at: float
router = APIRouter(prefix="/admin/stateless", tags=["admin-stateless"])
@router.get("/stuck", response_model=List[StuckRunResponse])
async def list_stuck_runs(
min_age: int = Query(default=5, ge=1, le=120),
admin_user: Dict = Depends(require_admin)
) -> List[StuckRunResponse]:
from core.agents.pipeline.stateless import recovery
stuck = await recovery.get_stuck(min_age)
logger.info(f"[Admin] {admin_user.get('id')} listed {len(stuck)} stuck runs")
return [StuckRunResponse(**r) for r in stuck]
@router.post("/resume/{run_id}", response_model=RecoveryResponse)
async def force_resume(
run_id: str,
admin_user: Dict = Depends(require_admin)
) -> RecoveryResponse:
from core.agents.pipeline.stateless import recovery
result = await recovery.force_resume(run_id)
logger.info(f"[Admin] {admin_user.get('id')} resumed {run_id}: {result.success}")
return RecoveryResponse(
run_id=result.run_id,
success=result.success,
action=result.action,
message=result.message,
error=result.error,
)
@router.post("/complete/{run_id}", response_model=RecoveryResponse)
async def force_complete(
run_id: str,
reason: str = Query(default="admin"),
admin_user: Dict = Depends(require_admin)
) -> RecoveryResponse:
from core.agents.pipeline.stateless import recovery
result = await recovery.force_complete(run_id, reason)
logger.info(f"[Admin] {admin_user.get('id')} completed {run_id}: {reason}")
return RecoveryResponse(
run_id=result.run_id,
success=result.success,
action=result.action,
message=result.message,
error=result.error,
)
@router.post("/fail/{run_id}", response_model=RecoveryResponse)
async def force_fail(
run_id: str,
request: ForceFailRequest,
admin_user: Dict = Depends(require_admin)
) -> RecoveryResponse:
from core.agents.pipeline.stateless import recovery
result = await recovery.force_fail(run_id, request.error)
logger.info(f"[Admin] {admin_user.get('id')} failed {run_id}: {request.error}")
return RecoveryResponse(
run_id=result.run_id,
success=result.success,
action=result.action,
message=result.message,
error=result.error,
)
@router.get("/run/{run_id}")
async def get_run_info(
run_id: str,
admin_user: Dict = Depends(require_admin)
) -> Dict[str, Any]:
from core.agents.pipeline.stateless import ownership
info = await ownership.get_info(run_id)
if not info:
raise HTTPException(status_code=404, detail=f"Run {run_id} not found")
return info
@router.get("/dashboard")
async def get_dashboard(admin_user: Dict = Depends(require_admin)) -> Dict[str, Any]:
from core.agents.pipeline.stateless import metrics, recovery, ownership, write_buffer
from core.agents.pipeline.stateless.persistence import wal, dlq
from core.agents import repo as agents_repo
stuck = await recovery.get_stuck(5)
health = metrics.check_health()
wal_stats = await wal.get_stats()
dlq_stats = await dlq.get_stats()
active_threads_count = await agents_repo.get_active_thread_ids_count()
return {
"active_runs": active_threads_count,
"active_runs_metric": int(metrics.active_runs.get()),
"owned_runs": ownership.owned_count,
"pending_writes": int(metrics.pending_writes.get()),
"stuck_count": len(stuck),
"runs_started": metrics.runs_started.get(),
"runs_completed": metrics.runs_completed.get(),
"runs_failed": metrics.runs_failed.get(),
"runs_recovered": metrics.runs_recovered.get(),
"runs_rejected": metrics.runs_rejected.get(),
"flush_latency_avg": metrics.flush_latency.avg(),
"flush_latency_p99": metrics.flush_latency.percentile(99),
"wal": wal_stats,
"dlq": dlq_stats,
"healthy": health["healthy"],
"alerts": health["alerts"],
}
@router.get("/active-threads")
async def get_active_threads(admin_user: Dict = Depends(require_admin)) -> Dict[str, Any]:
from core.agents import repo as agents_repo
active_threads = await agents_repo.get_active_thread_ids_with_runs()
return {
"count": len(active_threads),
"threads": active_threads,
}
@router.get("/health")
async def get_health() -> Dict[str, Any]:
from core.agents.pipeline.stateless import lifecycle
from fastapi.responses import JSONResponse
health = await lifecycle.get_health()
if health.get("shutting_down") or not health.get("healthy"):
return JSONResponse(content=health, status_code=503)
return health
@router.get("/metrics")
async def get_metrics(admin_user: Dict = Depends(require_admin)) -> str:
from core.agents.pipeline.stateless import metrics
return metrics.to_prometheus()
@router.post("/sweep")
async def trigger_sweep(admin_user: Dict = Depends(require_admin)) -> Dict[str, Any]:
from core.agents.pipeline.stateless import recovery
result = await recovery.sweep()
logger.info(f"[Admin] {admin_user.get('id')} triggered sweep: {result}")
return result
@router.post("/flush")
async def trigger_flush(admin_user: Dict = Depends(require_admin)) -> Dict[str, Any]:
from core.agents.pipeline.stateless import write_buffer
result = await write_buffer.flush_all()
total = sum(result.values())
logger.info(f"[Admin] {admin_user.get('id')} triggered flush: {total} writes")
return {"runs": len(result), "total": total, "details": result}
@router.get("/dlq/entries", response_model=List[DLQEntryResponse])
async def list_dlq_entries(
count: int = Query(default=50, ge=1, le=500),
run_id: Optional[str] = None,
admin_user: Dict = Depends(require_admin)
) -> List[DLQEntryResponse]:
from core.agents.pipeline.stateless.persistence import dlq
entries = await dlq.get_entries(count=count, run_id=run_id)
return [
DLQEntryResponse(
entry_id=e.entry_id,
run_id=e.run_id,
write_type=e.write_type,
error=e.error,
attempt_count=e.attempt_count,
created_at=e.created_at,
failed_at=e.failed_at,
)
for e in entries
]
@router.post("/dlq/retry/{entry_id}")
async def retry_dlq_entry(
entry_id: str,
admin_user: Dict = Depends(require_admin)
) -> Dict[str, Any]:
from core.agents.pipeline.stateless.persistence import dlq
success = await dlq.retry_entry(entry_id)
logger.info(f"[Admin] {admin_user.get('id')} retried DLQ entry {entry_id}: {success}")
return {"success": success, "entry_id": entry_id}
@router.delete("/dlq/{entry_id}")
async def delete_dlq_entry(
entry_id: str,
admin_user: Dict = Depends(require_admin)
) -> Dict[str, Any]:
from core.agents.pipeline.stateless.persistence import dlq
success = await dlq.delete_entry(entry_id)
logger.info(f"[Admin] {admin_user.get('id')} deleted DLQ entry {entry_id}: {success}")
return {"success": success, "entry_id": entry_id}
@router.post("/dlq/purge")
async def purge_dlq(
older_than_hours: Optional[int] = Query(default=None, ge=1, le=168),
admin_user: Dict = Depends(require_admin)
) -> Dict[str, Any]:
from core.agents.pipeline.stateless.persistence import dlq
older_than_seconds = older_than_hours * 3600 if older_than_hours else None
deleted = await dlq.purge(older_than_seconds)
logger.info(f"[Admin] {admin_user.get('id')} purged DLQ: {deleted} entries")
return {"deleted": deleted}
@router.get("/wal/stats")
async def get_wal_stats(admin_user: Dict = Depends(require_admin)) -> Dict[str, Any]:
from core.agents.pipeline.stateless.persistence import wal
return await wal.get_stats()
@router.get("/circuit-breakers")
async def get_circuit_breakers(admin_user: Dict = Depends(require_admin)) -> Dict[str, Any]:
from core.agents.pipeline.stateless.resilience.circuit_breaker import registry
return registry.to_dict()
@router.post("/circuit-breakers/reset")
async def reset_circuit_breakers(admin_user: Dict = Depends(require_admin)) -> Dict[str, Any]:
from core.agents.pipeline.stateless.resilience.circuit_breaker import registry
registry.reset_all()
logger.info(f"[Admin] {admin_user.get('id')} reset all circuit breakers")
return {"success": True}
@router.get("/rate-limiters")
async def get_rate_limiters(admin_user: Dict = Depends(require_admin)) -> Dict[str, Any]:
from core.agents.pipeline.stateless.resilience.rate_limiter import rate_limiter_registry
return rate_limiter_registry.to_dict()
@router.get("/backpressure")
async def get_backpressure(admin_user: Dict = Depends(require_admin)) -> Dict[str, Any]:
from core.agents.pipeline.stateless.resilience.backpressure import backpressure
return backpressure.to_dict()
@router.get("/metrics/history")
async def get_metrics_history(
minutes: int = Query(default=30, ge=5, le=120),
admin_user: Dict = Depends(require_admin)
) -> Dict[str, Any]:
from core.services import redis
from core.agents.pipeline.stateless import metrics
import json
import time
HISTORY_KEY = "stateless:metrics:history"
MAX_ENTRIES = 360
now = time.time()
current = {
"timestamp": now,
"active_runs": int(metrics.active_runs.get()),
"pending_writes": int(metrics.pending_writes.get()),
"runs_started": metrics.runs_started.get(),
"runs_completed": metrics.runs_completed.get(),
"runs_failed": metrics.runs_failed.get(),
"flush_latency_avg": metrics.flush_latency.avg(),
"flush_latency_p99": metrics.flush_latency.percentile(99),
"writes_dropped": metrics.writes_dropped.get(),
"dlq_entries": metrics.dlq_entries.get(),
}
try:
await redis.lpush(HISTORY_KEY, json.dumps(current))
await redis.ltrim(HISTORY_KEY, 0, MAX_ENTRIES - 1)
await redis.expire(HISTORY_KEY, 86400)
except Exception as e:
logger.warning(f"Failed to store metrics history: {e}")
cutoff = now - (minutes * 60)
history = []
try:
raw_entries = await redis.lrange(HISTORY_KEY, 0, MAX_ENTRIES)
for raw in raw_entries:
try:
entry = json.loads(raw)
if entry.get("timestamp", 0) >= cutoff:
history.append(entry)
except:
continue
history.reverse()
except Exception as e:
logger.warning(f"Failed to retrieve metrics history: {e}")
return {
"current": current,
"history": history,
"minutes": minutes,
}