Download core/admin/stateless_admin_api.py from Kyrlos11/nniback: direct link, hf CLI and curl.
- Browser
- Download file 11.6 kB
-
https://huggingface.co/spaces/Kyrlos11/nniback/resolve/main/core/admin/stateless_admin_api.py
- Command line
-
hf download hf://spaces/Kyrlos11/nniback/core/admin/stateless_admin_api.py
-
curl -L -o stateless_admin_api.py https://huggingface.co/spaces/Kyrlos11/nniback/resolve/main/core/admin/stateless_admin_api.py
11.6 kB
| from fastapi import APIRouter, HTTPException, Depends, Query | |
| from typing import Dict, Any, List, Optional | |
| from pydantic import BaseModel | |
| from core.utils.logger import logger | |
| from core.auth import require_admin | |
| class RecoveryResponse(BaseModel): | |
| run_id: str | |
| success: bool | |
| action: str | |
| message: str | |
| error: Optional[str] = None | |
| class StuckRunResponse(BaseModel): | |
| run_id: str | |
| thread_id: Optional[str] | |
| owner: Optional[str] | |
| status: Optional[str] | |
| heartbeat: Optional[float] | |
| heartbeat_age: Optional[float] | |
| start: Optional[float] | |
| duration: Optional[float] | |
| reason: Optional[str] | |
| class ForceFailRequest(BaseModel): | |
| error: str = "Admin terminated" | |
| class DLQEntryResponse(BaseModel): | |
| entry_id: str | |
| run_id: str | |
| write_type: str | |
| error: str | |
| attempt_count: int | |
| created_at: float | |
| failed_at: float | |
| router = APIRouter(prefix="/admin/stateless", tags=["admin-stateless"]) | |
| async def list_stuck_runs( | |
| min_age: int = Query(default=5, ge=1, le=120), | |
| admin_user: Dict = Depends(require_admin) | |
| ) -> List[StuckRunResponse]: | |
| from core.agents.pipeline.stateless import recovery | |
| stuck = await recovery.get_stuck(min_age) | |
| logger.info(f"[Admin] {admin_user.get('id')} listed {len(stuck)} stuck runs") | |
| return [StuckRunResponse(**r) for r in stuck] | |
| async def force_resume( | |
| run_id: str, | |
| admin_user: Dict = Depends(require_admin) | |
| ) -> RecoveryResponse: | |
| from core.agents.pipeline.stateless import recovery | |
| result = await recovery.force_resume(run_id) | |
| logger.info(f"[Admin] {admin_user.get('id')} resumed {run_id}: {result.success}") | |
| return RecoveryResponse( | |
| run_id=result.run_id, | |
| success=result.success, | |
| action=result.action, | |
| message=result.message, | |
| error=result.error, | |
| ) | |
| async def force_complete( | |
| run_id: str, | |
| reason: str = Query(default="admin"), | |
| admin_user: Dict = Depends(require_admin) | |
| ) -> RecoveryResponse: | |
| from core.agents.pipeline.stateless import recovery | |
| result = await recovery.force_complete(run_id, reason) | |
| logger.info(f"[Admin] {admin_user.get('id')} completed {run_id}: {reason}") | |
| return RecoveryResponse( | |
| run_id=result.run_id, | |
| success=result.success, | |
| action=result.action, | |
| message=result.message, | |
| error=result.error, | |
| ) | |
| async def force_fail( | |
| run_id: str, | |
| request: ForceFailRequest, | |
| admin_user: Dict = Depends(require_admin) | |
| ) -> RecoveryResponse: | |
| from core.agents.pipeline.stateless import recovery | |
| result = await recovery.force_fail(run_id, request.error) | |
| logger.info(f"[Admin] {admin_user.get('id')} failed {run_id}: {request.error}") | |
| return RecoveryResponse( | |
| run_id=result.run_id, | |
| success=result.success, | |
| action=result.action, | |
| message=result.message, | |
| error=result.error, | |
| ) | |
| async def get_run_info( | |
| run_id: str, | |
| admin_user: Dict = Depends(require_admin) | |
| ) -> Dict[str, Any]: | |
| from core.agents.pipeline.stateless import ownership | |
| info = await ownership.get_info(run_id) | |
| if not info: | |
| raise HTTPException(status_code=404, detail=f"Run {run_id} not found") | |
| return info | |
| async def get_dashboard(admin_user: Dict = Depends(require_admin)) -> Dict[str, Any]: | |
| from core.agents.pipeline.stateless import metrics, recovery, ownership, write_buffer | |
| from core.agents.pipeline.stateless.persistence import wal, dlq | |
| from core.agents import repo as agents_repo | |
| stuck = await recovery.get_stuck(5) | |
| health = metrics.check_health() | |
| wal_stats = await wal.get_stats() | |
| dlq_stats = await dlq.get_stats() | |
| active_threads_count = await agents_repo.get_active_thread_ids_count() | |
| return { | |
| "active_runs": active_threads_count, | |
| "active_runs_metric": int(metrics.active_runs.get()), | |
| "owned_runs": ownership.owned_count, | |
| "pending_writes": int(metrics.pending_writes.get()), | |
| "stuck_count": len(stuck), | |
| "runs_started": metrics.runs_started.get(), | |
| "runs_completed": metrics.runs_completed.get(), | |
| "runs_failed": metrics.runs_failed.get(), | |
| "runs_recovered": metrics.runs_recovered.get(), | |
| "runs_rejected": metrics.runs_rejected.get(), | |
| "flush_latency_avg": metrics.flush_latency.avg(), | |
| "flush_latency_p99": metrics.flush_latency.percentile(99), | |
| "wal": wal_stats, | |
| "dlq": dlq_stats, | |
| "healthy": health["healthy"], | |
| "alerts": health["alerts"], | |
| } | |
| async def get_active_threads(admin_user: Dict = Depends(require_admin)) -> Dict[str, Any]: | |
| from core.agents import repo as agents_repo | |
| active_threads = await agents_repo.get_active_thread_ids_with_runs() | |
| return { | |
| "count": len(active_threads), | |
| "threads": active_threads, | |
| } | |
| async def get_health() -> Dict[str, Any]: | |
| from core.agents.pipeline.stateless import lifecycle | |
| from fastapi.responses import JSONResponse | |
| health = await lifecycle.get_health() | |
| if health.get("shutting_down") or not health.get("healthy"): | |
| return JSONResponse(content=health, status_code=503) | |
| return health | |
| async def get_metrics(admin_user: Dict = Depends(require_admin)) -> str: | |
| from core.agents.pipeline.stateless import metrics | |
| return metrics.to_prometheus() | |
| async def trigger_sweep(admin_user: Dict = Depends(require_admin)) -> Dict[str, Any]: | |
| from core.agents.pipeline.stateless import recovery | |
| result = await recovery.sweep() | |
| logger.info(f"[Admin] {admin_user.get('id')} triggered sweep: {result}") | |
| return result | |
| async def trigger_flush(admin_user: Dict = Depends(require_admin)) -> Dict[str, Any]: | |
| from core.agents.pipeline.stateless import write_buffer | |
| result = await write_buffer.flush_all() | |
| total = sum(result.values()) | |
| logger.info(f"[Admin] {admin_user.get('id')} triggered flush: {total} writes") | |
| return {"runs": len(result), "total": total, "details": result} | |
| async def list_dlq_entries( | |
| count: int = Query(default=50, ge=1, le=500), | |
| run_id: Optional[str] = None, | |
| admin_user: Dict = Depends(require_admin) | |
| ) -> List[DLQEntryResponse]: | |
| from core.agents.pipeline.stateless.persistence import dlq | |
| entries = await dlq.get_entries(count=count, run_id=run_id) | |
| return [ | |
| DLQEntryResponse( | |
| entry_id=e.entry_id, | |
| run_id=e.run_id, | |
| write_type=e.write_type, | |
| error=e.error, | |
| attempt_count=e.attempt_count, | |
| created_at=e.created_at, | |
| failed_at=e.failed_at, | |
| ) | |
| for e in entries | |
| ] | |
| async def retry_dlq_entry( | |
| entry_id: str, | |
| admin_user: Dict = Depends(require_admin) | |
| ) -> Dict[str, Any]: | |
| from core.agents.pipeline.stateless.persistence import dlq | |
| success = await dlq.retry_entry(entry_id) | |
| logger.info(f"[Admin] {admin_user.get('id')} retried DLQ entry {entry_id}: {success}") | |
| return {"success": success, "entry_id": entry_id} | |
| async def delete_dlq_entry( | |
| entry_id: str, | |
| admin_user: Dict = Depends(require_admin) | |
| ) -> Dict[str, Any]: | |
| from core.agents.pipeline.stateless.persistence import dlq | |
| success = await dlq.delete_entry(entry_id) | |
| logger.info(f"[Admin] {admin_user.get('id')} deleted DLQ entry {entry_id}: {success}") | |
| return {"success": success, "entry_id": entry_id} | |
| async def purge_dlq( | |
| older_than_hours: Optional[int] = Query(default=None, ge=1, le=168), | |
| admin_user: Dict = Depends(require_admin) | |
| ) -> Dict[str, Any]: | |
| from core.agents.pipeline.stateless.persistence import dlq | |
| older_than_seconds = older_than_hours * 3600 if older_than_hours else None | |
| deleted = await dlq.purge(older_than_seconds) | |
| logger.info(f"[Admin] {admin_user.get('id')} purged DLQ: {deleted} entries") | |
| return {"deleted": deleted} | |
| async def get_wal_stats(admin_user: Dict = Depends(require_admin)) -> Dict[str, Any]: | |
| from core.agents.pipeline.stateless.persistence import wal | |
| return await wal.get_stats() | |
| async def get_circuit_breakers(admin_user: Dict = Depends(require_admin)) -> Dict[str, Any]: | |
| from core.agents.pipeline.stateless.resilience.circuit_breaker import registry | |
| return registry.to_dict() | |
| async def reset_circuit_breakers(admin_user: Dict = Depends(require_admin)) -> Dict[str, Any]: | |
| from core.agents.pipeline.stateless.resilience.circuit_breaker import registry | |
| registry.reset_all() | |
| logger.info(f"[Admin] {admin_user.get('id')} reset all circuit breakers") | |
| return {"success": True} | |
| async def get_rate_limiters(admin_user: Dict = Depends(require_admin)) -> Dict[str, Any]: | |
| from core.agents.pipeline.stateless.resilience.rate_limiter import rate_limiter_registry | |
| return rate_limiter_registry.to_dict() | |
| async def get_backpressure(admin_user: Dict = Depends(require_admin)) -> Dict[str, Any]: | |
| from core.agents.pipeline.stateless.resilience.backpressure import backpressure | |
| return backpressure.to_dict() | |
| async def get_metrics_history( | |
| minutes: int = Query(default=30, ge=5, le=120), | |
| admin_user: Dict = Depends(require_admin) | |
| ) -> Dict[str, Any]: | |
| from core.services import redis | |
| from core.agents.pipeline.stateless import metrics | |
| import json | |
| import time | |
| HISTORY_KEY = "stateless:metrics:history" | |
| MAX_ENTRIES = 360 | |
| now = time.time() | |
| current = { | |
| "timestamp": now, | |
| "active_runs": int(metrics.active_runs.get()), | |
| "pending_writes": int(metrics.pending_writes.get()), | |
| "runs_started": metrics.runs_started.get(), | |
| "runs_completed": metrics.runs_completed.get(), | |
| "runs_failed": metrics.runs_failed.get(), | |
| "flush_latency_avg": metrics.flush_latency.avg(), | |
| "flush_latency_p99": metrics.flush_latency.percentile(99), | |
| "writes_dropped": metrics.writes_dropped.get(), | |
| "dlq_entries": metrics.dlq_entries.get(), | |
| } | |
| try: | |
| await redis.lpush(HISTORY_KEY, json.dumps(current)) | |
| await redis.ltrim(HISTORY_KEY, 0, MAX_ENTRIES - 1) | |
| await redis.expire(HISTORY_KEY, 86400) | |
| except Exception as e: | |
| logger.warning(f"Failed to store metrics history: {e}") | |
| cutoff = now - (minutes * 60) | |
| history = [] | |
| try: | |
| raw_entries = await redis.lrange(HISTORY_KEY, 0, MAX_ENTRIES) | |
| for raw in raw_entries: | |
| try: | |
| entry = json.loads(raw) | |
| if entry.get("timestamp", 0) >= cutoff: | |
| history.append(entry) | |
| except: | |
| continue | |
| history.reverse() | |
| except Exception as e: | |
| logger.warning(f"Failed to retrieve metrics history: {e}") | |
| return { | |
| "current": current, | |
| "history": history, | |
| "minutes": minutes, | |
| } | |