thio / core /utils /lifecycle_tracker.py
kyrlos adel
cva1
7cfdc4c
Raw History Blame Contribute Delete
2.71 kB
"""
Agent run lifecycle tracking for cleanup failure diagnosis.
Usage:
from core.utils.lifecycle_tracker import log_run_start, log_run_cleanup, log_cleanup_error
log_run_start(agent_run_id, thread_id)
# ... run executes ...
log_cleanup_error(agent_run_id, "redis_expire", e) # if any cleanup step fails
log_run_cleanup(agent_run_id, success=True, final_status="completed")
"""
from typing import Optional, Dict, List
import time
from core.utils.logger import logger
# Track active runs: agent_run_id -> (start_time, thread_id)
_active_runs: Dict[str, tuple] = {}
def log_run_start(agent_run_id: str, thread_id: str) -> None:
"""Log when an agent run starts execution."""
_active_runs[agent_run_id] = (time.time(), thread_id)
logger.info(
f"[LIFECYCLE] START agent_run={agent_run_id} "
f"thread={thread_id} active_count={len(_active_runs)}"
)
def log_cleanup_error(agent_run_id: str, step: str, error: Exception) -> None:
"""Log when a specific cleanup step fails."""
logger.error(
f"[LIFECYCLE] CLEANUP_ERROR agent_run={agent_run_id} "
f"step={step} error={type(error).__name__}: {str(error)[:100]}"
)
def log_run_cleanup(
agent_run_id: str,
success: bool,
reason: Optional[str] = None,
final_status: Optional[str] = None,
cleanup_errors: Optional[List[str]] = None
) -> None:
"""Log when an agent run cleanup completes (success or failure)."""
start_info = _active_runs.pop(agent_run_id, None)
if start_info:
duration = time.time() - start_info[0]
thread_id = start_info[1]
else:
duration = 0.0
thread_id = "unknown"
if success:
logger.info(
f"[LIFECYCLE] CLEANUP_OK agent_run={agent_run_id} "
f"thread={thread_id} status={final_status} "
f"duration={duration:.1f}s active_count={len(_active_runs)}"
)
else:
errors_str = f" errors={cleanup_errors}" if cleanup_errors else ""
logger.error(
f"[LIFECYCLE] CLEANUP_FAIL agent_run={agent_run_id} "
f"thread={thread_id} reason={reason}{errors_str} "
f"duration={duration:.1f}s active_count={len(_active_runs)}"
)
def get_active_runs() -> Dict[str, float]:
"""Get currently active runs with their start times (for watchdog)."""
return {rid: info[0] for rid, info in _active_runs.items()}
def get_stale_runs(max_age_seconds: int = 3600) -> List[str]:
"""Get agent_run_ids that have been running longer than max_age_seconds."""
now = time.time()
return [
rid for rid, info in _active_runs.items()
if (now - info[0]) > max_age_seconds
]