File size: 3,249 Bytes
22087d0 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 | """
Agent run lifecycle tracking for cleanup failure diagnosis.
Usage:
from core.utils.lifecycle_tracker import log_run_start, log_run_cleanup, log_cleanup_error
log_run_start(agent_run_id, thread_id)
# ... run executes ...
log_cleanup_error(agent_run_id, "redis_expire", e) # if any cleanup step fails
log_run_cleanup(agent_run_id, success=True, final_status="completed")
"""
from typing import Optional, Dict, List
import time
from core.utils.logger import logger
# Track active runs: agent_run_id -> (start_time, thread_id)
_active_runs: Dict[str, tuple] = {}
def log_run_start(agent_run_id: str, thread_id: str) -> None:
"""Log when an agent run starts execution."""
_active_runs[agent_run_id] = (time.time(), thread_id)
logger.info(
f"[LIFECYCLE] START agent_run={agent_run_id} "
f"thread={thread_id} active_count={len(_active_runs)}"
)
def log_cleanup_error(agent_run_id: str, step: str, error: Exception) -> None:
"""Log when a specific cleanup step fails."""
logger.error(
f"[LIFECYCLE] CLEANUP_ERROR agent_run={agent_run_id} "
f"step={step} error={type(error).__name__}: {str(error)[:100]}"
)
def log_run_cleanup(
agent_run_id: str,
success: bool,
reason: Optional[str] = None,
final_status: Optional[str] = None,
cleanup_errors: Optional[List[str]] = None
) -> None:
"""Log when an agent run cleanup completes (success or failure)."""
start_info = _active_runs.pop(agent_run_id, None)
if start_info:
duration = time.time() - start_info[0]
thread_id = start_info[1]
else:
duration = 0.0
thread_id = "unknown"
if success:
logger.info(
f"[LIFECYCLE] CLEANUP_OK agent_run={agent_run_id} "
f"thread={thread_id} status={final_status} "
f"duration={duration:.1f}s active_count={len(_active_runs)}"
)
else:
errors_str = f" errors={cleanup_errors}" if cleanup_errors else ""
logger.error(
f"[LIFECYCLE] CLEANUP_FAIL agent_run={agent_run_id} "
f"thread={thread_id} reason={reason}{errors_str} "
f"duration={duration:.1f}s active_count={len(_active_runs)}"
)
def get_active_runs() -> Dict[str, float]:
"""Get currently active runs with their start times (for watchdog)."""
return {rid: info[0] for rid, info in _active_runs.items()}
def get_stale_runs(max_age_seconds: int = 3600) -> List[str]:
"""Get agent_run_ids that have been running longer than max_age_seconds."""
now = time.time()
return [
rid for rid, info in _active_runs.items()
if (now - info[0]) > max_age_seconds
]
def evict_stale_runs(max_age_seconds: int = 7200) -> int:
"""Remove entries older than max_age_seconds (default 2h) from _active_runs.
These are zombie entries where cleanup never ran (e.g. process crash mid-run).
"""
now = time.time()
stale = [
rid for rid, info in _active_runs.items()
if (now - info[0]) > max_age_seconds
]
for rid in stale:
_active_runs.pop(rid, None)
logger.warning(f"[LIFECYCLE] Evicted stale run {rid} from _active_runs (zombie)")
return len(stale)
|