File size: 3,249 Bytes
22087d0
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
"""
Agent run lifecycle tracking for cleanup failure diagnosis.

Usage:
    from core.utils.lifecycle_tracker import log_run_start, log_run_cleanup, log_cleanup_error

    log_run_start(agent_run_id, thread_id)
    # ... run executes ...
    log_cleanup_error(agent_run_id, "redis_expire", e)  # if any cleanup step fails
    log_run_cleanup(agent_run_id, success=True, final_status="completed")
"""

from typing import Optional, Dict, List
import time
from core.utils.logger import logger

# Track active runs: agent_run_id -> (start_time, thread_id)
_active_runs: Dict[str, tuple] = {}


def log_run_start(agent_run_id: str, thread_id: str) -> None:
    """Log when an agent run starts execution."""
    _active_runs[agent_run_id] = (time.time(), thread_id)
    logger.info(
        f"[LIFECYCLE] START agent_run={agent_run_id} "
        f"thread={thread_id} active_count={len(_active_runs)}"
    )


def log_cleanup_error(agent_run_id: str, step: str, error: Exception) -> None:
    """Log when a specific cleanup step fails."""
    logger.error(
        f"[LIFECYCLE] CLEANUP_ERROR agent_run={agent_run_id} "
        f"step={step} error={type(error).__name__}: {str(error)[:100]}"
    )


def log_run_cleanup(
    agent_run_id: str, 
    success: bool, 
    reason: Optional[str] = None, 
    final_status: Optional[str] = None,
    cleanup_errors: Optional[List[str]] = None
) -> None:
    """Log when an agent run cleanup completes (success or failure)."""
    start_info = _active_runs.pop(agent_run_id, None)
    if start_info:
        duration = time.time() - start_info[0]
        thread_id = start_info[1]
    else:
        duration = 0.0
        thread_id = "unknown"
    
    if success:
        logger.info(
            f"[LIFECYCLE] CLEANUP_OK agent_run={agent_run_id} "
            f"thread={thread_id} status={final_status} "
            f"duration={duration:.1f}s active_count={len(_active_runs)}"
        )
    else:
        errors_str = f" errors={cleanup_errors}" if cleanup_errors else ""
        logger.error(
            f"[LIFECYCLE] CLEANUP_FAIL agent_run={agent_run_id} "
            f"thread={thread_id} reason={reason}{errors_str} "
            f"duration={duration:.1f}s active_count={len(_active_runs)}"
        )


def get_active_runs() -> Dict[str, float]:
    """Get currently active runs with their start times (for watchdog)."""
    return {rid: info[0] for rid, info in _active_runs.items()}


def get_stale_runs(max_age_seconds: int = 3600) -> List[str]:
    """Get agent_run_ids that have been running longer than max_age_seconds."""
    now = time.time()
    return [
        rid for rid, info in _active_runs.items()
        if (now - info[0]) > max_age_seconds
    ]


def evict_stale_runs(max_age_seconds: int = 7200) -> int:
    """Remove entries older than max_age_seconds (default 2h) from _active_runs.

    These are zombie entries where cleanup never ran (e.g. process crash mid-run).
    """
    now = time.time()
    stale = [
        rid for rid, info in _active_runs.items()
        if (now - info[0]) > max_age_seconds
    ]
    for rid in stale:
        _active_runs.pop(rid, None)
        logger.warning(f"[LIFECYCLE] Evicted stale run {rid} from _active_runs (zombie)")
    return len(stale)