Download core/utils/lifecycle_tracker.py from Kyrlos11/nniback: direct link, hf CLI and curl.
- Browser
- Download file 3.25 kB
-
https://huggingface.co/spaces/Kyrlos11/nniback/resolve/main/core/utils/lifecycle_tracker.py
- Command line
-
hf download hf://spaces/Kyrlos11/nniback/core/utils/lifecycle_tracker.py
-
curl -L -o lifecycle_tracker.py https://huggingface.co/spaces/Kyrlos11/nniback/resolve/main/core/utils/lifecycle_tracker.py
3.25 kB
| """ | |
| Agent run lifecycle tracking for cleanup failure diagnosis. | |
| Usage: | |
| from core.utils.lifecycle_tracker import log_run_start, log_run_cleanup, log_cleanup_error | |
| log_run_start(agent_run_id, thread_id) | |
| # ... run executes ... | |
| log_cleanup_error(agent_run_id, "redis_expire", e) # if any cleanup step fails | |
| log_run_cleanup(agent_run_id, success=True, final_status="completed") | |
| """ | |
| from typing import Optional, Dict, List | |
| import time | |
| from core.utils.logger import logger | |
| # Track active runs: agent_run_id -> (start_time, thread_id) | |
| _active_runs: Dict[str, tuple] = {} | |
| def log_run_start(agent_run_id: str, thread_id: str) -> None: | |
| """Log when an agent run starts execution.""" | |
| _active_runs[agent_run_id] = (time.time(), thread_id) | |
| logger.info( | |
| f"[LIFECYCLE] START agent_run={agent_run_id} " | |
| f"thread={thread_id} active_count={len(_active_runs)}" | |
| ) | |
| def log_cleanup_error(agent_run_id: str, step: str, error: Exception) -> None: | |
| """Log when a specific cleanup step fails.""" | |
| logger.error( | |
| f"[LIFECYCLE] CLEANUP_ERROR agent_run={agent_run_id} " | |
| f"step={step} error={type(error).__name__}: {str(error)[:100]}" | |
| ) | |
| def log_run_cleanup( | |
| agent_run_id: str, | |
| success: bool, | |
| reason: Optional[str] = None, | |
| final_status: Optional[str] = None, | |
| cleanup_errors: Optional[List[str]] = None | |
| ) -> None: | |
| """Log when an agent run cleanup completes (success or failure).""" | |
| start_info = _active_runs.pop(agent_run_id, None) | |
| if start_info: | |
| duration = time.time() - start_info[0] | |
| thread_id = start_info[1] | |
| else: | |
| duration = 0.0 | |
| thread_id = "unknown" | |
| if success: | |
| logger.info( | |
| f"[LIFECYCLE] CLEANUP_OK agent_run={agent_run_id} " | |
| f"thread={thread_id} status={final_status} " | |
| f"duration={duration:.1f}s active_count={len(_active_runs)}" | |
| ) | |
| else: | |
| errors_str = f" errors={cleanup_errors}" if cleanup_errors else "" | |
| logger.error( | |
| f"[LIFECYCLE] CLEANUP_FAIL agent_run={agent_run_id} " | |
| f"thread={thread_id} reason={reason}{errors_str} " | |
| f"duration={duration:.1f}s active_count={len(_active_runs)}" | |
| ) | |
| def get_active_runs() -> Dict[str, float]: | |
| """Get currently active runs with their start times (for watchdog).""" | |
| return {rid: info[0] for rid, info in _active_runs.items()} | |
| def get_stale_runs(max_age_seconds: int = 3600) -> List[str]: | |
| """Get agent_run_ids that have been running longer than max_age_seconds.""" | |
| now = time.time() | |
| return [ | |
| rid for rid, info in _active_runs.items() | |
| if (now - info[0]) > max_age_seconds | |
| ] | |
| def evict_stale_runs(max_age_seconds: int = 7200) -> int: | |
| """Remove entries older than max_age_seconds (default 2h) from _active_runs. | |
| These are zombie entries where cleanup never ran (e.g. process crash mid-run). | |
| """ | |
| now = time.time() | |
| stale = [ | |
| rid for rid, info in _active_runs.items() | |
| if (now - info[0]) > max_age_seconds | |
| ] | |
| for rid in stale: | |
| _active_runs.pop(rid, None) | |
| logger.warning(f"[LIFECYCLE] Evicted stale run {rid} from _active_runs (zombie)") | |
| return len(stale) | |