"""LPU Background Controller - the Windows-resident monitoring process. Footprint by design: torch-free, ~15 MB RAM, near-zero CPU when idle. State machine (spec section 11): START -> HARDWARE DISCOVERY -> IDLE/MONITORING IDLE --(compatible process found)--> ACTIVE MONITORING ACTIVE --(process exits)--> IDLE + resource cleanup Idle behavior: poll process list every idle_poll_seconds (default 5 s), sample GPU state at most every gpu_sample_seconds (default 2 s) and ONLY when idle_gpu_monitoring is enabled. No CUDA contexts are created here - this process never initializes CUDA, so it can never interfere with a training process's GPU state. Active behavior: when a launched session exists (sessions dir) or a discovered python process holds GPU memory, poll at active_poll_seconds and append metric records for reporting. """ from __future__ import annotations import json import os import sys import time from pathlib import Path from typing import List, Optional from ..config import load_config from ..ipc import SessionChannel, list_active_sessions from ..monitoring.logger import get_logger, lpu_home from ..monitoring.gpu_monitor import GPUMonitor from .lifecycle import Lifecycle, find_controller_pid from .process_discovery import ProcessDiscovery STATE_PATH = "status.json" class BackgroundController: def __init__(self, config=None): self.config = config or load_config() self.home = lpu_home() self.log = get_logger( "controller", log_dir=self.home / "logs", level=self.config.get("monitoring.log_level", "INFO"), ) self.lifecycle = Lifecycle("controller", log=self.log) self.gpu_monitor = GPUMonitor( sample_seconds=self.config.get("monitoring.gpu_sample_seconds", 2.0), enabled=self.config.get("monitoring.enabled", True) and self.config.get("monitoring.idle_gpu_monitoring", True), log_dir=self.home / "logs", ) self.discovery = ProcessDiscovery( deep_scan=self.config.get("integration.deep_scan", False), log_dir=self.home / "logs", gpu_monitor=self.gpu_monitor, ) self._state = "init" self._session_metrics = None # JsonRecorder, opened on demand self._last_discovery: List[dict] = [] self._started_ts = time.time() # -- state ------------------------------------------------------------------ def write_state(self) -> None: path = self.home / STATE_PATH tmp = path.with_suffix(".tmp") payload = { "state": self._state, "pid": os.getpid(), "started_ts": self._started_ts, "ts": time.time(), "gpu_available": self.gpu_monitor.available, "gpu_count": self.gpu_monitor.device_count, "driver_version": self.gpu_monitor.driver_version(), "gpus": [s.as_dict() for s in self.gpu_monitor.sample_all(force=True)] if self.gpu_monitor.available else [], "python_processes_seen": len(self._last_discovery), "sessions_active": len(list_active_sessions(max_age_s=15.0)), "config": str(self.config.path) if self.config.path else None, } try: tmp.write_text(json.dumps(payload, indent=2), encoding="utf-8") os.replace(tmp, path) except OSError as exc: # pragma: no cover self.log.warning("state write failed: %s", exc) # -- main loop --------------------------------------------------------------- def run(self, max_seconds: Optional[float] = None) -> str: """Blocking loop; returns exit reason string (for tests).""" if not self.lifecycle.acquire_lock(): self.log.info("controller already running; this instance exits") return "already_running" self.lifecycle.clear_stop() # a leftover stop flag must not brick the next start self.lifecycle.install_signal_handlers() self.lifecycle.on_stop(self._cleanup) self._state = "hardware_discovery" self.log.info( "controller starting: os=%s py=%s gpus=%d driver=%s", os.name, f"{sys.version_info.major}.{sys.version_info.minor}", self.gpu_monitor.device_count, self.gpu_monitor.driver_version(), ) if not self.gpu_monitor.available: self.log.warning( "NVML unavailable (no driver or nvidia-ml-py missing): " "GPU metrics will be reported as unavailable - never fabricated" ) deadline = (time.time() + max_seconds) if max_seconds else None try: self._state = "idle" while not self.lifecycle.stop_requested(): if deadline and time.time() > deadline: return "timeout" try: self._tick() except Exception: # noqa: BLE001 - one bad tick never kills the runtime self.log.exception("tick error (continuing)") time.sleep(self._sleep_seconds()) return "stopped" finally: self._cleanup() self.lifecycle.release_lock() def _sleep_seconds(self) -> float: if self._state == "active": return self.config.get("monitoring.active_poll_seconds", 1.0) return self.config.get("monitoring.idle_poll_seconds", 5.0) def _tick(self) -> None: cfg_enabled = self.config.get("integration.process_discovery", True) procs = self.discovery.scan() if cfg_enabled else [] self._last_discovery = [p.summary() for p in procs] sessions = list_active_sessions(max_age_s=15.0) gpu_busy = any(p.gpu_vram_mb for p in procs) new_state = "active" if (sessions or gpu_busy) else "idle" if new_state != self._state: self.log.info("state %s -> %s (sessions=%d gpu_busy=%s)", self._state, new_state, len(sessions), gpu_busy) self._state = new_state if self._state == "active" and self.gpu_monitor.available: for s in self.gpu_monitor.sample_all(): self._record({"kind": "gpu", **s.as_dict()}) self.write_state() def _record(self, record: dict) -> None: if self._session_metrics is None: from ..monitoring.logger import JsonRecorder path = self.home / "logs" / "controller_metrics.jsonl" self._session_metrics = JsonRecorder(path) self._session_metrics.write(record) def _cleanup(self) -> None: try: if self._session_metrics is not None: self._session_metrics.close() self._session_metrics = None except Exception: # noqa: BLE001 pass try: self.gpu_monitor.close() except Exception: # noqa: BLE001 pass self._state = "shutdown" try: self.write_state() except Exception: # noqa: BLE001 pass self.log.info("controller cleanup complete") def main(argv: Optional[List[str]] = None) -> int: """Entry point: python -m lpu start (or lpu.runtime.controller) [--max-seconds N].""" argv = list(sys.argv[1:] if argv is None else argv) max_s = None if "--max-seconds" in argv: i = argv.index("--max-seconds") try: max_s = float(argv[i + 1]) except (IndexError, ValueError): print("--max-seconds needs a numeric value", file=sys.stderr) return 2 ctrl = BackgroundController() reason = ctrl.run(max_seconds=max_s) print(f"[lpu] controller exited: {reason}") return 0 if __name__ == "__main__": raise SystemExit(main())