Spaces:
Running
Running
File size: 7,817 Bytes
e59bd9d | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 151 152 153 154 155 156 157 158 159 160 161 162 163 164 165 166 167 168 169 170 171 172 173 174 175 176 177 178 179 180 181 182 183 184 185 186 187 188 189 190 191 192 193 194 195 | """LPU Background Controller - the Windows-resident monitoring process.
Footprint by design: torch-free, ~15 MB RAM, near-zero CPU when idle.
State machine (spec section 11):
START -> HARDWARE DISCOVERY -> IDLE/MONITORING
IDLE --(compatible process found)--> ACTIVE MONITORING
ACTIVE --(process exits)--> IDLE + resource cleanup
Idle behavior: poll process list every idle_poll_seconds (default 5 s),
sample GPU state at most every gpu_sample_seconds (default 2 s) and ONLY
when idle_gpu_monitoring is enabled. No CUDA contexts are created here -
this process never initializes CUDA, so it can never interfere with a
training process's GPU state.
Active behavior: when a launched session exists (sessions dir) or a
discovered python process holds GPU memory, poll at active_poll_seconds and
append metric records for reporting.
"""
from __future__ import annotations
import json
import os
import sys
import time
from pathlib import Path
from typing import List, Optional
from ..config import load_config
from ..ipc import SessionChannel, list_active_sessions
from ..monitoring.logger import get_logger, lpu_home
from ..monitoring.gpu_monitor import GPUMonitor
from .lifecycle import Lifecycle, find_controller_pid
from .process_discovery import ProcessDiscovery
STATE_PATH = "status.json"
class BackgroundController:
def __init__(self, config=None):
self.config = config or load_config()
self.home = lpu_home()
self.log = get_logger(
"controller", log_dir=self.home / "logs",
level=self.config.get("monitoring.log_level", "INFO"),
)
self.lifecycle = Lifecycle("controller", log=self.log)
self.gpu_monitor = GPUMonitor(
sample_seconds=self.config.get("monitoring.gpu_sample_seconds", 2.0),
enabled=self.config.get("monitoring.enabled", True)
and self.config.get("monitoring.idle_gpu_monitoring", True),
log_dir=self.home / "logs",
)
self.discovery = ProcessDiscovery(
deep_scan=self.config.get("integration.deep_scan", False),
log_dir=self.home / "logs",
gpu_monitor=self.gpu_monitor,
)
self._state = "init"
self._session_metrics = None # JsonRecorder, opened on demand
self._last_discovery: List[dict] = []
self._started_ts = time.time()
# -- state ------------------------------------------------------------------
def write_state(self) -> None:
path = self.home / STATE_PATH
tmp = path.with_suffix(".tmp")
payload = {
"state": self._state,
"pid": os.getpid(),
"started_ts": self._started_ts,
"ts": time.time(),
"gpu_available": self.gpu_monitor.available,
"gpu_count": self.gpu_monitor.device_count,
"driver_version": self.gpu_monitor.driver_version(),
"gpus": [s.as_dict() for s in self.gpu_monitor.sample_all(force=True)]
if self.gpu_monitor.available else [],
"python_processes_seen": len(self._last_discovery),
"sessions_active": len(list_active_sessions(max_age_s=15.0)),
"config": str(self.config.path) if self.config.path else None,
}
try:
tmp.write_text(json.dumps(payload, indent=2), encoding="utf-8")
os.replace(tmp, path)
except OSError as exc: # pragma: no cover
self.log.warning("state write failed: %s", exc)
# -- main loop ---------------------------------------------------------------
def run(self, max_seconds: Optional[float] = None) -> str:
"""Blocking loop; returns exit reason string (for tests)."""
if not self.lifecycle.acquire_lock():
self.log.info("controller already running; this instance exits")
return "already_running"
self.lifecycle.clear_stop() # a leftover stop flag must not brick the next start
self.lifecycle.install_signal_handlers()
self.lifecycle.on_stop(self._cleanup)
self._state = "hardware_discovery"
self.log.info(
"controller starting: os=%s py=%s gpus=%d driver=%s",
os.name, f"{sys.version_info.major}.{sys.version_info.minor}",
self.gpu_monitor.device_count, self.gpu_monitor.driver_version(),
)
if not self.gpu_monitor.available:
self.log.warning(
"NVML unavailable (no driver or nvidia-ml-py missing): "
"GPU metrics will be reported as unavailable - never fabricated"
)
deadline = (time.time() + max_seconds) if max_seconds else None
try:
self._state = "idle"
while not self.lifecycle.stop_requested():
if deadline and time.time() > deadline:
return "timeout"
try:
self._tick()
except Exception: # noqa: BLE001 - one bad tick never kills the runtime
self.log.exception("tick error (continuing)")
time.sleep(self._sleep_seconds())
return "stopped"
finally:
self._cleanup()
self.lifecycle.release_lock()
def _sleep_seconds(self) -> float:
if self._state == "active":
return self.config.get("monitoring.active_poll_seconds", 1.0)
return self.config.get("monitoring.idle_poll_seconds", 5.0)
def _tick(self) -> None:
cfg_enabled = self.config.get("integration.process_discovery", True)
procs = self.discovery.scan() if cfg_enabled else []
self._last_discovery = [p.summary() for p in procs]
sessions = list_active_sessions(max_age_s=15.0)
gpu_busy = any(p.gpu_vram_mb for p in procs)
new_state = "active" if (sessions or gpu_busy) else "idle"
if new_state != self._state:
self.log.info("state %s -> %s (sessions=%d gpu_busy=%s)",
self._state, new_state, len(sessions), gpu_busy)
self._state = new_state
if self._state == "active" and self.gpu_monitor.available:
for s in self.gpu_monitor.sample_all():
self._record({"kind": "gpu", **s.as_dict()})
self.write_state()
def _record(self, record: dict) -> None:
if self._session_metrics is None:
from ..monitoring.logger import JsonRecorder
path = self.home / "logs" / "controller_metrics.jsonl"
self._session_metrics = JsonRecorder(path)
self._session_metrics.write(record)
def _cleanup(self) -> None:
try:
if self._session_metrics is not None:
self._session_metrics.close()
self._session_metrics = None
except Exception: # noqa: BLE001
pass
try:
self.gpu_monitor.close()
except Exception: # noqa: BLE001
pass
self._state = "shutdown"
try:
self.write_state()
except Exception: # noqa: BLE001
pass
self.log.info("controller cleanup complete")
def main(argv: Optional[List[str]] = None) -> int:
"""Entry point: python -m lpu start (or lpu.runtime.controller) [--max-seconds N]."""
argv = list(sys.argv[1:] if argv is None else argv)
max_s = None
if "--max-seconds" in argv:
i = argv.index("--max-seconds")
try:
max_s = float(argv[i + 1])
except (IndexError, ValueError):
print("--max-seconds needs a numeric value", file=sys.stderr)
return 2
ctrl = BackgroundController()
reason = ctrl.run(max_seconds=max_s)
print(f"[lpu] controller exited: {reason}")
return 0
if __name__ == "__main__":
raise SystemExit(main())
|