Virtual-LPU / lpu /runtime /controller.py
iparaco's picture
Upload 118 files
e59bd9d verified
Raw History Blame Contribute Delete
7.82 kB
"""LPU Background Controller - the Windows-resident monitoring process.
Footprint by design: torch-free, ~15 MB RAM, near-zero CPU when idle.
State machine (spec section 11):
START -> HARDWARE DISCOVERY -> IDLE/MONITORING
IDLE --(compatible process found)--> ACTIVE MONITORING
ACTIVE --(process exits)--> IDLE + resource cleanup
Idle behavior: poll process list every idle_poll_seconds (default 5 s),
sample GPU state at most every gpu_sample_seconds (default 2 s) and ONLY
when idle_gpu_monitoring is enabled. No CUDA contexts are created here -
this process never initializes CUDA, so it can never interfere with a
training process's GPU state.
Active behavior: when a launched session exists (sessions dir) or a
discovered python process holds GPU memory, poll at active_poll_seconds and
append metric records for reporting.
"""
from __future__ import annotations
import json
import os
import sys
import time
from pathlib import Path
from typing import List, Optional
from ..config import load_config
from ..ipc import SessionChannel, list_active_sessions
from ..monitoring.logger import get_logger, lpu_home
from ..monitoring.gpu_monitor import GPUMonitor
from .lifecycle import Lifecycle, find_controller_pid
from .process_discovery import ProcessDiscovery
STATE_PATH = "status.json"
class BackgroundController:
def __init__(self, config=None):
self.config = config or load_config()
self.home = lpu_home()
self.log = get_logger(
"controller", log_dir=self.home / "logs",
level=self.config.get("monitoring.log_level", "INFO"),
)
self.lifecycle = Lifecycle("controller", log=self.log)
self.gpu_monitor = GPUMonitor(
sample_seconds=self.config.get("monitoring.gpu_sample_seconds", 2.0),
enabled=self.config.get("monitoring.enabled", True)
and self.config.get("monitoring.idle_gpu_monitoring", True),
log_dir=self.home / "logs",
)
self.discovery = ProcessDiscovery(
deep_scan=self.config.get("integration.deep_scan", False),
log_dir=self.home / "logs",
gpu_monitor=self.gpu_monitor,
)
self._state = "init"
self._session_metrics = None # JsonRecorder, opened on demand
self._last_discovery: List[dict] = []
self._started_ts = time.time()
# -- state ------------------------------------------------------------------
def write_state(self) -> None:
path = self.home / STATE_PATH
tmp = path.with_suffix(".tmp")
payload = {
"state": self._state,
"pid": os.getpid(),
"started_ts": self._started_ts,
"ts": time.time(),
"gpu_available": self.gpu_monitor.available,
"gpu_count": self.gpu_monitor.device_count,
"driver_version": self.gpu_monitor.driver_version(),
"gpus": [s.as_dict() for s in self.gpu_monitor.sample_all(force=True)]
if self.gpu_monitor.available else [],
"python_processes_seen": len(self._last_discovery),
"sessions_active": len(list_active_sessions(max_age_s=15.0)),
"config": str(self.config.path) if self.config.path else None,
}
try:
tmp.write_text(json.dumps(payload, indent=2), encoding="utf-8")
os.replace(tmp, path)
except OSError as exc: # pragma: no cover
self.log.warning("state write failed: %s", exc)
# -- main loop ---------------------------------------------------------------
def run(self, max_seconds: Optional[float] = None) -> str:
"""Blocking loop; returns exit reason string (for tests)."""
if not self.lifecycle.acquire_lock():
self.log.info("controller already running; this instance exits")
return "already_running"
self.lifecycle.clear_stop() # a leftover stop flag must not brick the next start
self.lifecycle.install_signal_handlers()
self.lifecycle.on_stop(self._cleanup)
self._state = "hardware_discovery"
self.log.info(
"controller starting: os=%s py=%s gpus=%d driver=%s",
os.name, f"{sys.version_info.major}.{sys.version_info.minor}",
self.gpu_monitor.device_count, self.gpu_monitor.driver_version(),
)
if not self.gpu_monitor.available:
self.log.warning(
"NVML unavailable (no driver or nvidia-ml-py missing): "
"GPU metrics will be reported as unavailable - never fabricated"
)
deadline = (time.time() + max_seconds) if max_seconds else None
try:
self._state = "idle"
while not self.lifecycle.stop_requested():
if deadline and time.time() > deadline:
return "timeout"
try:
self._tick()
except Exception: # noqa: BLE001 - one bad tick never kills the runtime
self.log.exception("tick error (continuing)")
time.sleep(self._sleep_seconds())
return "stopped"
finally:
self._cleanup()
self.lifecycle.release_lock()
def _sleep_seconds(self) -> float:
if self._state == "active":
return self.config.get("monitoring.active_poll_seconds", 1.0)
return self.config.get("monitoring.idle_poll_seconds", 5.0)
def _tick(self) -> None:
cfg_enabled = self.config.get("integration.process_discovery", True)
procs = self.discovery.scan() if cfg_enabled else []
self._last_discovery = [p.summary() for p in procs]
sessions = list_active_sessions(max_age_s=15.0)
gpu_busy = any(p.gpu_vram_mb for p in procs)
new_state = "active" if (sessions or gpu_busy) else "idle"
if new_state != self._state:
self.log.info("state %s -> %s (sessions=%d gpu_busy=%s)",
self._state, new_state, len(sessions), gpu_busy)
self._state = new_state
if self._state == "active" and self.gpu_monitor.available:
for s in self.gpu_monitor.sample_all():
self._record({"kind": "gpu", **s.as_dict()})
self.write_state()
def _record(self, record: dict) -> None:
if self._session_metrics is None:
from ..monitoring.logger import JsonRecorder
path = self.home / "logs" / "controller_metrics.jsonl"
self._session_metrics = JsonRecorder(path)
self._session_metrics.write(record)
def _cleanup(self) -> None:
try:
if self._session_metrics is not None:
self._session_metrics.close()
self._session_metrics = None
except Exception: # noqa: BLE001
pass
try:
self.gpu_monitor.close()
except Exception: # noqa: BLE001
pass
self._state = "shutdown"
try:
self.write_state()
except Exception: # noqa: BLE001
pass
self.log.info("controller cleanup complete")
def main(argv: Optional[List[str]] = None) -> int:
"""Entry point: python -m lpu start (or lpu.runtime.controller) [--max-seconds N]."""
argv = list(sys.argv[1:] if argv is None else argv)
max_s = None
if "--max-seconds" in argv:
i = argv.index("--max-seconds")
try:
max_s = float(argv[i + 1])
except (IndexError, ValueError):
print("--max-seconds needs a numeric value", file=sys.stderr)
return 2
ctrl = BackgroundController()
reason = ctrl.run(max_seconds=max_s)
print(f"[lpu] controller exited: {reason}")
return 0
if __name__ == "__main__":
raise SystemExit(main())