Spaces:
Running
Running
Download lpu/runtime/controller.py from iparaco/Virtual-LPU: direct link, hf CLI and curl.
- Browser
- Download file 7.82 kB
-
https://huggingface.co/spaces/iparaco/Virtual-LPU/resolve/refs%2Fpr%2F2/lpu/runtime/controller.py
- Command line
-
hf download hf://spaces/iparaco/Virtual-LPU@refs/pr/2/lpu/runtime/controller.py
-
curl -L -o controller.py https://huggingface.co/spaces/iparaco/Virtual-LPU/resolve/refs%2Fpr%2F2/lpu/runtime/controller.py
7.82 kB
| """LPU Background Controller - the Windows-resident monitoring process. | |
| Footprint by design: torch-free, ~15 MB RAM, near-zero CPU when idle. | |
| State machine (spec section 11): | |
| START -> HARDWARE DISCOVERY -> IDLE/MONITORING | |
| IDLE --(compatible process found)--> ACTIVE MONITORING | |
| ACTIVE --(process exits)--> IDLE + resource cleanup | |
| Idle behavior: poll process list every idle_poll_seconds (default 5 s), | |
| sample GPU state at most every gpu_sample_seconds (default 2 s) and ONLY | |
| when idle_gpu_monitoring is enabled. No CUDA contexts are created here - | |
| this process never initializes CUDA, so it can never interfere with a | |
| training process's GPU state. | |
| Active behavior: when a launched session exists (sessions dir) or a | |
| discovered python process holds GPU memory, poll at active_poll_seconds and | |
| append metric records for reporting. | |
| """ | |
| from __future__ import annotations | |
| import json | |
| import os | |
| import sys | |
| import time | |
| from pathlib import Path | |
| from typing import List, Optional | |
| from ..config import load_config | |
| from ..ipc import SessionChannel, list_active_sessions | |
| from ..monitoring.logger import get_logger, lpu_home | |
| from ..monitoring.gpu_monitor import GPUMonitor | |
| from .lifecycle import Lifecycle, find_controller_pid | |
| from .process_discovery import ProcessDiscovery | |
| STATE_PATH = "status.json" | |
| class BackgroundController: | |
| def __init__(self, config=None): | |
| self.config = config or load_config() | |
| self.home = lpu_home() | |
| self.log = get_logger( | |
| "controller", log_dir=self.home / "logs", | |
| level=self.config.get("monitoring.log_level", "INFO"), | |
| ) | |
| self.lifecycle = Lifecycle("controller", log=self.log) | |
| self.gpu_monitor = GPUMonitor( | |
| sample_seconds=self.config.get("monitoring.gpu_sample_seconds", 2.0), | |
| enabled=self.config.get("monitoring.enabled", True) | |
| and self.config.get("monitoring.idle_gpu_monitoring", True), | |
| log_dir=self.home / "logs", | |
| ) | |
| self.discovery = ProcessDiscovery( | |
| deep_scan=self.config.get("integration.deep_scan", False), | |
| log_dir=self.home / "logs", | |
| gpu_monitor=self.gpu_monitor, | |
| ) | |
| self._state = "init" | |
| self._session_metrics = None # JsonRecorder, opened on demand | |
| self._last_discovery: List[dict] = [] | |
| self._started_ts = time.time() | |
| # -- state ------------------------------------------------------------------ | |
| def write_state(self) -> None: | |
| path = self.home / STATE_PATH | |
| tmp = path.with_suffix(".tmp") | |
| payload = { | |
| "state": self._state, | |
| "pid": os.getpid(), | |
| "started_ts": self._started_ts, | |
| "ts": time.time(), | |
| "gpu_available": self.gpu_monitor.available, | |
| "gpu_count": self.gpu_monitor.device_count, | |
| "driver_version": self.gpu_monitor.driver_version(), | |
| "gpus": [s.as_dict() for s in self.gpu_monitor.sample_all(force=True)] | |
| if self.gpu_monitor.available else [], | |
| "python_processes_seen": len(self._last_discovery), | |
| "sessions_active": len(list_active_sessions(max_age_s=15.0)), | |
| "config": str(self.config.path) if self.config.path else None, | |
| } | |
| try: | |
| tmp.write_text(json.dumps(payload, indent=2), encoding="utf-8") | |
| os.replace(tmp, path) | |
| except OSError as exc: # pragma: no cover | |
| self.log.warning("state write failed: %s", exc) | |
| # -- main loop --------------------------------------------------------------- | |
| def run(self, max_seconds: Optional[float] = None) -> str: | |
| """Blocking loop; returns exit reason string (for tests).""" | |
| if not self.lifecycle.acquire_lock(): | |
| self.log.info("controller already running; this instance exits") | |
| return "already_running" | |
| self.lifecycle.clear_stop() # a leftover stop flag must not brick the next start | |
| self.lifecycle.install_signal_handlers() | |
| self.lifecycle.on_stop(self._cleanup) | |
| self._state = "hardware_discovery" | |
| self.log.info( | |
| "controller starting: os=%s py=%s gpus=%d driver=%s", | |
| os.name, f"{sys.version_info.major}.{sys.version_info.minor}", | |
| self.gpu_monitor.device_count, self.gpu_monitor.driver_version(), | |
| ) | |
| if not self.gpu_monitor.available: | |
| self.log.warning( | |
| "NVML unavailable (no driver or nvidia-ml-py missing): " | |
| "GPU metrics will be reported as unavailable - never fabricated" | |
| ) | |
| deadline = (time.time() + max_seconds) if max_seconds else None | |
| try: | |
| self._state = "idle" | |
| while not self.lifecycle.stop_requested(): | |
| if deadline and time.time() > deadline: | |
| return "timeout" | |
| try: | |
| self._tick() | |
| except Exception: # noqa: BLE001 - one bad tick never kills the runtime | |
| self.log.exception("tick error (continuing)") | |
| time.sleep(self._sleep_seconds()) | |
| return "stopped" | |
| finally: | |
| self._cleanup() | |
| self.lifecycle.release_lock() | |
| def _sleep_seconds(self) -> float: | |
| if self._state == "active": | |
| return self.config.get("monitoring.active_poll_seconds", 1.0) | |
| return self.config.get("monitoring.idle_poll_seconds", 5.0) | |
| def _tick(self) -> None: | |
| cfg_enabled = self.config.get("integration.process_discovery", True) | |
| procs = self.discovery.scan() if cfg_enabled else [] | |
| self._last_discovery = [p.summary() for p in procs] | |
| sessions = list_active_sessions(max_age_s=15.0) | |
| gpu_busy = any(p.gpu_vram_mb for p in procs) | |
| new_state = "active" if (sessions or gpu_busy) else "idle" | |
| if new_state != self._state: | |
| self.log.info("state %s -> %s (sessions=%d gpu_busy=%s)", | |
| self._state, new_state, len(sessions), gpu_busy) | |
| self._state = new_state | |
| if self._state == "active" and self.gpu_monitor.available: | |
| for s in self.gpu_monitor.sample_all(): | |
| self._record({"kind": "gpu", **s.as_dict()}) | |
| self.write_state() | |
| def _record(self, record: dict) -> None: | |
| if self._session_metrics is None: | |
| from ..monitoring.logger import JsonRecorder | |
| path = self.home / "logs" / "controller_metrics.jsonl" | |
| self._session_metrics = JsonRecorder(path) | |
| self._session_metrics.write(record) | |
| def _cleanup(self) -> None: | |
| try: | |
| if self._session_metrics is not None: | |
| self._session_metrics.close() | |
| self._session_metrics = None | |
| except Exception: # noqa: BLE001 | |
| pass | |
| try: | |
| self.gpu_monitor.close() | |
| except Exception: # noqa: BLE001 | |
| pass | |
| self._state = "shutdown" | |
| try: | |
| self.write_state() | |
| except Exception: # noqa: BLE001 | |
| pass | |
| self.log.info("controller cleanup complete") | |
| def main(argv: Optional[List[str]] = None) -> int: | |
| """Entry point: python -m lpu start (or lpu.runtime.controller) [--max-seconds N].""" | |
| argv = list(sys.argv[1:] if argv is None else argv) | |
| max_s = None | |
| if "--max-seconds" in argv: | |
| i = argv.index("--max-seconds") | |
| try: | |
| max_s = float(argv[i + 1]) | |
| except (IndexError, ValueError): | |
| print("--max-seconds needs a numeric value", file=sys.stderr) | |
| return 2 | |
| ctrl = BackgroundController() | |
| reason = ctrl.run(max_seconds=max_s) | |
| print(f"[lpu] controller exited: {reason}") | |
| return 0 | |
| if __name__ == "__main__": | |
| raise SystemExit(main()) | |