File size: 7,817 Bytes
e59bd9d
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
"""LPU Background Controller - the Windows-resident monitoring process.

Footprint by design: torch-free, ~15 MB RAM, near-zero CPU when idle.

State machine (spec section 11):

    START -> HARDWARE DISCOVERY -> IDLE/MONITORING
    IDLE  --(compatible process found)--> ACTIVE MONITORING
    ACTIVE --(process exits)--> IDLE + resource cleanup

Idle behavior: poll process list every idle_poll_seconds (default 5 s),
sample GPU state at most every gpu_sample_seconds (default 2 s) and ONLY
when idle_gpu_monitoring is enabled. No CUDA contexts are created here -
this process never initializes CUDA, so it can never interfere with a
training process's GPU state.

Active behavior: when a launched session exists (sessions dir) or a
discovered python process holds GPU memory, poll at active_poll_seconds and
append metric records for reporting.
"""

from __future__ import annotations

import json
import os
import sys
import time
from pathlib import Path
from typing import List, Optional

from ..config import load_config
from ..ipc import SessionChannel, list_active_sessions
from ..monitoring.logger import get_logger, lpu_home
from ..monitoring.gpu_monitor import GPUMonitor
from .lifecycle import Lifecycle, find_controller_pid
from .process_discovery import ProcessDiscovery

STATE_PATH = "status.json"


class BackgroundController:
    def __init__(self, config=None):
        self.config = config or load_config()
        self.home = lpu_home()
        self.log = get_logger(
            "controller", log_dir=self.home / "logs",
            level=self.config.get("monitoring.log_level", "INFO"),
        )
        self.lifecycle = Lifecycle("controller", log=self.log)
        self.gpu_monitor = GPUMonitor(
            sample_seconds=self.config.get("monitoring.gpu_sample_seconds", 2.0),
            enabled=self.config.get("monitoring.enabled", True)
            and self.config.get("monitoring.idle_gpu_monitoring", True),
            log_dir=self.home / "logs",
        )
        self.discovery = ProcessDiscovery(
            deep_scan=self.config.get("integration.deep_scan", False),
            log_dir=self.home / "logs",
            gpu_monitor=self.gpu_monitor,
        )
        self._state = "init"
        self._session_metrics = None  # JsonRecorder, opened on demand
        self._last_discovery: List[dict] = []
        self._started_ts = time.time()

    # -- state ------------------------------------------------------------------
    def write_state(self) -> None:
        path = self.home / STATE_PATH
        tmp = path.with_suffix(".tmp")
        payload = {
            "state": self._state,
            "pid": os.getpid(),
            "started_ts": self._started_ts,
            "ts": time.time(),
            "gpu_available": self.gpu_monitor.available,
            "gpu_count": self.gpu_monitor.device_count,
            "driver_version": self.gpu_monitor.driver_version(),
            "gpus": [s.as_dict() for s in self.gpu_monitor.sample_all(force=True)]
            if self.gpu_monitor.available else [],
            "python_processes_seen": len(self._last_discovery),
            "sessions_active": len(list_active_sessions(max_age_s=15.0)),
            "config": str(self.config.path) if self.config.path else None,
        }
        try:
            tmp.write_text(json.dumps(payload, indent=2), encoding="utf-8")
            os.replace(tmp, path)
        except OSError as exc:  # pragma: no cover
            self.log.warning("state write failed: %s", exc)

    # -- main loop ---------------------------------------------------------------
    def run(self, max_seconds: Optional[float] = None) -> str:
        """Blocking loop; returns exit reason string (for tests)."""
        if not self.lifecycle.acquire_lock():
            self.log.info("controller already running; this instance exits")
            return "already_running"
        self.lifecycle.clear_stop()  # a leftover stop flag must not brick the next start
        self.lifecycle.install_signal_handlers()
        self.lifecycle.on_stop(self._cleanup)
        self._state = "hardware_discovery"
        self.log.info(
            "controller starting: os=%s py=%s gpus=%d driver=%s",
            os.name, f"{sys.version_info.major}.{sys.version_info.minor}",
            self.gpu_monitor.device_count, self.gpu_monitor.driver_version(),
        )
        if not self.gpu_monitor.available:
            self.log.warning(
                "NVML unavailable (no driver or nvidia-ml-py missing): "
                "GPU metrics will be reported as unavailable - never fabricated"
            )
        deadline = (time.time() + max_seconds) if max_seconds else None
        try:
            self._state = "idle"
            while not self.lifecycle.stop_requested():
                if deadline and time.time() > deadline:
                    return "timeout"
                try:
                    self._tick()
                except Exception:  # noqa: BLE001 - one bad tick never kills the runtime
                    self.log.exception("tick error (continuing)")
                time.sleep(self._sleep_seconds())
            return "stopped"
        finally:
            self._cleanup()
            self.lifecycle.release_lock()

    def _sleep_seconds(self) -> float:
        if self._state == "active":
            return self.config.get("monitoring.active_poll_seconds", 1.0)
        return self.config.get("monitoring.idle_poll_seconds", 5.0)

    def _tick(self) -> None:
        cfg_enabled = self.config.get("integration.process_discovery", True)
        procs = self.discovery.scan() if cfg_enabled else []
        self._last_discovery = [p.summary() for p in procs]
        sessions = list_active_sessions(max_age_s=15.0)
        gpu_busy = any(p.gpu_vram_mb for p in procs)

        new_state = "active" if (sessions or gpu_busy) else "idle"
        if new_state != self._state:
            self.log.info("state %s -> %s (sessions=%d gpu_busy=%s)",
                          self._state, new_state, len(sessions), gpu_busy)
            self._state = new_state

        if self._state == "active" and self.gpu_monitor.available:
            for s in self.gpu_monitor.sample_all():
                self._record({"kind": "gpu", **s.as_dict()})
        self.write_state()

    def _record(self, record: dict) -> None:
        if self._session_metrics is None:
            from ..monitoring.logger import JsonRecorder

            path = self.home / "logs" / "controller_metrics.jsonl"
            self._session_metrics = JsonRecorder(path)
        self._session_metrics.write(record)

    def _cleanup(self) -> None:
        try:
            if self._session_metrics is not None:
                self._session_metrics.close()
                self._session_metrics = None
        except Exception:  # noqa: BLE001
            pass
        try:
            self.gpu_monitor.close()
        except Exception:  # noqa: BLE001
            pass
        self._state = "shutdown"
        try:
            self.write_state()
        except Exception:  # noqa: BLE001
            pass
        self.log.info("controller cleanup complete")


def main(argv: Optional[List[str]] = None) -> int:
    """Entry point: python -m lpu start (or lpu.runtime.controller) [--max-seconds N]."""
    argv = list(sys.argv[1:] if argv is None else argv)
    max_s = None
    if "--max-seconds" in argv:
        i = argv.index("--max-seconds")
        try:
            max_s = float(argv[i + 1])
        except (IndexError, ValueError):
            print("--max-seconds needs a numeric value", file=sys.stderr)
            return 2
    ctrl = BackgroundController()
    reason = ctrl.run(max_seconds=max_s)
    print(f"[lpu] controller exited: {reason}")
    return 0


if __name__ == "__main__":
    raise SystemExit(main())