"""LPU Core Engine - the single authoritative execution engine. Composes hardware abstraction, scheduler, memory manager, precision manager and execution policy into one decision point. Produces an ExecutionPlan that the training session applies. All sub-decisions carry measured/estimated labels so reports stay honest. """ from __future__ import annotations import time from dataclasses import dataclass, field from typing import Any, Dict, List, Optional from ..compat import require_torch from ..hardware.gpu_detector import detect_gpus, p2p_matrix from ..memory.memory_manager import MemoryManager from .execution_policy import Policy, get_policy from .scheduler import MultiGPUScheduler, SchedulerDecision from .workload_analyzer import WorkloadProfile, analyze_model @dataclass class ExecutionPlan: created_ts: float hardware: Dict[str, Any] workload: Dict[str, Any] scheduler: Dict[str, Any] memory: Dict[str, Any] precision: Dict[str, Any] environment: Dict[str, Any] notes: List[str] = field(default_factory=list) def summary(self) -> dict: return { "created_ts": self.created_ts, "hardware": self.hardware, "workload": self.workload, "scheduler": self.scheduler, "memory": self.memory, "precision": self.precision, "environment": self.environment, "notes": self.notes, } class LPUEngine: """Central decision engine. One instance per process (per session).""" def __init__(self, config, log_dir=None): from ..monitoring.logger import get_logger self.config = config self._log = get_logger("engine", log_dir=log_dir, console=False) self.policy: Policy = get_policy(config.get("performance.mode", "performance")) self.scheduler = MultiGPUScheduler(config, log_dir=log_dir) self.memory = MemoryManager(config, log_dir=log_dir) # -- analysis -------------------------------------------------------------- def analyze(self, model=None, dataloader=None, example_input=None, batch_size=None) -> ExecutionPlan: """Gather facts and decide. Safe to call without CUDA (degraded plan). V4: `batch_size` overrides the profile's batch size so the scheduler decides with the REAL requested batch (V3 always saw 1, which forced the tiny-batch single-GPU gate - audit finding 3.1/3.3). """ notes: List[str] = [] hardware = { "gpu_count": 0, "gpus": [], "p2p": {}, "torch": None, } torch = require_torch() hw_notes = [] if torch.cuda.is_available(): gpus = detect_gpus() hardware["gpu_count"] = len(gpus) hardware["gpus"] = [g.summary() for g in gpus] hardware["p2p"] = p2p_matrix(len(gpus)) if len(gpus) > 1 else {} self.memory.refresh() hardware["memory"] = { f"cuda:{i}": { "free_vram_mb": st.free_vram_mb, "total_vram_mb": st.total_vram_mb, } for i, st in self.memory.devices.items() } else: hw_notes.append("no CUDA available in this interpreter: plan is CPU-advisory only") workload: Dict[str, Any] = {"provided": False} sched: Dict[str, Any] = {"strategy": "none", "measured": False, "reason": "no model provided"} precision: Dict[str, Any] = {"mode": "fp32", "reason": "not analyzed (no model/CUDA)"} if model is not None: profile = analyze_model(model, example_input=example_input, dataloader=dataloader) if batch_size is not None: profile.batch_size = batch_size workload = {"provided": True, **profile.summary()} if torch.cuda.is_available(): try: dec: SchedulerDecision = self.scheduler.decide( profile, model, example_input, batch_size=profile.batch_size, ) sched = { "strategy": dec.strategy, "device_ids": dec.device_ids, "reason": dec.reason, "measured": dec.measured, "probe_results_s": dec.probe_results, "p2p_matrix": dec.p2p_matrix, } except Exception as exc: # noqa: BLE001 sched = {"strategy": "single", "measured": False, "reason": f"decision failed: {exc}"} notes.append(f"scheduler decision error: {exc}") # precision recommendation is produced by PrecisionManager at # session start (it needs benchmark evidence); here we only # record the policy inputs precision = { "mode": self.config.get("precision.mode", "auto"), "fp32_fallback": self.config.get("precision.fp32_fallback", True), "decision": "deferred to PrecisionManager (requires measured stability probe)", } env = { "cudnn_benchmark": self.config.get("training.cudnn_benchmark", True), "channels_last": self.config.get("training.channels_last", False), "prefetch": self.config.get("training.prefetch", True), "compile_backend": self.config.get("training.compile_backend", "off"), } mem = { "policy": self.config.get("memory.policy", "gpu_first"), "automatic_batch_size": self.config.get("memory.automatic_batch_size", True), "headroom_mb": self.config.get("memory.vram_headroom_mb", 1024.0), } notes.extend(hw_notes) return ExecutionPlan( created_ts=time.time(), hardware=hardware, workload=workload, scheduler=sched, memory=mem, precision=precision, environment=env, notes=notes, )