"""Adaptive Optimization Engine (mission §2.E, §7, §8). Lifecycle of every change: PROPOSED -> (validated? / user-confirmed?) -> APPLY-ROUTE -> APPLIED |-> QUEUED_AT_BOUNDARY |-> NEEDS_SDK |-> NEEDS_RESTART any applied/queued change -> ROLLBACK (restores the old value) Safety rules (mission §7 levels + §21 rules 9/10/13/15): - level 3 (LR / optimizer / loss / scheduler / EMA / precision / sampling / augmentation / validation protocol) is NEVER auto-applied; every single application requires the user's explicit confirmation, and we still only route it through the SDK boundary or a restart instruction. - level 2 requires either a passed controlled validation (measured A/B via the SDK) or a recorded user override ("user_override_no_validation"). - level 1 may auto-apply only when config optimizer.auto_apply_level1=true AND an SDK channel is attached; otherwise it is a manual proposal. - restart-required parameters are NEVER claimed as "applied": the ledger records status needs_restart plus the exact instruction. - expected impact is always labeled "Estimate Only"; confidence is "Measured" only when the proposal was derived from measured evidence. Every state transition is journaled (in-memory ledger + SQLite param_changes via the storage callback) so the whole decision history is auditable. """ from __future__ import annotations import itertools import time import uuid from dataclasses import dataclass, field from typing import Callable, Dict, List, Optional from .parameters import PARAM_SPECS, ParamSpec, ChangeClass, get_spec APPLY_STATUSES = ( "applied", # SDK acked the change (hot-reload or boundary) "queued_at_boundary", # SDK accepted, will ack at next batch/update/epoch edge "needs_sdk", # no SDK channel: manual instruction recorded "needs_restart", # restart-required class: never applied live "failed", # SDK refused or timed out (nothing changed) ) @dataclass class Validation: method: str # "sdk_ab_window" | "user_override" | "none" before_metric: Optional[float] = None after_metric: Optional[float] = None metric_name: Optional[str] = None verdict: str = "not_run" # passed | failed | not_run | user_override detail: str = "" def as_dict(self) -> dict: return { "method": self.method, "metric_name": self.metric_name, "before": self.before_metric, "after": self.after_metric, "verdict": self.verdict, "detail": self.detail, } @dataclass class Proposal: id: str param: str current: object proposed: object level: int change_class: str reason: str evidence: Dict[str, object] confidence: str # Measured | Estimated ts: float = field(default_factory=time.time) status: str = "proposed" # proposed | approved | rejected | # applied | queued_at_boundary | # needs_sdk | needs_restart | failed | # rolled_back validation: Validation = field(default_factory=lambda: Validation(method="none")) user_confirmed: bool = False expected_impact: str = "Estimate Only" def as_dict(self) -> dict: return { "id": self.id, "param": self.param, "current": self.current, "proposed": self.proposed, "level": self.level, "change_class": self.change_class, "reason": self.reason, "evidence": self.evidence, "confidence": self.confidence, "ts": self.ts, "status": self.status, "validation": self.validation.as_dict(), "user_confirmed": self.user_confirmed, "expected_impact": self.expected_impact, } @dataclass class ChangeRecord: id: str proposal_id: str param: str old_value: object new_value: object level: int reason: str status: str # same apply statuses + rolled_back applied_ts: Optional[float] = None result: str = "" rollback_ts: Optional[float] = None rollback_result: str = "" def as_dict(self) -> dict: return { "id": self.id, "proposal_id": self.proposal_id, "param": self.param, "old_value": self.old_value, "new_value": self.new_value, "level": self.level, "reason": self.reason, "status": self.status, "applied_ts": self.applied_ts, "result": self.result, "rollback_ts": self.rollback_ts, "rollback_result": self.rollback_result, } # callback the service injects: (param, value, boundary: bool, extra: dict) # -> {"status": one of APPLY_STATUSES, "detail": str} CommandSender = Callable[[str, object, bool, dict], Dict[str, object]] # callback to persist change records: (ChangeRecord.as_dict()) -> None LedgerWriter = Callable[[dict], None] class OptimizerEngine: def __init__(self, config: Optional[dict] = None, command_sender: Optional[CommandSender] = None, ledger_writer: Optional[LedgerWriter] = None): cfg = (config or {}).get("optimizer", {}) self.enabled = bool(cfg.get("enabled", True)) self.auto_apply_level1 = bool(cfg.get("auto_apply_level1", False)) self.cooldown_s = float(cfg.get("cooldown_s", 900)) self.max_proposals = int(cfg.get("max_proposals", 8)) self._sender = command_sender self._ledger_writer = ledger_writer self.proposals: Dict[str, Proposal] = {} self.ledger: List[ChangeRecord] = [] self._ids = itertools.count(1) self._last_proposed: Dict[str, float] = {} # ------------------------------------------------------------------ propose def _new_id(self, prefix: str) -> str: return f"{prefix}-{next(self._ids)}-{uuid.uuid4().hex[:6]}" def propose(self, param: str, current, proposed, reason: str, evidence: Dict[str, object], confidence: str = "Measured") -> Optional[Proposal]: if not self.enabled: return None try: spec: ParamSpec = get_spec(param) except KeyError: return None # unregistered parameters are never proposed now = time.time() if now - self._last_proposed.get(param, 0.0) < self.cooldown_s: return None # dedup: same param + same target already on the board for p in self.proposals.values(): if p.param == param and p.proposed == proposed and p.status in ( "proposed", "approved", "queued_at_boundary", "needs_sdk", "needs_restart"): return None if len([p for p in self.proposals.values() if p.status == "proposed"]) >= self.max_proposals: return None self._last_proposed[param] = now prop = Proposal( id=self._new_id("prop"), param=param, current=current, proposed=proposed, level=int(spec.level), change_class=spec.change_class.value, reason=reason, evidence=evidence, confidence=confidence, ) self.proposals[prop.id] = prop return prop def evaluate(self, facts: Dict[str, object], bottleneck: Optional[dict]) -> List[Proposal]: """Derive proposals from measured facts + the current bottleneck report. Pure decision logic; the sender applies nothing here.""" made: List[Proposal] = [] if not self.enabled: return made cat = (bottleneck or {}).get("category", "") dwr = facts.get("data_wait_ratio_mean") workers = facts.get("num_workers") prefetch = facts.get("prefetch_factor") pin = facts.get("pin_memory") persistent = facts.get("persistent_workers") gpu_idle_frac = facts.get("gpu_idle_fraction") vram_used_pct = facts.get("vram_used_pct_mean") oom_events = facts.get("oom_events") micro = facts.get("micro_batch_size") accum = facts.get("gradient_accumulation") if "DataLoader Bound" in cat and dwr is not None and dwr >= 0.5: if workers is not None and workers < 4: made.append(self.propose( "dataloader.num_workers", workers, max(2, min(8, workers * 2 if workers else 4)), "Data-wait ratio {:.2f} >= 0.5: GPUs are starving while " "batches are being prepared".format(round(dwr, 3)), {"data_wait_ratio": round(dwr, 3), "gpu_idle_fraction": gpu_idle_frac, "current_workers": workers}, confidence="Measured") or None) if prefetch is not None and prefetch < 2: made.append(self.propose( "dataloader.prefetch_factor", prefetch, 2, "Prefetch depth 1 leaves no overlap between batch " "preparation and compute", {"current_prefetch_factor": prefetch, "data_wait_ratio": round(dwr or 0.0, 3)}, confidence="Measured") or None) if pin is False: made.append(self.propose( "dataloader.pin_memory", False, True, "Unpinned memory forces synchronous H2D copies", {"current_pin_memory": False, "note": "level 1 data-path change; no math effect"}, confidence="Measured") or None) if persistent is False and (workers or 0) > 0: made.append(self.propose( "dataloader.persistent_workers", False, True, "Windows spawn re-imports the dataset module every epoch; " "persistent workers pay that cost once", {"current_persistent_workers": False, "workers": workers}, confidence="Measured") or None) if "VRAM Capacity Bound" in cat and vram_used_pct is not None: if micro is not None and micro > 1 and (oom_events or 0) > 0: made.append(self.propose( "training.micro_batch_size", micro, micro // 2, "VRAM used {:.1f}% with OOM events; halve micro-batch and " "double accumulation to keep effective batch constant " "(requires validation)".format(round(vram_used_pct, 1)), {"vram_used_pct": round(vram_used_pct, 1), "oom_events": oom_events, "effective_batch_preserved": True}, confidence="Measured") or None) return [p for p in made if p] # ------------------------------------------------------------------ approve def approve(self, proposal_id: str) -> Optional[Proposal]: p = self.proposals.get(proposal_id) if p and p.status == "proposed": p.status = "approved" p.user_confirmed = True return p def reject(self, proposal_id: str) -> Optional[Proposal]: p = self.proposals.get(proposal_id) if p and p.status in ("proposed", "approved"): p.status = "rejected" return p def record_validation(self, proposal_id: str, validation: Validation) -> Optional[Proposal]: p = self.proposals.get(proposal_id) if p: p.validation = validation return p # -------------------------------------------------------------------- apply def apply(self, proposal_id: str, user_confirmed: bool = False, override_validation: bool = False) -> dict: p = self.proposals.get(proposal_id) if p is None: return {"ok": False, "error": "unknown proposal"} if p.status not in ("proposed", "approved", "needs_sdk"): return {"ok": False, "error": f"proposal status is {p.status!r}; " "only proposed/approved can be applied"} spec = PARAM_SPECS[p.param] # --- gate 1: level 3 always requires this explicit confirmation --- if spec.level == 3 and not (user_confirmed or p.user_confirmed): return {"ok": False, "error": "level-3 parameter: explicit user " "confirmation required for EVERY apply"} # --- gate 2: level 2 requires a passed validation or recorded override if spec.validation_required: verdict = p.validation.verdict if verdict != "passed": if override_validation and (user_confirmed or p.user_confirmed): p.validation = Validation( method="user_override", verdict="user_override", detail="user explicitly applied without validation") else: return {"ok": False, "error": "level-2 parameter needs a " "passed validation (or an " "explicit user override)"} # --- gate 3: restart-required parameters are never applied live ---- if spec.change_class == ChangeClass.RESTART_REQUIRED or spec.applied_via == "restart": p.status = "needs_restart" rec = self._ledger(p, "needs_restart", result=spec.restart_instruction or "restart required") return {"ok": True, "status": "needs_restart", "instruction": spec.restart_instruction, "change_id": rec.id, "note": "LPU never applies restart-required parameters to a " "live process; follow the instruction and restart"} # --- gate 4: level-1 auto-apply only when configured + SDK present --- auto = (spec.level == 1 and self.auto_apply_level1 and not (user_confirmed or p.user_confirmed)) if self._sender is None: p.status = "needs_sdk" rec = self._ledger(p, "needs_sdk", result="no command channel: apply manually in the " "project (change is journaled here)") return {"ok": True, "status": "needs_sdk", "change_id": rec.id, "note": "no Integration SDK attached - change recorded as " "a manual instruction, NOT applied"} boundary = spec.change_class == ChangeClass.SAFE_BOUNDARY extra = {"proposal_id": p.id, "level": int(spec.level), "reason": p.reason} result = self._sender(p.param, p.proposed, boundary, extra) status = result.get("status", "failed") if status not in APPLY_STATUSES: status = "failed" p.status = status if status == "applied": p.user_confirmed = True rec = self._ledger(p, status, result=str(result.get("detail", ""))) return {"ok": status in ("applied", "queued_at_boundary", "needs_sdk", "needs_restart"), "status": status, "detail": result.get("detail", ""), "change_id": rec.id, "auto_applied": auto} # ----------------------------------------------------------------- rollback def mark_boundary_applied(self, param: str, value) -> Optional[ChangeRecord]: """A safe-boundary change the SDK earlier queued has now crossed its boundary (epoch/update edge): flip the ledger record to applied.""" for rec in reversed(self.ledger): if (rec.param == param and rec.new_value == value and rec.status == "queued_at_boundary"): rec.status = "applied" rec.applied_ts = time.time() rec.result = (rec.result + " | boundary crossed, SDK acked applied").strip(" |") self._write(rec) prop = self.proposals.get(rec.proposal_id) if prop: prop.status = "applied" return rec return None def rollback(self, change_id: str) -> dict: rec = next((c for c in self.ledger if c.id == change_id), None) if rec is None: return {"ok": False, "error": "unknown change id"} if rec.status == "rolled_back": return {"ok": False, "error": "change already rolled back"} if rec.status not in ("applied", "queued_at_boundary"): return {"ok": False, "error": f"status {rec.status!r} cannot be " "rolled back (nothing was applied)"} if self._sender is None: rec.status = "rolled_back" rec.rollback_ts = time.time() rec.rollback_result = "manual rollback required (no SDK channel)" self._write(rec) return {"ok": True, "status": "rolled_back", "note": "no SDK channel: verify the manual rollback"} spec = PARAM_SPECS[rec.param] boundary = spec.change_class == ChangeClass.SAFE_BOUNDARY result = self._sender(rec.param, rec.old_value, boundary, {"rollback_of": rec.id, "reason": "rollback"}) ok = result.get("status") in ("applied", "queued_at_boundary") rec.status = "rolled_back" rec.rollback_ts = time.time() rec.rollback_result = str(result.get("detail", result.get("status", ""))) self._write(rec) prop = self.proposals.get(rec.proposal_id) if prop: prop.status = "rolled_back" return {"ok": ok, "status": rec.status, "detail": rec.rollback_result} # ------------------------------------------------------------------- ledger def _ledger(self, p: Proposal, status: str, result: str = "") -> ChangeRecord: rec = ChangeRecord( id=self._new_id("chg"), proposal_id=p.id, param=p.param, old_value=p.current, new_value=p.proposed, level=p.level, reason=p.reason, status=status, applied_ts=time.time() if status in ("applied", "queued_at_boundary", "needs_restart", "needs_sdk") else None, result=result, ) self.ledger.append(rec) self._write(rec) return rec def _write(self, rec: ChangeRecord) -> None: if self._ledger_writer is not None: try: self._ledger_writer(rec.as_dict()) except Exception: # noqa: BLE001 - ledger write must not break control pass # ------------------------------------------------------------------ reports def proposals_view(self) -> List[dict]: return [p.as_dict() for p in sorted(self.proposals.values(), key=lambda x: -x.ts)] def ledger_view(self, limit: int = 50) -> List[dict]: return [c.as_dict() for c in self.ledger[-limit:]]