Virtual-LPU / lpu /accelerator /optimizer.py
iparaco's picture
Upload 118 files
e59bd9d verified
Raw History Blame Contribute Delete
19.3 kB
"""Adaptive Optimization Engine (mission §2.E, §7, §8).
Lifecycle of every change:
PROPOSED -> (validated? / user-confirmed?) -> APPLY-ROUTE -> APPLIED
|-> QUEUED_AT_BOUNDARY
|-> NEEDS_SDK
|-> NEEDS_RESTART
any applied/queued change -> ROLLBACK (restores the old value)
Safety rules (mission §7 levels + §21 rules 9/10/13/15):
- level 3 (LR / optimizer / loss / scheduler / EMA / precision / sampling /
augmentation / validation protocol) is NEVER auto-applied; every single
application requires the user's explicit confirmation, and we still only
route it through the SDK boundary or a restart instruction.
- level 2 requires either a passed controlled validation (measured A/B via
the SDK) or a recorded user override ("user_override_no_validation").
- level 1 may auto-apply only when config optimizer.auto_apply_level1=true
AND an SDK channel is attached; otherwise it is a manual proposal.
- restart-required parameters are NEVER claimed as "applied": the ledger
records status needs_restart plus the exact instruction.
- expected impact is always labeled "Estimate Only"; confidence is
"Measured" only when the proposal was derived from measured evidence.
Every state transition is journaled (in-memory ledger + SQLite param_changes
via the storage callback) so the whole decision history is auditable.
"""
from __future__ import annotations
import itertools
import time
import uuid
from dataclasses import dataclass, field
from typing import Callable, Dict, List, Optional
from .parameters import PARAM_SPECS, ParamSpec, ChangeClass, get_spec
APPLY_STATUSES = (
"applied", # SDK acked the change (hot-reload or boundary)
"queued_at_boundary", # SDK accepted, will ack at next batch/update/epoch edge
"needs_sdk", # no SDK channel: manual instruction recorded
"needs_restart", # restart-required class: never applied live
"failed", # SDK refused or timed out (nothing changed)
)
@dataclass
class Validation:
method: str # "sdk_ab_window" | "user_override" | "none"
before_metric: Optional[float] = None
after_metric: Optional[float] = None
metric_name: Optional[str] = None
verdict: str = "not_run" # passed | failed | not_run | user_override
detail: str = ""
def as_dict(self) -> dict:
return {
"method": self.method,
"metric_name": self.metric_name,
"before": self.before_metric,
"after": self.after_metric,
"verdict": self.verdict,
"detail": self.detail,
}
@dataclass
class Proposal:
id: str
param: str
current: object
proposed: object
level: int
change_class: str
reason: str
evidence: Dict[str, object]
confidence: str # Measured | Estimated
ts: float = field(default_factory=time.time)
status: str = "proposed" # proposed | approved | rejected |
# applied | queued_at_boundary |
# needs_sdk | needs_restart | failed |
# rolled_back
validation: Validation = field(default_factory=lambda: Validation(method="none"))
user_confirmed: bool = False
expected_impact: str = "Estimate Only"
def as_dict(self) -> dict:
return {
"id": self.id, "param": self.param,
"current": self.current, "proposed": self.proposed,
"level": self.level, "change_class": self.change_class,
"reason": self.reason, "evidence": self.evidence,
"confidence": self.confidence, "ts": self.ts,
"status": self.status, "validation": self.validation.as_dict(),
"user_confirmed": self.user_confirmed,
"expected_impact": self.expected_impact,
}
@dataclass
class ChangeRecord:
id: str
proposal_id: str
param: str
old_value: object
new_value: object
level: int
reason: str
status: str # same apply statuses + rolled_back
applied_ts: Optional[float] = None
result: str = ""
rollback_ts: Optional[float] = None
rollback_result: str = ""
def as_dict(self) -> dict:
return {
"id": self.id, "proposal_id": self.proposal_id,
"param": self.param, "old_value": self.old_value,
"new_value": self.new_value, "level": self.level,
"reason": self.reason, "status": self.status,
"applied_ts": self.applied_ts, "result": self.result,
"rollback_ts": self.rollback_ts,
"rollback_result": self.rollback_result,
}
# callback the service injects: (param, value, boundary: bool, extra: dict)
# -> {"status": one of APPLY_STATUSES, "detail": str}
CommandSender = Callable[[str, object, bool, dict], Dict[str, object]]
# callback to persist change records: (ChangeRecord.as_dict()) -> None
LedgerWriter = Callable[[dict], None]
class OptimizerEngine:
def __init__(self, config: Optional[dict] = None,
command_sender: Optional[CommandSender] = None,
ledger_writer: Optional[LedgerWriter] = None):
cfg = (config or {}).get("optimizer", {})
self.enabled = bool(cfg.get("enabled", True))
self.auto_apply_level1 = bool(cfg.get("auto_apply_level1", False))
self.cooldown_s = float(cfg.get("cooldown_s", 900))
self.max_proposals = int(cfg.get("max_proposals", 8))
self._sender = command_sender
self._ledger_writer = ledger_writer
self.proposals: Dict[str, Proposal] = {}
self.ledger: List[ChangeRecord] = []
self._ids = itertools.count(1)
self._last_proposed: Dict[str, float] = {}
# ------------------------------------------------------------------ propose
def _new_id(self, prefix: str) -> str:
return f"{prefix}-{next(self._ids)}-{uuid.uuid4().hex[:6]}"
def propose(self, param: str, current, proposed, reason: str,
evidence: Dict[str, object], confidence: str = "Measured") -> Optional[Proposal]:
if not self.enabled:
return None
try:
spec: ParamSpec = get_spec(param)
except KeyError:
return None # unregistered parameters are never proposed
now = time.time()
if now - self._last_proposed.get(param, 0.0) < self.cooldown_s:
return None
# dedup: same param + same target already on the board
for p in self.proposals.values():
if p.param == param and p.proposed == proposed and p.status in (
"proposed", "approved", "queued_at_boundary", "needs_sdk",
"needs_restart"):
return None
if len([p for p in self.proposals.values()
if p.status == "proposed"]) >= self.max_proposals:
return None
self._last_proposed[param] = now
prop = Proposal(
id=self._new_id("prop"), param=param, current=current,
proposed=proposed, level=int(spec.level),
change_class=spec.change_class.value, reason=reason,
evidence=evidence, confidence=confidence,
)
self.proposals[prop.id] = prop
return prop
def evaluate(self, facts: Dict[str, object],
bottleneck: Optional[dict]) -> List[Proposal]:
"""Derive proposals from measured facts + the current bottleneck
report. Pure decision logic; the sender applies nothing here."""
made: List[Proposal] = []
if not self.enabled:
return made
cat = (bottleneck or {}).get("category", "")
dwr = facts.get("data_wait_ratio_mean")
workers = facts.get("num_workers")
prefetch = facts.get("prefetch_factor")
pin = facts.get("pin_memory")
persistent = facts.get("persistent_workers")
gpu_idle_frac = facts.get("gpu_idle_fraction")
vram_used_pct = facts.get("vram_used_pct_mean")
oom_events = facts.get("oom_events")
micro = facts.get("micro_batch_size")
accum = facts.get("gradient_accumulation")
if "DataLoader Bound" in cat and dwr is not None and dwr >= 0.5:
if workers is not None and workers < 4:
made.append(self.propose(
"dataloader.num_workers", workers,
max(2, min(8, workers * 2 if workers else 4)),
"Data-wait ratio {:.2f} >= 0.5: GPUs are starving while "
"batches are being prepared".format(round(dwr, 3)),
{"data_wait_ratio": round(dwr, 3),
"gpu_idle_fraction": gpu_idle_frac,
"current_workers": workers},
confidence="Measured") or None)
if prefetch is not None and prefetch < 2:
made.append(self.propose(
"dataloader.prefetch_factor", prefetch, 2,
"Prefetch depth 1 leaves no overlap between batch "
"preparation and compute",
{"current_prefetch_factor": prefetch,
"data_wait_ratio": round(dwr or 0.0, 3)},
confidence="Measured") or None)
if pin is False:
made.append(self.propose(
"dataloader.pin_memory", False, True,
"Unpinned memory forces synchronous H2D copies",
{"current_pin_memory": False,
"note": "level 1 data-path change; no math effect"},
confidence="Measured") or None)
if persistent is False and (workers or 0) > 0:
made.append(self.propose(
"dataloader.persistent_workers", False, True,
"Windows spawn re-imports the dataset module every epoch; "
"persistent workers pay that cost once",
{"current_persistent_workers": False, "workers": workers},
confidence="Measured") or None)
if "VRAM Capacity Bound" in cat and vram_used_pct is not None:
if micro is not None and micro > 1 and (oom_events or 0) > 0:
made.append(self.propose(
"training.micro_batch_size", micro, micro // 2,
"VRAM used {:.1f}% with OOM events; halve micro-batch and "
"double accumulation to keep effective batch constant "
"(requires validation)".format(round(vram_used_pct, 1)),
{"vram_used_pct": round(vram_used_pct, 1),
"oom_events": oom_events,
"effective_batch_preserved": True},
confidence="Measured") or None)
return [p for p in made if p]
# ------------------------------------------------------------------ approve
def approve(self, proposal_id: str) -> Optional[Proposal]:
p = self.proposals.get(proposal_id)
if p and p.status == "proposed":
p.status = "approved"
p.user_confirmed = True
return p
def reject(self, proposal_id: str) -> Optional[Proposal]:
p = self.proposals.get(proposal_id)
if p and p.status in ("proposed", "approved"):
p.status = "rejected"
return p
def record_validation(self, proposal_id: str, validation: Validation) -> Optional[Proposal]:
p = self.proposals.get(proposal_id)
if p:
p.validation = validation
return p
# -------------------------------------------------------------------- apply
def apply(self, proposal_id: str, user_confirmed: bool = False,
override_validation: bool = False) -> dict:
p = self.proposals.get(proposal_id)
if p is None:
return {"ok": False, "error": "unknown proposal"}
if p.status not in ("proposed", "approved", "needs_sdk"):
return {"ok": False, "error": f"proposal status is {p.status!r}; "
"only proposed/approved can be applied"}
spec = PARAM_SPECS[p.param]
# --- gate 1: level 3 always requires this explicit confirmation ---
if spec.level == 3 and not (user_confirmed or p.user_confirmed):
return {"ok": False, "error": "level-3 parameter: explicit user "
"confirmation required for EVERY apply"}
# --- gate 2: level 2 requires a passed validation or recorded override
if spec.validation_required:
verdict = p.validation.verdict
if verdict != "passed":
if override_validation and (user_confirmed or p.user_confirmed):
p.validation = Validation(
method="user_override", verdict="user_override",
detail="user explicitly applied without validation")
else:
return {"ok": False, "error": "level-2 parameter needs a "
"passed validation (or an "
"explicit user override)"}
# --- gate 3: restart-required parameters are never applied live ----
if spec.change_class == ChangeClass.RESTART_REQUIRED or spec.applied_via == "restart":
p.status = "needs_restart"
rec = self._ledger(p, "needs_restart",
result=spec.restart_instruction or "restart required")
return {"ok": True, "status": "needs_restart",
"instruction": spec.restart_instruction,
"change_id": rec.id,
"note": "LPU never applies restart-required parameters to a "
"live process; follow the instruction and restart"}
# --- gate 4: level-1 auto-apply only when configured + SDK present ---
auto = (spec.level == 1 and self.auto_apply_level1
and not (user_confirmed or p.user_confirmed))
if self._sender is None:
p.status = "needs_sdk"
rec = self._ledger(p, "needs_sdk",
result="no command channel: apply manually in the "
"project (change is journaled here)")
return {"ok": True, "status": "needs_sdk", "change_id": rec.id,
"note": "no Integration SDK attached - change recorded as "
"a manual instruction, NOT applied"}
boundary = spec.change_class == ChangeClass.SAFE_BOUNDARY
extra = {"proposal_id": p.id, "level": int(spec.level),
"reason": p.reason}
result = self._sender(p.param, p.proposed, boundary, extra)
status = result.get("status", "failed")
if status not in APPLY_STATUSES:
status = "failed"
p.status = status
if status == "applied":
p.user_confirmed = True
rec = self._ledger(p, status, result=str(result.get("detail", "")))
return {"ok": status in ("applied", "queued_at_boundary", "needs_sdk",
"needs_restart"),
"status": status, "detail": result.get("detail", ""),
"change_id": rec.id, "auto_applied": auto}
# ----------------------------------------------------------------- rollback
def mark_boundary_applied(self, param: str, value) -> Optional[ChangeRecord]:
"""A safe-boundary change the SDK earlier queued has now crossed its
boundary (epoch/update edge): flip the ledger record to applied."""
for rec in reversed(self.ledger):
if (rec.param == param and rec.new_value == value
and rec.status == "queued_at_boundary"):
rec.status = "applied"
rec.applied_ts = time.time()
rec.result = (rec.result + " | boundary crossed, SDK acked applied").strip(" |")
self._write(rec)
prop = self.proposals.get(rec.proposal_id)
if prop:
prop.status = "applied"
return rec
return None
def rollback(self, change_id: str) -> dict:
rec = next((c for c in self.ledger if c.id == change_id), None)
if rec is None:
return {"ok": False, "error": "unknown change id"}
if rec.status == "rolled_back":
return {"ok": False, "error": "change already rolled back"}
if rec.status not in ("applied", "queued_at_boundary"):
return {"ok": False, "error": f"status {rec.status!r} cannot be "
"rolled back (nothing was applied)"}
if self._sender is None:
rec.status = "rolled_back"
rec.rollback_ts = time.time()
rec.rollback_result = "manual rollback required (no SDK channel)"
self._write(rec)
return {"ok": True, "status": "rolled_back",
"note": "no SDK channel: verify the manual rollback"}
spec = PARAM_SPECS[rec.param]
boundary = spec.change_class == ChangeClass.SAFE_BOUNDARY
result = self._sender(rec.param, rec.old_value, boundary,
{"rollback_of": rec.id, "reason": "rollback"})
ok = result.get("status") in ("applied", "queued_at_boundary")
rec.status = "rolled_back"
rec.rollback_ts = time.time()
rec.rollback_result = str(result.get("detail", result.get("status", "")))
self._write(rec)
prop = self.proposals.get(rec.proposal_id)
if prop:
prop.status = "rolled_back"
return {"ok": ok, "status": rec.status, "detail": rec.rollback_result}
# ------------------------------------------------------------------- ledger
def _ledger(self, p: Proposal, status: str, result: str = "") -> ChangeRecord:
rec = ChangeRecord(
id=self._new_id("chg"), proposal_id=p.id, param=p.param,
old_value=p.current, new_value=p.proposed, level=p.level,
reason=p.reason, status=status,
applied_ts=time.time() if status in ("applied", "queued_at_boundary",
"needs_restart", "needs_sdk") else None,
result=result,
)
self.ledger.append(rec)
self._write(rec)
return rec
def _write(self, rec: ChangeRecord) -> None:
if self._ledger_writer is not None:
try:
self._ledger_writer(rec.as_dict())
except Exception: # noqa: BLE001 - ledger write must not break control
pass
# ------------------------------------------------------------------ reports
def proposals_view(self) -> List[dict]:
return [p.as_dict() for p in
sorted(self.proposals.values(), key=lambda x: -x.ts)]
def ledger_view(self, limit: int = 50) -> List[dict]:
return [c.as_dict() for c in self.ledger[-limit:]]