Spaces:
Running
Running
Download lpu/accelerator/optimizer.py from iparaco/Virtual-LPU: direct link, hf CLI and curl.
- Browser
- Download file 19.3 kB
-
https://huggingface.co/spaces/iparaco/Virtual-LPU/resolve/main/lpu/accelerator/optimizer.py
- Command line
-
hf download hf://spaces/iparaco/Virtual-LPU/lpu/accelerator/optimizer.py
-
curl -L -o optimizer.py https://huggingface.co/spaces/iparaco/Virtual-LPU/resolve/main/lpu/accelerator/optimizer.py
19.3 kB
| """Adaptive Optimization Engine (mission §2.E, §7, §8). | |
| Lifecycle of every change: | |
| PROPOSED -> (validated? / user-confirmed?) -> APPLY-ROUTE -> APPLIED | |
| |-> QUEUED_AT_BOUNDARY | |
| |-> NEEDS_SDK | |
| |-> NEEDS_RESTART | |
| any applied/queued change -> ROLLBACK (restores the old value) | |
| Safety rules (mission §7 levels + §21 rules 9/10/13/15): | |
| - level 3 (LR / optimizer / loss / scheduler / EMA / precision / sampling / | |
| augmentation / validation protocol) is NEVER auto-applied; every single | |
| application requires the user's explicit confirmation, and we still only | |
| route it through the SDK boundary or a restart instruction. | |
| - level 2 requires either a passed controlled validation (measured A/B via | |
| the SDK) or a recorded user override ("user_override_no_validation"). | |
| - level 1 may auto-apply only when config optimizer.auto_apply_level1=true | |
| AND an SDK channel is attached; otherwise it is a manual proposal. | |
| - restart-required parameters are NEVER claimed as "applied": the ledger | |
| records status needs_restart plus the exact instruction. | |
| - expected impact is always labeled "Estimate Only"; confidence is | |
| "Measured" only when the proposal was derived from measured evidence. | |
| Every state transition is journaled (in-memory ledger + SQLite param_changes | |
| via the storage callback) so the whole decision history is auditable. | |
| """ | |
| from __future__ import annotations | |
| import itertools | |
| import time | |
| import uuid | |
| from dataclasses import dataclass, field | |
| from typing import Callable, Dict, List, Optional | |
| from .parameters import PARAM_SPECS, ParamSpec, ChangeClass, get_spec | |
| APPLY_STATUSES = ( | |
| "applied", # SDK acked the change (hot-reload or boundary) | |
| "queued_at_boundary", # SDK accepted, will ack at next batch/update/epoch edge | |
| "needs_sdk", # no SDK channel: manual instruction recorded | |
| "needs_restart", # restart-required class: never applied live | |
| "failed", # SDK refused or timed out (nothing changed) | |
| ) | |
| class Validation: | |
| method: str # "sdk_ab_window" | "user_override" | "none" | |
| before_metric: Optional[float] = None | |
| after_metric: Optional[float] = None | |
| metric_name: Optional[str] = None | |
| verdict: str = "not_run" # passed | failed | not_run | user_override | |
| detail: str = "" | |
| def as_dict(self) -> dict: | |
| return { | |
| "method": self.method, | |
| "metric_name": self.metric_name, | |
| "before": self.before_metric, | |
| "after": self.after_metric, | |
| "verdict": self.verdict, | |
| "detail": self.detail, | |
| } | |
| class Proposal: | |
| id: str | |
| param: str | |
| current: object | |
| proposed: object | |
| level: int | |
| change_class: str | |
| reason: str | |
| evidence: Dict[str, object] | |
| confidence: str # Measured | Estimated | |
| ts: float = field(default_factory=time.time) | |
| status: str = "proposed" # proposed | approved | rejected | | |
| # applied | queued_at_boundary | | |
| # needs_sdk | needs_restart | failed | | |
| # rolled_back | |
| validation: Validation = field(default_factory=lambda: Validation(method="none")) | |
| user_confirmed: bool = False | |
| expected_impact: str = "Estimate Only" | |
| def as_dict(self) -> dict: | |
| return { | |
| "id": self.id, "param": self.param, | |
| "current": self.current, "proposed": self.proposed, | |
| "level": self.level, "change_class": self.change_class, | |
| "reason": self.reason, "evidence": self.evidence, | |
| "confidence": self.confidence, "ts": self.ts, | |
| "status": self.status, "validation": self.validation.as_dict(), | |
| "user_confirmed": self.user_confirmed, | |
| "expected_impact": self.expected_impact, | |
| } | |
| class ChangeRecord: | |
| id: str | |
| proposal_id: str | |
| param: str | |
| old_value: object | |
| new_value: object | |
| level: int | |
| reason: str | |
| status: str # same apply statuses + rolled_back | |
| applied_ts: Optional[float] = None | |
| result: str = "" | |
| rollback_ts: Optional[float] = None | |
| rollback_result: str = "" | |
| def as_dict(self) -> dict: | |
| return { | |
| "id": self.id, "proposal_id": self.proposal_id, | |
| "param": self.param, "old_value": self.old_value, | |
| "new_value": self.new_value, "level": self.level, | |
| "reason": self.reason, "status": self.status, | |
| "applied_ts": self.applied_ts, "result": self.result, | |
| "rollback_ts": self.rollback_ts, | |
| "rollback_result": self.rollback_result, | |
| } | |
| # callback the service injects: (param, value, boundary: bool, extra: dict) | |
| # -> {"status": one of APPLY_STATUSES, "detail": str} | |
| CommandSender = Callable[[str, object, bool, dict], Dict[str, object]] | |
| # callback to persist change records: (ChangeRecord.as_dict()) -> None | |
| LedgerWriter = Callable[[dict], None] | |
| class OptimizerEngine: | |
| def __init__(self, config: Optional[dict] = None, | |
| command_sender: Optional[CommandSender] = None, | |
| ledger_writer: Optional[LedgerWriter] = None): | |
| cfg = (config or {}).get("optimizer", {}) | |
| self.enabled = bool(cfg.get("enabled", True)) | |
| self.auto_apply_level1 = bool(cfg.get("auto_apply_level1", False)) | |
| self.cooldown_s = float(cfg.get("cooldown_s", 900)) | |
| self.max_proposals = int(cfg.get("max_proposals", 8)) | |
| self._sender = command_sender | |
| self._ledger_writer = ledger_writer | |
| self.proposals: Dict[str, Proposal] = {} | |
| self.ledger: List[ChangeRecord] = [] | |
| self._ids = itertools.count(1) | |
| self._last_proposed: Dict[str, float] = {} | |
| # ------------------------------------------------------------------ propose | |
| def _new_id(self, prefix: str) -> str: | |
| return f"{prefix}-{next(self._ids)}-{uuid.uuid4().hex[:6]}" | |
| def propose(self, param: str, current, proposed, reason: str, | |
| evidence: Dict[str, object], confidence: str = "Measured") -> Optional[Proposal]: | |
| if not self.enabled: | |
| return None | |
| try: | |
| spec: ParamSpec = get_spec(param) | |
| except KeyError: | |
| return None # unregistered parameters are never proposed | |
| now = time.time() | |
| if now - self._last_proposed.get(param, 0.0) < self.cooldown_s: | |
| return None | |
| # dedup: same param + same target already on the board | |
| for p in self.proposals.values(): | |
| if p.param == param and p.proposed == proposed and p.status in ( | |
| "proposed", "approved", "queued_at_boundary", "needs_sdk", | |
| "needs_restart"): | |
| return None | |
| if len([p for p in self.proposals.values() | |
| if p.status == "proposed"]) >= self.max_proposals: | |
| return None | |
| self._last_proposed[param] = now | |
| prop = Proposal( | |
| id=self._new_id("prop"), param=param, current=current, | |
| proposed=proposed, level=int(spec.level), | |
| change_class=spec.change_class.value, reason=reason, | |
| evidence=evidence, confidence=confidence, | |
| ) | |
| self.proposals[prop.id] = prop | |
| return prop | |
| def evaluate(self, facts: Dict[str, object], | |
| bottleneck: Optional[dict]) -> List[Proposal]: | |
| """Derive proposals from measured facts + the current bottleneck | |
| report. Pure decision logic; the sender applies nothing here.""" | |
| made: List[Proposal] = [] | |
| if not self.enabled: | |
| return made | |
| cat = (bottleneck or {}).get("category", "") | |
| dwr = facts.get("data_wait_ratio_mean") | |
| workers = facts.get("num_workers") | |
| prefetch = facts.get("prefetch_factor") | |
| pin = facts.get("pin_memory") | |
| persistent = facts.get("persistent_workers") | |
| gpu_idle_frac = facts.get("gpu_idle_fraction") | |
| vram_used_pct = facts.get("vram_used_pct_mean") | |
| oom_events = facts.get("oom_events") | |
| micro = facts.get("micro_batch_size") | |
| accum = facts.get("gradient_accumulation") | |
| if "DataLoader Bound" in cat and dwr is not None and dwr >= 0.5: | |
| if workers is not None and workers < 4: | |
| made.append(self.propose( | |
| "dataloader.num_workers", workers, | |
| max(2, min(8, workers * 2 if workers else 4)), | |
| "Data-wait ratio {:.2f} >= 0.5: GPUs are starving while " | |
| "batches are being prepared".format(round(dwr, 3)), | |
| {"data_wait_ratio": round(dwr, 3), | |
| "gpu_idle_fraction": gpu_idle_frac, | |
| "current_workers": workers}, | |
| confidence="Measured") or None) | |
| if prefetch is not None and prefetch < 2: | |
| made.append(self.propose( | |
| "dataloader.prefetch_factor", prefetch, 2, | |
| "Prefetch depth 1 leaves no overlap between batch " | |
| "preparation and compute", | |
| {"current_prefetch_factor": prefetch, | |
| "data_wait_ratio": round(dwr or 0.0, 3)}, | |
| confidence="Measured") or None) | |
| if pin is False: | |
| made.append(self.propose( | |
| "dataloader.pin_memory", False, True, | |
| "Unpinned memory forces synchronous H2D copies", | |
| {"current_pin_memory": False, | |
| "note": "level 1 data-path change; no math effect"}, | |
| confidence="Measured") or None) | |
| if persistent is False and (workers or 0) > 0: | |
| made.append(self.propose( | |
| "dataloader.persistent_workers", False, True, | |
| "Windows spawn re-imports the dataset module every epoch; " | |
| "persistent workers pay that cost once", | |
| {"current_persistent_workers": False, "workers": workers}, | |
| confidence="Measured") or None) | |
| if "VRAM Capacity Bound" in cat and vram_used_pct is not None: | |
| if micro is not None and micro > 1 and (oom_events or 0) > 0: | |
| made.append(self.propose( | |
| "training.micro_batch_size", micro, micro // 2, | |
| "VRAM used {:.1f}% with OOM events; halve micro-batch and " | |
| "double accumulation to keep effective batch constant " | |
| "(requires validation)".format(round(vram_used_pct, 1)), | |
| {"vram_used_pct": round(vram_used_pct, 1), | |
| "oom_events": oom_events, | |
| "effective_batch_preserved": True}, | |
| confidence="Measured") or None) | |
| return [p for p in made if p] | |
| # ------------------------------------------------------------------ approve | |
| def approve(self, proposal_id: str) -> Optional[Proposal]: | |
| p = self.proposals.get(proposal_id) | |
| if p and p.status == "proposed": | |
| p.status = "approved" | |
| p.user_confirmed = True | |
| return p | |
| def reject(self, proposal_id: str) -> Optional[Proposal]: | |
| p = self.proposals.get(proposal_id) | |
| if p and p.status in ("proposed", "approved"): | |
| p.status = "rejected" | |
| return p | |
| def record_validation(self, proposal_id: str, validation: Validation) -> Optional[Proposal]: | |
| p = self.proposals.get(proposal_id) | |
| if p: | |
| p.validation = validation | |
| return p | |
| # -------------------------------------------------------------------- apply | |
| def apply(self, proposal_id: str, user_confirmed: bool = False, | |
| override_validation: bool = False) -> dict: | |
| p = self.proposals.get(proposal_id) | |
| if p is None: | |
| return {"ok": False, "error": "unknown proposal"} | |
| if p.status not in ("proposed", "approved", "needs_sdk"): | |
| return {"ok": False, "error": f"proposal status is {p.status!r}; " | |
| "only proposed/approved can be applied"} | |
| spec = PARAM_SPECS[p.param] | |
| # --- gate 1: level 3 always requires this explicit confirmation --- | |
| if spec.level == 3 and not (user_confirmed or p.user_confirmed): | |
| return {"ok": False, "error": "level-3 parameter: explicit user " | |
| "confirmation required for EVERY apply"} | |
| # --- gate 2: level 2 requires a passed validation or recorded override | |
| if spec.validation_required: | |
| verdict = p.validation.verdict | |
| if verdict != "passed": | |
| if override_validation and (user_confirmed or p.user_confirmed): | |
| p.validation = Validation( | |
| method="user_override", verdict="user_override", | |
| detail="user explicitly applied without validation") | |
| else: | |
| return {"ok": False, "error": "level-2 parameter needs a " | |
| "passed validation (or an " | |
| "explicit user override)"} | |
| # --- gate 3: restart-required parameters are never applied live ---- | |
| if spec.change_class == ChangeClass.RESTART_REQUIRED or spec.applied_via == "restart": | |
| p.status = "needs_restart" | |
| rec = self._ledger(p, "needs_restart", | |
| result=spec.restart_instruction or "restart required") | |
| return {"ok": True, "status": "needs_restart", | |
| "instruction": spec.restart_instruction, | |
| "change_id": rec.id, | |
| "note": "LPU never applies restart-required parameters to a " | |
| "live process; follow the instruction and restart"} | |
| # --- gate 4: level-1 auto-apply only when configured + SDK present --- | |
| auto = (spec.level == 1 and self.auto_apply_level1 | |
| and not (user_confirmed or p.user_confirmed)) | |
| if self._sender is None: | |
| p.status = "needs_sdk" | |
| rec = self._ledger(p, "needs_sdk", | |
| result="no command channel: apply manually in the " | |
| "project (change is journaled here)") | |
| return {"ok": True, "status": "needs_sdk", "change_id": rec.id, | |
| "note": "no Integration SDK attached - change recorded as " | |
| "a manual instruction, NOT applied"} | |
| boundary = spec.change_class == ChangeClass.SAFE_BOUNDARY | |
| extra = {"proposal_id": p.id, "level": int(spec.level), | |
| "reason": p.reason} | |
| result = self._sender(p.param, p.proposed, boundary, extra) | |
| status = result.get("status", "failed") | |
| if status not in APPLY_STATUSES: | |
| status = "failed" | |
| p.status = status | |
| if status == "applied": | |
| p.user_confirmed = True | |
| rec = self._ledger(p, status, result=str(result.get("detail", ""))) | |
| return {"ok": status in ("applied", "queued_at_boundary", "needs_sdk", | |
| "needs_restart"), | |
| "status": status, "detail": result.get("detail", ""), | |
| "change_id": rec.id, "auto_applied": auto} | |
| # ----------------------------------------------------------------- rollback | |
| def mark_boundary_applied(self, param: str, value) -> Optional[ChangeRecord]: | |
| """A safe-boundary change the SDK earlier queued has now crossed its | |
| boundary (epoch/update edge): flip the ledger record to applied.""" | |
| for rec in reversed(self.ledger): | |
| if (rec.param == param and rec.new_value == value | |
| and rec.status == "queued_at_boundary"): | |
| rec.status = "applied" | |
| rec.applied_ts = time.time() | |
| rec.result = (rec.result + " | boundary crossed, SDK acked applied").strip(" |") | |
| self._write(rec) | |
| prop = self.proposals.get(rec.proposal_id) | |
| if prop: | |
| prop.status = "applied" | |
| return rec | |
| return None | |
| def rollback(self, change_id: str) -> dict: | |
| rec = next((c for c in self.ledger if c.id == change_id), None) | |
| if rec is None: | |
| return {"ok": False, "error": "unknown change id"} | |
| if rec.status == "rolled_back": | |
| return {"ok": False, "error": "change already rolled back"} | |
| if rec.status not in ("applied", "queued_at_boundary"): | |
| return {"ok": False, "error": f"status {rec.status!r} cannot be " | |
| "rolled back (nothing was applied)"} | |
| if self._sender is None: | |
| rec.status = "rolled_back" | |
| rec.rollback_ts = time.time() | |
| rec.rollback_result = "manual rollback required (no SDK channel)" | |
| self._write(rec) | |
| return {"ok": True, "status": "rolled_back", | |
| "note": "no SDK channel: verify the manual rollback"} | |
| spec = PARAM_SPECS[rec.param] | |
| boundary = spec.change_class == ChangeClass.SAFE_BOUNDARY | |
| result = self._sender(rec.param, rec.old_value, boundary, | |
| {"rollback_of": rec.id, "reason": "rollback"}) | |
| ok = result.get("status") in ("applied", "queued_at_boundary") | |
| rec.status = "rolled_back" | |
| rec.rollback_ts = time.time() | |
| rec.rollback_result = str(result.get("detail", result.get("status", ""))) | |
| self._write(rec) | |
| prop = self.proposals.get(rec.proposal_id) | |
| if prop: | |
| prop.status = "rolled_back" | |
| return {"ok": ok, "status": rec.status, "detail": rec.rollback_result} | |
| # ------------------------------------------------------------------- ledger | |
| def _ledger(self, p: Proposal, status: str, result: str = "") -> ChangeRecord: | |
| rec = ChangeRecord( | |
| id=self._new_id("chg"), proposal_id=p.id, param=p.param, | |
| old_value=p.current, new_value=p.proposed, level=p.level, | |
| reason=p.reason, status=status, | |
| applied_ts=time.time() if status in ("applied", "queued_at_boundary", | |
| "needs_restart", "needs_sdk") else None, | |
| result=result, | |
| ) | |
| self.ledger.append(rec) | |
| self._write(rec) | |
| return rec | |
| def _write(self, rec: ChangeRecord) -> None: | |
| if self._ledger_writer is not None: | |
| try: | |
| self._ledger_writer(rec.as_dict()) | |
| except Exception: # noqa: BLE001 - ledger write must not break control | |
| pass | |
| # ------------------------------------------------------------------ reports | |
| def proposals_view(self) -> List[dict]: | |
| return [p.as_dict() for p in | |
| sorted(self.proposals.values(), key=lambda x: -x.ts)] | |
| def ledger_view(self, limit: int = 50) -> List[dict]: | |
| return [c.as_dict() for c in self.ledger[-limit:]] | |