File size: 19,319 Bytes
e59bd9d
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
"""Adaptive Optimization Engine (mission §2.E, §7, §8).

Lifecycle of every change:

    PROPOSED -> (validated? / user-confirmed?) -> APPLY-ROUTE -> APPLIED
                                                        |-> QUEUED_AT_BOUNDARY
                                                        |-> NEEDS_SDK
                                                        |-> NEEDS_RESTART
    any applied/queued change  ->  ROLLBACK (restores the old value)

Safety rules (mission §7 levels + §21 rules 9/10/13/15):
- level 3 (LR / optimizer / loss / scheduler / EMA / precision / sampling /
  augmentation / validation protocol) is NEVER auto-applied; every single
  application requires the user's explicit confirmation, and we still only
  route it through the SDK boundary or a restart instruction.
- level 2 requires either a passed controlled validation (measured A/B via
  the SDK) or a recorded user override ("user_override_no_validation").
- level 1 may auto-apply only when config optimizer.auto_apply_level1=true
  AND an SDK channel is attached; otherwise it is a manual proposal.
- restart-required parameters are NEVER claimed as "applied": the ledger
  records status needs_restart plus the exact instruction.
- expected impact is always labeled "Estimate Only"; confidence is
  "Measured" only when the proposal was derived from measured evidence.

Every state transition is journaled (in-memory ledger + SQLite param_changes
via the storage callback) so the whole decision history is auditable.
"""

from __future__ import annotations

import itertools
import time
import uuid
from dataclasses import dataclass, field
from typing import Callable, Dict, List, Optional

from .parameters import PARAM_SPECS, ParamSpec, ChangeClass, get_spec

APPLY_STATUSES = (
    "applied",            # SDK acked the change (hot-reload or boundary)
    "queued_at_boundary", # SDK accepted, will ack at next batch/update/epoch edge
    "needs_sdk",          # no SDK channel: manual instruction recorded
    "needs_restart",      # restart-required class: never applied live
    "failed",             # SDK refused or timed out (nothing changed)
)


@dataclass
class Validation:
    method: str                      # "sdk_ab_window" | "user_override" | "none"
    before_metric: Optional[float] = None
    after_metric: Optional[float] = None
    metric_name: Optional[str] = None
    verdict: str = "not_run"        # passed | failed | not_run | user_override
    detail: str = ""

    def as_dict(self) -> dict:
        return {
            "method": self.method,
            "metric_name": self.metric_name,
            "before": self.before_metric,
            "after": self.after_metric,
            "verdict": self.verdict,
            "detail": self.detail,
        }


@dataclass
class Proposal:
    id: str
    param: str
    current: object
    proposed: object
    level: int
    change_class: str
    reason: str
    evidence: Dict[str, object]
    confidence: str                       # Measured | Estimated
    ts: float = field(default_factory=time.time)
    status: str = "proposed"              # proposed | approved | rejected |
                                          # applied | queued_at_boundary |
                                          # needs_sdk | needs_restart | failed |
                                          # rolled_back
    validation: Validation = field(default_factory=lambda: Validation(method="none"))
    user_confirmed: bool = False
    expected_impact: str = "Estimate Only"

    def as_dict(self) -> dict:
        return {
            "id": self.id, "param": self.param,
            "current": self.current, "proposed": self.proposed,
            "level": self.level, "change_class": self.change_class,
            "reason": self.reason, "evidence": self.evidence,
            "confidence": self.confidence, "ts": self.ts,
            "status": self.status, "validation": self.validation.as_dict(),
            "user_confirmed": self.user_confirmed,
            "expected_impact": self.expected_impact,
        }


@dataclass
class ChangeRecord:
    id: str
    proposal_id: str
    param: str
    old_value: object
    new_value: object
    level: int
    reason: str
    status: str                       # same apply statuses + rolled_back
    applied_ts: Optional[float] = None
    result: str = ""
    rollback_ts: Optional[float] = None
    rollback_result: str = ""

    def as_dict(self) -> dict:
        return {
            "id": self.id, "proposal_id": self.proposal_id,
            "param": self.param, "old_value": self.old_value,
            "new_value": self.new_value, "level": self.level,
            "reason": self.reason, "status": self.status,
            "applied_ts": self.applied_ts, "result": self.result,
            "rollback_ts": self.rollback_ts,
            "rollback_result": self.rollback_result,
        }


# callback the service injects: (param, value, boundary: bool, extra: dict)
# -> {"status": one of APPLY_STATUSES, "detail": str}
CommandSender = Callable[[str, object, bool, dict], Dict[str, object]]
# callback to persist change records: (ChangeRecord.as_dict()) -> None
LedgerWriter = Callable[[dict], None]


class OptimizerEngine:
    def __init__(self, config: Optional[dict] = None,
                 command_sender: Optional[CommandSender] = None,
                 ledger_writer: Optional[LedgerWriter] = None):
        cfg = (config or {}).get("optimizer", {})
        self.enabled = bool(cfg.get("enabled", True))
        self.auto_apply_level1 = bool(cfg.get("auto_apply_level1", False))
        self.cooldown_s = float(cfg.get("cooldown_s", 900))
        self.max_proposals = int(cfg.get("max_proposals", 8))
        self._sender = command_sender
        self._ledger_writer = ledger_writer
        self.proposals: Dict[str, Proposal] = {}
        self.ledger: List[ChangeRecord] = []
        self._ids = itertools.count(1)
        self._last_proposed: Dict[str, float] = {}

    # ------------------------------------------------------------------ propose
    def _new_id(self, prefix: str) -> str:
        return f"{prefix}-{next(self._ids)}-{uuid.uuid4().hex[:6]}"

    def propose(self, param: str, current, proposed, reason: str,
                evidence: Dict[str, object], confidence: str = "Measured") -> Optional[Proposal]:
        if not self.enabled:
            return None
        try:
            spec: ParamSpec = get_spec(param)
        except KeyError:
            return None  # unregistered parameters are never proposed
        now = time.time()
        if now - self._last_proposed.get(param, 0.0) < self.cooldown_s:
            return None
        # dedup: same param + same target already on the board
        for p in self.proposals.values():
            if p.param == param and p.proposed == proposed and p.status in (
                    "proposed", "approved", "queued_at_boundary", "needs_sdk",
                    "needs_restart"):
                return None
        if len([p for p in self.proposals.values()
                if p.status == "proposed"]) >= self.max_proposals:
            return None
        self._last_proposed[param] = now
        prop = Proposal(
            id=self._new_id("prop"), param=param, current=current,
            proposed=proposed, level=int(spec.level),
            change_class=spec.change_class.value, reason=reason,
            evidence=evidence, confidence=confidence,
        )
        self.proposals[prop.id] = prop
        return prop

    def evaluate(self, facts: Dict[str, object],
                 bottleneck: Optional[dict]) -> List[Proposal]:
        """Derive proposals from measured facts + the current bottleneck
        report. Pure decision logic; the sender applies nothing here."""
        made: List[Proposal] = []
        if not self.enabled:
            return made
        cat = (bottleneck or {}).get("category", "")
        dwr = facts.get("data_wait_ratio_mean")
        workers = facts.get("num_workers")
        prefetch = facts.get("prefetch_factor")
        pin = facts.get("pin_memory")
        persistent = facts.get("persistent_workers")
        gpu_idle_frac = facts.get("gpu_idle_fraction")
        vram_used_pct = facts.get("vram_used_pct_mean")
        oom_events = facts.get("oom_events")
        micro = facts.get("micro_batch_size")
        accum = facts.get("gradient_accumulation")

        if "DataLoader Bound" in cat and dwr is not None and dwr >= 0.5:
            if workers is not None and workers < 4:
                made.append(self.propose(
                    "dataloader.num_workers", workers,
                    max(2, min(8, workers * 2 if workers else 4)),
                    "Data-wait ratio {:.2f} >= 0.5: GPUs are starving while "
                    "batches are being prepared".format(round(dwr, 3)),
                    {"data_wait_ratio": round(dwr, 3),
                     "gpu_idle_fraction": gpu_idle_frac,
                     "current_workers": workers},
                    confidence="Measured") or None)
            if prefetch is not None and prefetch < 2:
                made.append(self.propose(
                    "dataloader.prefetch_factor", prefetch, 2,
                    "Prefetch depth 1 leaves no overlap between batch "
                    "preparation and compute",
                    {"current_prefetch_factor": prefetch,
                     "data_wait_ratio": round(dwr or 0.0, 3)},
                    confidence="Measured") or None)
            if pin is False:
                made.append(self.propose(
                    "dataloader.pin_memory", False, True,
                    "Unpinned memory forces synchronous H2D copies",
                    {"current_pin_memory": False,
                     "note": "level 1 data-path change; no math effect"},
                    confidence="Measured") or None)
            if persistent is False and (workers or 0) > 0:
                made.append(self.propose(
                    "dataloader.persistent_workers", False, True,
                    "Windows spawn re-imports the dataset module every epoch; "
                    "persistent workers pay that cost once",
                    {"current_persistent_workers": False, "workers": workers},
                    confidence="Measured") or None)

        if "VRAM Capacity Bound" in cat and vram_used_pct is not None:
            if micro is not None and micro > 1 and (oom_events or 0) > 0:
                made.append(self.propose(
                    "training.micro_batch_size", micro, micro // 2,
                    "VRAM used {:.1f}% with OOM events; halve micro-batch and "
                    "double accumulation to keep effective batch constant "
                    "(requires validation)".format(round(vram_used_pct, 1)),
                    {"vram_used_pct": round(vram_used_pct, 1),
                     "oom_events": oom_events,
                     "effective_batch_preserved": True},
                    confidence="Measured") or None)

        return [p for p in made if p]

    # ------------------------------------------------------------------ approve
    def approve(self, proposal_id: str) -> Optional[Proposal]:
        p = self.proposals.get(proposal_id)
        if p and p.status == "proposed":
            p.status = "approved"
            p.user_confirmed = True
        return p

    def reject(self, proposal_id: str) -> Optional[Proposal]:
        p = self.proposals.get(proposal_id)
        if p and p.status in ("proposed", "approved"):
            p.status = "rejected"
        return p

    def record_validation(self, proposal_id: str, validation: Validation) -> Optional[Proposal]:
        p = self.proposals.get(proposal_id)
        if p:
            p.validation = validation
        return p

    # -------------------------------------------------------------------- apply
    def apply(self, proposal_id: str, user_confirmed: bool = False,
              override_validation: bool = False) -> dict:
        p = self.proposals.get(proposal_id)
        if p is None:
            return {"ok": False, "error": "unknown proposal"}
        if p.status not in ("proposed", "approved", "needs_sdk"):
            return {"ok": False, "error": f"proposal status is {p.status!r}; "
                                          "only proposed/approved can be applied"}
        spec = PARAM_SPECS[p.param]

        # --- gate 1: level 3 always requires this explicit confirmation ---
        if spec.level == 3 and not (user_confirmed or p.user_confirmed):
            return {"ok": False, "error": "level-3 parameter: explicit user "
                                          "confirmation required for EVERY apply"}

        # --- gate 2: level 2 requires a passed validation or recorded override
        if spec.validation_required:
            verdict = p.validation.verdict
            if verdict != "passed":
                if override_validation and (user_confirmed or p.user_confirmed):
                    p.validation = Validation(
                        method="user_override", verdict="user_override",
                        detail="user explicitly applied without validation")
                else:
                    return {"ok": False, "error": "level-2 parameter needs a "
                                                  "passed validation (or an "
                                                  "explicit user override)"}

        # --- gate 3: restart-required parameters are never applied live ----
        if spec.change_class == ChangeClass.RESTART_REQUIRED or spec.applied_via == "restart":
            p.status = "needs_restart"
            rec = self._ledger(p, "needs_restart",
                               result=spec.restart_instruction or "restart required")
            return {"ok": True, "status": "needs_restart",
                    "instruction": spec.restart_instruction,
                    "change_id": rec.id,
                    "note": "LPU never applies restart-required parameters to a "
                            "live process; follow the instruction and restart"}

        # --- gate 4: level-1 auto-apply only when configured + SDK present ---
        auto = (spec.level == 1 and self.auto_apply_level1
                and not (user_confirmed or p.user_confirmed))

        if self._sender is None:
            p.status = "needs_sdk"
            rec = self._ledger(p, "needs_sdk",
                               result="no command channel: apply manually in the "
                                      "project (change is journaled here)")
            return {"ok": True, "status": "needs_sdk", "change_id": rec.id,
                    "note": "no Integration SDK attached - change recorded as "
                            "a manual instruction, NOT applied"}

        boundary = spec.change_class == ChangeClass.SAFE_BOUNDARY
        extra = {"proposal_id": p.id, "level": int(spec.level),
                 "reason": p.reason}
        result = self._sender(p.param, p.proposed, boundary, extra)
        status = result.get("status", "failed")
        if status not in APPLY_STATUSES:
            status = "failed"
        p.status = status
        if status == "applied":
            p.user_confirmed = True
        rec = self._ledger(p, status, result=str(result.get("detail", "")))
        return {"ok": status in ("applied", "queued_at_boundary", "needs_sdk",
                                 "needs_restart"),
                "status": status, "detail": result.get("detail", ""),
                "change_id": rec.id, "auto_applied": auto}

    # ----------------------------------------------------------------- rollback
    def mark_boundary_applied(self, param: str, value) -> Optional[ChangeRecord]:
        """A safe-boundary change the SDK earlier queued has now crossed its
        boundary (epoch/update edge): flip the ledger record to applied."""
        for rec in reversed(self.ledger):
            if (rec.param == param and rec.new_value == value
                    and rec.status == "queued_at_boundary"):
                rec.status = "applied"
                rec.applied_ts = time.time()
                rec.result = (rec.result + " | boundary crossed, SDK acked applied").strip(" |")
                self._write(rec)
                prop = self.proposals.get(rec.proposal_id)
                if prop:
                    prop.status = "applied"
                return rec
        return None

    def rollback(self, change_id: str) -> dict:
        rec = next((c for c in self.ledger if c.id == change_id), None)
        if rec is None:
            return {"ok": False, "error": "unknown change id"}
        if rec.status == "rolled_back":
            return {"ok": False, "error": "change already rolled back"}
        if rec.status not in ("applied", "queued_at_boundary"):
            return {"ok": False, "error": f"status {rec.status!r} cannot be "
                                          "rolled back (nothing was applied)"}
        if self._sender is None:
            rec.status = "rolled_back"
            rec.rollback_ts = time.time()
            rec.rollback_result = "manual rollback required (no SDK channel)"
            self._write(rec)
            return {"ok": True, "status": "rolled_back",
                    "note": "no SDK channel: verify the manual rollback"}
        spec = PARAM_SPECS[rec.param]
        boundary = spec.change_class == ChangeClass.SAFE_BOUNDARY
        result = self._sender(rec.param, rec.old_value, boundary,
                              {"rollback_of": rec.id, "reason": "rollback"})
        ok = result.get("status") in ("applied", "queued_at_boundary")
        rec.status = "rolled_back"
        rec.rollback_ts = time.time()
        rec.rollback_result = str(result.get("detail", result.get("status", "")))
        self._write(rec)
        prop = self.proposals.get(rec.proposal_id)
        if prop:
            prop.status = "rolled_back"
        return {"ok": ok, "status": rec.status, "detail": rec.rollback_result}

    # ------------------------------------------------------------------- ledger
    def _ledger(self, p: Proposal, status: str, result: str = "") -> ChangeRecord:
        rec = ChangeRecord(
            id=self._new_id("chg"), proposal_id=p.id, param=p.param,
            old_value=p.current, new_value=p.proposed, level=p.level,
            reason=p.reason, status=status,
            applied_ts=time.time() if status in ("applied", "queued_at_boundary",
                                                 "needs_restart", "needs_sdk") else None,
            result=result,
        )
        self.ledger.append(rec)
        self._write(rec)
        return rec

    def _write(self, rec: ChangeRecord) -> None:
        if self._ledger_writer is not None:
            try:
                self._ledger_writer(rec.as_dict())
            except Exception:  # noqa: BLE001 - ledger write must not break control
                pass

    # ------------------------------------------------------------------ reports
    def proposals_view(self) -> List[dict]:
        return [p.as_dict() for p in
                sorted(self.proposals.values(), key=lambda x: -x.ts)]

    def ledger_view(self, limit: int = 50) -> List[dict]:
        return [c.as_dict() for c in self.ledger[-limit:]]