Virtual-LPU / lpu /accelerator /parameters.py
iparaco's picture
Upload 118 files
e59bd9d verified
Raw History Blame Contribute Delete
7.81 kB
"""Parameter governance registry (mission §7 optimization levels + §8 classes).
Every tunable parameter LPU may propose a change for is registered here with:
level 1 = data-path/runtime only, no effect on model math
2 = numerics-preserving, must be validated before auto-apply
3 = affects training semantics: NEVER auto-applied, explicit
user confirmation required every single time
change_class hot_reload - changeable mid-run via the Integration SDK
safe_boundary - changeable only at a batch/update/epoch
boundary (trainer acks the boundary apply)
restart_required - cannot change live; LPU only records the
decision + exact restart instruction
applied_via "sdk" - live command through the session channel
"restart"- LPU never applies it; the user restarts
"lpu" - LPU-internal setting (monitoring rates etc.)
The registry is the single source of truth for the optimizer engine AND the
dashboard parameter table. Unknown parameters are refused everywhere.
"""
from __future__ import annotations
from dataclasses import dataclass
from enum import Enum
from typing import Dict, Optional
class ParamLevel(int, Enum):
LEVEL_1_DATA_PATH = 1
LEVEL_2_NUMERICS_PRESERVING = 2
LEVEL_3_TRAINING_SEMANTICS = 3
class ChangeClass(str, Enum):
HOT_RELOAD = "hot_reload"
SAFE_BOUNDARY = "safe_boundary"
RESTART_REQUIRED = "restart_required"
@dataclass(frozen=True)
class ParamSpec:
name: str
level: ParamLevel
change_class: ChangeClass
applied_via: str # "sdk" | "restart" | "lpu"
description: str
user_approval_required: bool # level 3 -> always True
validation_required: bool # level 2 -> True unless user overrides
restart_instruction: Optional[str] = None
def _spec(name, level, cls, via, desc, restart_instruction=None):
return ParamSpec(
name=name,
level=ParamLevel(level),
change_class=ChangeClass(cls),
applied_via=via,
description=desc,
user_approval_required=(level == 3),
validation_required=(level == 2),
restart_instruction=restart_instruction,
)
PARAM_SPECS: Dict[str, ParamSpec] = {
# ---- Level 1: data-path / runtime (no effect on model math) ----------
"dataloader.num_workers": _spec(
"dataloader.num_workers", 1, "safe_boundary", "sdk",
"DataLoader worker processes (Windows spawn: workers re-import the "
"dataset module; persistent_workers avoids paying that per epoch)."),
"dataloader.prefetch_factor": _spec(
"dataloader.prefetch_factor", 1, "safe_boundary", "sdk",
"Batches prefetched per worker. Modest values keep worker RAM bounded."),
"dataloader.persistent_workers": _spec(
"dataloader.persistent_workers", 1, "safe_boundary", "sdk",
"Keep workers alive across epochs (Windows spawn cost is paid once)."),
"dataloader.pin_memory": _spec(
"dataloader.pin_memory", 1, "safe_boundary", "sdk",
"Pinned host memory enables async H2D copies."),
"runtime.monitoring_rate_s": _spec(
"runtime.monitoring_rate_s", 1, "hot_reload", "sdk",
"SDK heartbeat/metrics interval in seconds (LPU-side sampling only)."),
# ---- Level 2: numerics-preserving (validated before apply) ----------
"training.micro_batch_size": _spec(
"training.micro_batch_size", 2, "safe_boundary", "sdk",
"Per-step micro-batch. Effective batch must stay constant via "
"accumulation: micro * accum * world == effective."),
"training.gradient_accumulation": _spec(
"training.gradient_accumulation", 2, "safe_boundary", "sdk",
"Accumulation steps; changed together with micro-batch to keep the "
"effective batch size EXACTLY constant."),
"gpu.strategy": _spec(
"gpu.strategy", 2, "restart_required", "restart",
"single | data_parallel | ddp. Strategy cannot be swapped live; "
"requires a controlled restart (and a measured probe comparison).",
restart_instruction=(
"Stop training gracefully, set gpu.strategy in the project config, "
"then relaunch. LPU never swaps the strategy of a live process.")),
"memory.vram_headroom_mb": _spec(
"memory.vram_headroom_mb", 2, "hot_reload", "lpu",
"VRAM headroom LPU keeps free when advising batch sizes (LPU-side)."),
# ---- Level 3: training semantics (explicit user confirmation) -------
"training.learning_rate": _spec(
"training.learning_rate", 3, "safe_boundary", "sdk",
"Learning rate. Affects convergence; only applied on an update "
"boundary with the user's explicit confirmation."),
"training.ema_decay": _spec(
"training.ema_decay", 3, "safe_boundary", "sdk",
"EMA decay. Affects the validation/EMA model path."),
"training.precision": _spec(
"training.precision", 3, "restart_required", "restart",
"fp32 | fp16. Changes numerics on Pascal (no Tensor Cores); needs the "
"measured probe to pass before any restart applies it.",
restart_instruction=(
"Run `lpu bench` to validate fp16 numerics and speed on the rig, "
"then set precision in the project config and restart training.")),
"training.optimizer": _spec(
"training.optimizer", 3, "restart_required", "restart",
"Optimizer type/weights. Cannot change live.",
restart_instruction="Change in project config and restart training."),
"training.loss_weights": _spec(
"training.loss_weights", 3, "restart_required", "restart",
"Loss composition. Cannot change live.",
restart_instruction="Change in project config and restart training."),
"training.scheduler": _spec(
"training.scheduler", 3, "restart_required", "restart",
"LR schedule shape. Cannot change live.",
restart_instruction="Change in project config and restart training."),
"training.sampling_policy": _spec(
"training.sampling_policy", 3, "restart_required", "restart",
"Dataset sampling order policy. Changing it mid-run alters the data "
"stream seen by the model.",
restart_instruction="Change in project config and restart training."),
"training.data_augmentation": _spec(
"training.data_augmentation", 3, "restart_required", "restart",
"Augmentation pipeline changes the training distribution.",
restart_instruction="Change in project config and restart training."),
}
def get_spec(name: str) -> ParamSpec:
try:
return PARAM_SPECS[name]
except KeyError:
raise KeyError(
f"parameter {name!r} is not registered in the LPU V7 governance "
f"registry; known parameters: {sorted(PARAM_SPECS)}"
) from None
def classify_for_dashboard() -> list:
"""Parameter table for the dashboard: one row per governed parameter."""
out = []
for name, spec in sorted(PARAM_SPECS.items()):
out.append({
"name": name,
"level": int(spec.level),
"level_label": {
1: "L1 data-path (no math change)",
2: "L2 numerics-preserving (validated)",
3: "L3 training semantics (user-confirm)",
}[int(spec.level)],
"change_class": spec.change_class.value,
"applied_via": spec.applied_via,
"user_approval_required": spec.user_approval_required,
"validation_required": spec.validation_required,
"description": spec.description,
})
return out