mirror presets.py from geolip 0.7.0 @0c946b3 (hub_ckpt; default 0 = v1 verbatim)
Browse files- presets.py +10 -2
presets.py
CHANGED
|
@@ -54,6 +54,12 @@ class AlephLMConfig:
|
|
| 54 |
# lawful supply (K <= 2*hub_D per book; ROUND 5e). 1 = the v1 layout,
|
| 55 |
# bit-identical state dict. Old manifests load via the default.
|
| 56 |
hub_const: int = 1
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 57 |
|
| 58 |
def to_dict(self):
|
| 59 |
d = asdict(self)
|
|
@@ -157,7 +163,8 @@ PRESETS: dict[str, Preset] = {
|
|
| 157 |
hub_layers=tuple(range(32)),
|
| 158 |
hub_K=256, hub_D=256, hub_const=16,
|
| 159 |
bank_experts=6, bank_ff=1024,
|
| 160 |
-
head_K=256, head_D=256, hub_chunk=256
|
|
|
|
| 161 |
train=TrainConfig(micro_batch=4, grad_accum=16,
|
| 162 |
governor="minsep", governor_theta=45.0),
|
| 163 |
curriculum=_curriculum(500_000_000, 8_000_000_000, 16_000_000_000),
|
|
@@ -170,7 +177,8 @@ PRESETS: dict[str, Preset] = {
|
|
| 170 |
hub_layers=tuple(range(20)),
|
| 171 |
hub_K=64, hub_D=128, hub_const=4,
|
| 172 |
bank_experts=3, bank_ff=1024,
|
| 173 |
-
head_K=256, head_D=256, hub_chunk=256
|
|
|
|
| 174 |
train=TrainConfig(micro_batch=16, grad_accum=4,
|
| 175 |
governor="minsep", governor_theta=45.0),
|
| 176 |
curriculum=_curriculum(300_000_000, 5_000_000_000, 10_000_000_000),
|
|
|
|
| 54 |
# lawful supply (K <= 2*hub_D per book; ROUND 5e). 1 = the v1 layout,
|
| 55 |
# bit-identical state dict. Old manifests load via the default.
|
| 56 |
hub_const: int = 1
|
| 57 |
+
# Activation checkpointing (training only; inference/decode untouched).
|
| 58 |
+
# 0 = off (v1 verbatim). 1 = recompute the hub read in backward.
|
| 59 |
+
# 2 = also recompute the bank branch. At v2 scale (16 books x ctx 8192
|
| 60 |
+
# x 32 layers) the retained scan tensors alone exceed a 95GB card —
|
| 61 |
+
# measured OOM, Blackwell preflight 2026-08-26. ~2x hub recompute cost.
|
| 62 |
+
hub_ckpt: int = 0
|
| 63 |
|
| 64 |
def to_dict(self):
|
| 65 |
d = asdict(self)
|
|
|
|
| 163 |
hub_layers=tuple(range(32)),
|
| 164 |
hub_K=256, hub_D=256, hub_const=16,
|
| 165 |
bank_experts=6, bank_ff=1024,
|
| 166 |
+
head_K=256, head_D=256, hub_chunk=256,
|
| 167 |
+
hub_ckpt=2),
|
| 168 |
train=TrainConfig(micro_batch=4, grad_accum=16,
|
| 169 |
governor="minsep", governor_theta=45.0),
|
| 170 |
curriculum=_curriculum(500_000_000, 8_000_000_000, 16_000_000_000),
|
|
|
|
| 177 |
hub_layers=tuple(range(20)),
|
| 178 |
hub_K=64, hub_D=128, hub_const=4,
|
| 179 |
bank_experts=3, bank_ff=1024,
|
| 180 |
+
head_K=256, head_D=256, hub_chunk=256,
|
| 181 |
+
hub_ckpt=2),
|
| 182 |
train=TrainConfig(micro_batch=16, grad_accum=4,
|
| 183 |
governor="minsep", governor_theta=45.0),
|
| 184 |
curriculum=_curriculum(300_000_000, 5_000_000_000, 10_000_000_000),
|