vendored code: comment-only cleanup (alephllm 0.10.8); AST-identical, no behaviour change
Browse files- attention.py +1 -1
- governor.py +2 -2
- head.py +1 -1
- presets.py +9 -9
attention.py
CHANGED
|
@@ -66,7 +66,7 @@ class _Constellation(nn.Module):
|
|
| 66 |
|
| 67 |
The multi-constellation hub is the PRODUCT-CODE form (B2: independent
|
| 68 |
frames compose, .859 -> .955 monotone in members) at lawful supply
|
| 69 |
-
(
|
| 70 |
32 dims ran 16x and crowded into 333-646 duplicate pairs)."""
|
| 71 |
|
| 72 |
def __init__(self, d: int, K: int, D: int, tau: float):
|
|
|
|
| 66 |
|
| 67 |
The multi-constellation hub is the PRODUCT-CODE form (B2: independent
|
| 68 |
frames compose, .859 -> .955 monotone in members) at lawful supply
|
| 69 |
+
(aleph-splat-0 TECHNICAL_ROUND5.md, round 5e: K <= 2*D per address space β v1's single 512-anchor book in
|
| 70 |
32 dims ran 16x and crowded into 333-646 duplicate pairs)."""
|
| 71 |
|
| 72 |
def __init__(self, d: int, K: int, D: int, tau: float):
|
governor.py
CHANGED
|
@@ -1,6 +1,6 @@
|
|
| 1 |
-
"""The anchor governor β min-separation projection (
|
| 2 |
|
| 3 |
-
Measured
|
| 4 |
crowded past the separation floor flutter under the positional sweep (the
|
| 5 |
static logit gap is quadratic in separation, the oscillation linear), and the
|
| 6 |
crowding tail tracks seed fragility. The governor holds the space open FROM
|
|
|
|
| 1 |
+
"""The anchor governor β min-separation projection (2026-08-25).
|
| 2 |
|
| 3 |
+
Measured (aleph-splat-0 TECHNICAL_ROUND5.md, rounds 5e/5f): anchors
|
| 4 |
crowded past the separation floor flutter under the positional sweep (the
|
| 5 |
static logit gap is quadratic in separation, the oscillation linear), and the
|
| 6 |
crowding tail tracks seed fragility. The governor holds the space open FROM
|
head.py
CHANGED
|
@@ -1,4 +1,4 @@
|
|
| 1 |
-
"""DualHead β the standard readout plus
|
| 2 |
|
| 3 |
logits = W_h h + W_s s(h), W_s == 0 at init
|
| 4 |
|
|
|
|
| 1 |
+
"""DualHead β the standard readout plus an aleph read, born null.
|
| 2 |
|
| 3 |
logits = W_h h + W_s s(h), W_s == 0 at init
|
| 4 |
|
presets.py
CHANGED
|
@@ -51,7 +51,7 @@ class AlephLMConfig:
|
|
| 51 |
tie_embeddings: bool = False # BPE crafts tie; byte crafts cannot (trigram)
|
| 52 |
hub_chunk: int = 128 # chunked-scan block for the hub prefix memories
|
| 53 |
# v2 (2026-08-26): multi-constellation hubs β the product-code form at
|
| 54 |
-
# lawful supply (K <= 2*hub_D per book;
|
| 55 |
# bit-identical state dict. Old manifests load via the default.
|
| 56 |
hub_const: int = 1
|
| 57 |
# Activation checkpointing (training only; inference/decode untouched).
|
|
@@ -105,14 +105,14 @@ class TrainConfig:
|
|
| 105 |
# v3: a lead-ruled change of the mix (the minted lexicon) β a resume
|
| 106 |
# whose recipe differs ONLY by it is accepted and recorded with this note
|
| 107 |
data_plane_amendment: str | None = None
|
| 108 |
-
# The anchor governor (
|
| 109 |
# min-separation projection over hub/head codebooks β preventive
|
| 110 |
# anti-crowding, identity when slack, zero parameters, outside the
|
| 111 |
# task gradient (the no-balance-machinery law is untouched).
|
| 112 |
governor: str = "" # "" off (v1 verbatim) | "minsep"
|
| 113 |
governor_theta: float = 45.0 # deg; scale ~ gamma*(D): 45 at D=256
|
| 114 |
governor_every: int = 8 # steps between slack checks (~free)
|
| 115 |
-
# Post-revival address freeze (0.8.2;
|
| 116 |
# BOUNDARY-WRITE head revival, proj + head codebook freeze so the
|
| 117 |
# self-burial channel (proj rotating to codebook-orthogonality,
|
| 118 |
# measured 2/2 crafts) is structurally closed β only W_s trains.
|
|
@@ -132,7 +132,7 @@ class TrainConfig:
|
|
| 132 |
|
| 133 |
|
| 134 |
# All missions upload to the one training repo, each under its own prefix
|
| 135 |
-
# (
|
| 136 |
TRAINING_REPO = "AbstractPhil/alephllm-mini-beatrix-training"
|
| 137 |
|
| 138 |
|
|
@@ -187,12 +187,12 @@ PRESETS: dict[str, Preset] = {
|
|
| 187 |
train=TrainConfig(micro_batch=48, grad_accum=3),
|
| 188 |
curriculum=_curriculum(300_000_000, 3_000_000_000, 6_000_000_000),
|
| 189 |
),
|
| 190 |
-
# v2 (2026-08-26
|
| 191 |
# a hub in every block, multi-constellation product code at lawful
|
| 192 |
# supply (16 books x 256 anchors in 256-dim spaces = 1.0x supply;
|
| 193 |
# v1's single book ran 16x and crowded), governed from birth, ctx 8192
|
| 194 |
# where the O(L) read is ~4.5x cheaper than the MHA equivalent.
|
| 195 |
-
# ~873.7M params.
|
| 196 |
"mini-beatrix-2": Preset(
|
| 197 |
model=AlephLMConfig(name="mini-beatrix-2", d_model=1024, n_layers=32,
|
| 198 |
n_heads=16, context=8192,
|
|
@@ -210,8 +210,8 @@ PRESETS: dict[str, Preset] = {
|
|
| 210 |
governor="minsep", governor_theta=45.0),
|
| 211 |
curriculum=_curriculum(500_000_000, 8_000_000_000, 16_000_000_000),
|
| 212 |
),
|
| 213 |
-
# THE
|
| 214 |
-
#
|
| 215 |
# full-splat craft one rung above v1 β d1024 L20 ctx4096, governed
|
| 216 |
# 4x64@128 books (4x supply headroom vs v1's crowded 16x). Also the
|
| 217 |
# screen bed for every v2-era gating cell. hub_ckpt=0: at 237M the
|
|
@@ -244,7 +244,7 @@ def make_v3_preset(n_layers: int = 24, d_model: int = 1024,
|
|
| 244 |
data_scale: float = 4.0, epoch_cap: float | None = None,
|
| 245 |
rebalance_to: str | None = None,
|
| 246 |
name: str | None = None) -> Preset:
|
| 247 |
-
"""The v3 craft (plan of
|
| 248 |
the solidified all-splat form at d1024 β a governed hub in EVERY
|
| 249 |
block, the certified hub geometry (4 books x 64 @ D128), banks
|
| 250 |
3 x ff1024, head 256@256, ctx 4096 β at a depth the throughput bench
|
|
|
|
| 51 |
tie_embeddings: bool = False # BPE crafts tie; byte crafts cannot (trigram)
|
| 52 |
hub_chunk: int = 128 # chunked-scan block for the hub prefix memories
|
| 53 |
# v2 (2026-08-26): multi-constellation hubs β the product-code form at
|
| 54 |
+
# lawful supply (K <= 2*hub_D per book; aleph-splat-0 TECHNICAL_ROUND5.md, round 5e). 1 = the v1 layout,
|
| 55 |
# bit-identical state dict. Old manifests load via the default.
|
| 56 |
hub_const: int = 1
|
| 57 |
# Activation checkpointing (training only; inference/decode untouched).
|
|
|
|
| 105 |
# v3: a lead-ruled change of the mix (the minted lexicon) β a resume
|
| 106 |
# whose recipe differs ONLY by it is accepted and recorded with this note
|
| 107 |
data_plane_amendment: str | None = None
|
| 108 |
+
# The anchor governor (2026-08-25; see model/governor.py): post-optimizer-step
|
| 109 |
# min-separation projection over hub/head codebooks β preventive
|
| 110 |
# anti-crowding, identity when slack, zero parameters, outside the
|
| 111 |
# task gradient (the no-balance-machinery law is untouched).
|
| 112 |
governor: str = "" # "" off (v1 verbatim) | "minsep"
|
| 113 |
governor_theta: float = 45.0 # deg; scale ~ gamma*(D): 45 at D=256
|
| 114 |
governor_every: int = 8 # steps between slack checks (~free)
|
| 115 |
+
# Post-revival address freeze (0.8.2; see train/revival.py): after the
|
| 116 |
# BOUNDARY-WRITE head revival, proj + head codebook freeze so the
|
| 117 |
# self-burial channel (proj rotating to codebook-orthogonality,
|
| 118 |
# measured 2/2 crafts) is structurally closed β only W_s trains.
|
|
|
|
| 132 |
|
| 133 |
|
| 134 |
# All missions upload to the one training repo, each under its own prefix
|
| 135 |
+
# (checkpoints + manifests + tensorboard for every craft).
|
| 136 |
TRAINING_REPO = "AbstractPhil/alephllm-mini-beatrix-training"
|
| 137 |
|
| 138 |
|
|
|
|
| 187 |
train=TrainConfig(micro_batch=48, grad_accum=3),
|
| 188 |
curriculum=_curriculum(300_000_000, 3_000_000_000, 6_000_000_000),
|
| 189 |
),
|
| 190 |
+
# v2 (2026-08-26): FULL-SPLAT β
|
| 191 |
# a hub in every block, multi-constellation product code at lawful
|
| 192 |
# supply (16 books x 256 anchors in 256-dim spaces = 1.0x supply;
|
| 193 |
# v1's single book ran 16x and crowded), governed from birth, ctx 8192
|
| 194 |
# where the O(L) read is ~4.5x cheaper than the MHA equivalent.
|
| 195 |
+
# ~873.7M params.
|
| 196 |
"mini-beatrix-2": Preset(
|
| 197 |
model=AlephLMConfig(name="mini-beatrix-2", d_model=1024, n_layers=32,
|
| 198 |
n_heads=16, context=8192,
|
|
|
|
| 210 |
governor="minsep", governor_theta=45.0),
|
| 211 |
curriculum=_curriculum(500_000_000, 8_000_000_000, 16_000_000_000),
|
| 212 |
),
|
| 213 |
+
# THE 2s MISSION (2026-08-26): the next stage up from v1, since the
|
| 214 |
+
# large shape above could not be trained on the available card: the lawful
|
| 215 |
# full-splat craft one rung above v1 β d1024 L20 ctx4096, governed
|
| 216 |
# 4x64@128 books (4x supply headroom vs v1's crowded 16x). Also the
|
| 217 |
# screen bed for every v2-era gating cell. hub_ckpt=0: at 237M the
|
|
|
|
| 244 |
data_scale: float = 4.0, epoch_cap: float | None = None,
|
| 245 |
rebalance_to: str | None = None,
|
| 246 |
name: str | None = None) -> Preset:
|
| 247 |
+
"""The v3 craft (the v3 plan of 2026-09-15; sizing 09-15):
|
| 248 |
the solidified all-splat form at d1024 β a governed hub in EVERY
|
| 249 |
block, the certified hub geometry (4 books x 64 @ D128), banks
|
| 250 |
3 x ff1024, head 256@256, ctx 4096 β at a depth the throughput bench
|