Upload folder using huggingface_hub
Browse files- LICENSE +21 -0
- README.md +75 -0
- artifacts/README.md +39 -0
- artifacts/q2_int8.pt +3 -0
- artifacts/q8g_int8.pt +3 -0
- kronllm/__init__.py +11 -0
- kronllm/bilingual.py +672 -0
- kronllm/cli.py +282 -0
- kronllm/config.py +735 -0
- kronllm/data.py +193 -0
- kronllm/diagnostics.py +386 -0
- kronllm/generate.py +43 -0
- kronllm/hf.py +229 -0
- kronllm/kron.py +854 -0
- kronllm/model.py +1086 -0
- kronllm/plots.py +134 -0
- kronllm/quantization.py +233 -0
- kronllm/testutils.py +29 -0
- kronllm/tokenizer.py +197 -0
- kronllm/train.py +571 -0
- pyproject.toml +57 -0
- requirements.txt +10 -0
LICENSE
ADDED
|
@@ -0,0 +1,21 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
MIT License
|
| 2 |
+
|
| 3 |
+
Copyright (c) 2026 Ivan Kholodilo
|
| 4 |
+
|
| 5 |
+
Permission is hereby granted, free of charge, to any person obtaining a copy
|
| 6 |
+
of this software and associated documentation files (the "Software"), to deal
|
| 7 |
+
in the Software without restriction, including without limitation the rights
|
| 8 |
+
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
| 9 |
+
copies of the Software, and to permit persons to whom the Software is
|
| 10 |
+
furnished to do so, subject to the following conditions:
|
| 11 |
+
|
| 12 |
+
The above copyright notice and this permission notice shall be included in all
|
| 13 |
+
copies or substantial portions of the Software.
|
| 14 |
+
|
| 15 |
+
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
| 16 |
+
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
| 17 |
+
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
| 18 |
+
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
| 19 |
+
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
| 20 |
+
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
| 21 |
+
SOFTWARE.
|
README.md
ADDED
|
@@ -0,0 +1,75 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
---
|
| 2 |
+
language:
|
| 3 |
+
- en
|
| 4 |
+
- ru
|
| 5 |
+
license: mit
|
| 6 |
+
library_name: pytorch
|
| 7 |
+
pipeline_tag: text-generation
|
| 8 |
+
tags:
|
| 9 |
+
- language-model
|
| 10 |
+
- kronecker
|
| 11 |
+
- small-language-model
|
| 12 |
+
- bilingual
|
| 13 |
+
- parameter-efficiency
|
| 14 |
+
---
|
| 15 |
+
|
| 16 |
+
# KronLLM
|
| 17 |
+
|
| 18 |
+
A compact bilingual (English/Russian) language model whose matrix layers are
|
| 19 |
+
built entirely from **Kronecker sums of rank-1 factors**. Every attention and
|
| 20 |
+
FFN projection is a composition of `L` factors instead of a dense `d × d`
|
| 21 |
+
weight: a layer that would need 102,400 numbers needs 15,753.
|
| 22 |
+
|
| 23 |
+
This repository hosts the model card, the code and the shipped int8
|
| 24 |
+
checkpoints. The training corpus is separate: see
|
| 25 |
+
[Icold21/kron-corpus](https://huggingface.co/datasets/Icold21/kron-corpus) and
|
| 26 |
+
the research log at [github.com/Icold21/kron-llm](https://github.com/Icold21/kron-llm).
|
| 27 |
+
|
| 28 |
+
## Shipped checkpoints
|
| 29 |
+
|
| 30 |
+
| file | parameters | size | quality |
|
| 31 |
+
|---|---:|---:|---|
|
| 32 |
+
| `q8g_int8.pt` | 2,851,280 | 3.14 MB | **0.7542 bits/byte** (TinyStories val, 61.4M tokens seen) |
|
| 33 |
+
| `q2_int8.pt` | 2,079,992 | 2.34 MB | previous model, same corpus |
|
| 34 |
+
|
| 35 |
+
Int8 quantisation costs +0.0003 bpb on Q8g. The fp32 checkpoint is 11.4 MB and
|
| 36 |
+
is not shipped.
|
| 37 |
+
|
| 38 |
+
## Usage
|
| 39 |
+
|
| 40 |
+
```python
|
| 41 |
+
import torch
|
| 42 |
+
from kronllm.model import KronTransformer
|
| 43 |
+
from kronllm.quantization import load_int8
|
| 44 |
+
|
| 45 |
+
ckpt = torch.load("q8g_int8.pt", map_location="cpu")
|
| 46 |
+
model = KronTransformer(ckpt["cfg"]).eval()
|
| 47 |
+
load_int8(model, ckpt["model"])
|
| 48 |
+
```
|
| 49 |
+
|
| 50 |
+
The model is a research artifact: it writes short English and Russian text and
|
| 51 |
+
is not an instruct-tuned assistant.
|
| 52 |
+
|
| 53 |
+
## Architecture at a glance
|
| 54 |
+
|
| 55 |
+
| | |
|
| 56 |
+
|---|---|
|
| 57 |
+
| layers / width | 24 / 320, 8 heads, seq 192 |
|
| 58 |
+
| composition | product over L=4 factors, K=6 terms, depth carry `K_win=2` |
|
| 59 |
+
| FFN | gated (SwiGLU-style) |
|
| 60 |
+
| tokenizer | BPE-4096, tied factorised embedding (rank 128) |
|
| 61 |
+
|
| 62 |
+
Measured, not assumed: the depth carry is worth 1.1 % bpb, freezing its scalar
|
| 63 |
+
0.8 %, and widening the window to four layers *costs* 1.4 %. Details in the
|
| 64 |
+
[research log](https://github.com/Icold21/kron-llm/blob/main/docs/results.md).
|
| 65 |
+
|
| 66 |
+
## Next
|
| 67 |
+
|
| 68 |
+
A 10M-parameter model on a 2B-token bilingual corpus (d=512, 25 layers,
|
| 69 |
+
K=10, 8K BPE), trained on a Kaggle T4. The corpus is published; the checkpoint
|
| 70 |
+
will appear here when the run finishes.
|
| 71 |
+
|
| 72 |
+
## Licence
|
| 73 |
+
|
| 74 |
+
Code and weights: MIT. The corpus keeps its sources' licences (see the dataset
|
| 75 |
+
card).
|
artifacts/README.md
ADDED
|
@@ -0,0 +1,39 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
# Artifacts
|
| 2 |
+
|
| 3 |
+
The shipped checkpoints, tracked directly in Git (each is a few megabytes).
|
| 4 |
+
They are the models referenced in the top-level [README](../README.md).
|
| 5 |
+
|
| 6 |
+
| file | model | precision | size | what |
|
| 7 |
+
|---|---|---|---|---|
|
| 8 |
+
| `q8g_int8.pt` | Q8g | int8 | 3.29 MB | **the shipped model** — 0.7542 bpb, 2,851,280 parameters |
|
| 9 |
+
| `q2_int8.pt` | Q2 | int8 | 2.34 MB | previous model, kept for comparison (0.8391 bpb, 2,079,992 parameters) |
|
| 10 |
+
|
| 11 |
+
The int8 files store symmetric per-channel scales; the forward pass is
|
| 12 |
+
unchanged fp32 after `load_any_checkpoint` dequantises them. Q8g int8 is
|
| 13 |
+
near-lossless (**+0.0003 bpb**, 3.55× smaller than fp32); Q2 int8 was
|
| 14 |
+
+0.0004 bpb.
|
| 15 |
+
|
| 16 |
+
The fp32 reference checkpoints stay outside Git, in the run directory:
|
| 17 |
+
`runs/Q8g_long/best.pt` (11.1 MB) and `runs/Q6_10k/best.pt` (9.7 MB).
|
| 18 |
+
|
| 19 |
+
## Loading
|
| 20 |
+
|
| 21 |
+
Through the CLI:
|
| 22 |
+
|
| 23 |
+
```bash
|
| 24 |
+
python -m kronllm.cli generate --run artifacts --checkpoint q8g_int8.pt \
|
| 25 |
+
--prompt "Once upon a time" --temperature 1.0 --top-k 8
|
| 26 |
+
```
|
| 27 |
+
|
| 28 |
+
Through the Hugging Face wrapper:
|
| 29 |
+
|
| 30 |
+
```python
|
| 31 |
+
from kronllm.hf import KronForCausalLM
|
| 32 |
+
from kronllm.tokenizer import load_tokenizer
|
| 33 |
+
|
| 34 |
+
model = KronForCausalLM.from_kron_checkpoint("artifacts/q8g_int8.pt")
|
| 35 |
+
tok = load_tokenizer(model.config.kron_config)
|
| 36 |
+
```
|
| 37 |
+
|
| 38 |
+
`from_kron_checkpoint` accepts fp32, int8 and int4 checkpoints. The returned
|
| 39 |
+
model is in evaluation mode on CPU; call `.to("cuda")` to move it to a GPU.
|
artifacts/q2_int8.pt
ADDED
|
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
version https://git-lfs.github.com/spec/v1
|
| 2 |
+
oid sha256:58190e8f40650ce2b27a36ea420d32e3d9ff0453036a5437ebe8ab4f7b14cdaa
|
| 3 |
+
size 2451616
|
artifacts/q8g_int8.pt
ADDED
|
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
version https://git-lfs.github.com/spec/v1
|
| 2 |
+
oid sha256:a25c53ca5bdf791127e4e10ed0254a3f562ed989ead1739b3b5ea2c3c75464a6
|
| 3 |
+
size 3293344
|
kronllm/__init__.py
ADDED
|
@@ -0,0 +1,11 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""KronLLM: a fully Kronecker-parameterised byte-level language model."""
|
| 2 |
+
from .kron import (KronCompose, apply_rope, effective_rank, kron_apply,
|
| 3 |
+
kron_materialize, rope_frequencies)
|
| 4 |
+
from .model import KronBlock, KronTransformer, RMSNorm
|
| 5 |
+
|
| 6 |
+
__all__ = [
|
| 7 |
+
"KronCompose", "KronTransformer", "KronBlock", "RMSNorm",
|
| 8 |
+
"kron_apply", "kron_materialize", "effective_rank",
|
| 9 |
+
"apply_rope", "rope_frequencies",
|
| 10 |
+
]
|
| 11 |
+
__version__ = "2.0.0"
|
kronllm/bilingual.py
ADDED
|
@@ -0,0 +1,672 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""Corpus construction for the bilingual English/Russian pretraining set.
|
| 2 |
+
|
| 3 |
+
This module implements the pipeline described in ``docs/data_plan.md``:
|
| 4 |
+
|
| 5 |
+
fetch -> normalise -> quality filter -> deduplicate -> mix -> split
|
| 6 |
+
|
| 7 |
+
It deliberately has no third-party dependencies beyond the standard library so
|
| 8 |
+
that the corpus can be rebuilt on a bare machine. The three things it is
|
| 9 |
+
careful about are:
|
| 10 |
+
|
| 11 |
+
* **licences** -- every shard is recorded with ``source``, ``licence``,
|
| 12 |
+
``url`` and ``sha256``; a shard without a known licence is rejected by
|
| 13 |
+
:func:`record_shard`;
|
| 14 |
+
* **language** -- a fast heuristic (script shares) instead of a model, so the
|
| 15 |
+
filter is explainable and reproducible;
|
| 16 |
+
* **boilerplate** -- MediaWiki pages are cleaned of templates, references,
|
| 17 |
+
tables and image links before they are counted as text.
|
| 18 |
+
|
| 19 |
+
The module is used by ``scripts/build_bilingual.py``.
|
| 20 |
+
"""
|
| 21 |
+
from __future__ import annotations
|
| 22 |
+
|
| 23 |
+
import gzip
|
| 24 |
+
import hashlib
|
| 25 |
+
import json
|
| 26 |
+
import re
|
| 27 |
+
import unicodedata
|
| 28 |
+
import zlib
|
| 29 |
+
from dataclasses import dataclass, field, asdict
|
| 30 |
+
from pathlib import Path
|
| 31 |
+
from typing import Iterable, Iterator, Optional, Sequence
|
| 32 |
+
|
| 33 |
+
# ---------------------------------------------------------------------------
|
| 34 |
+
# Normalisation and MediaWiki cleaning
|
| 35 |
+
# ---------------------------------------------------------------------------
|
| 36 |
+
|
| 37 |
+
#: Sections that carry references or navigation rather than prose.
|
| 38 |
+
BOILERPLATE_SECTIONS = {
|
| 39 |
+
# Russian
|
| 40 |
+
"примечания", "литература", "ссылки", "источники", "см. также",
|
| 41 |
+
"библиография", "комментарии", "сноски", "внешние ссылки",
|
| 42 |
+
# English
|
| 43 |
+
"references", "external links", "see also", "further reading",
|
| 44 |
+
"notes", "bibliography", "citations", "sources", "footnotes",
|
| 45 |
+
}
|
| 46 |
+
|
| 47 |
+
_RE_COMMENT = re.compile(r"<!--.*?-->", re.S)
|
| 48 |
+
_RE_REF = re.compile(r"<ref[^>]*?/>|<ref[^>]*?>.*?</ref>", re.S | re.I)
|
| 49 |
+
_RE_TAG = re.compile(r"</?[a-zA-Z][^>]{0,200}>")
|
| 50 |
+
_RE_TABLE = re.compile(r"\{\|.*?\|\}", re.S)
|
| 51 |
+
_RE_MATH = re.compile(r"<math>(.*?)</math>", re.S | re.I)
|
| 52 |
+
_RE_HEADING = re.compile(r"^(={2,6})\s*(.+?)\s*\1\s*$", re.M)
|
| 53 |
+
_RE_FILE = re.compile(
|
| 54 |
+
r"\[\[(?:Файл|File|Изображение|Image|Медиа|Media):[^\[\]]*?(?:\[\[[^\]]*\]\][^\[\]]*?)*\]\]",
|
| 55 |
+
re.I)
|
| 56 |
+
_RE_CATEGORY = re.compile(
|
| 57 |
+
r"\[\[(?:Категория|Category):[^\[\]]*\]\]", re.I)
|
| 58 |
+
_RE_INTERWIKI = re.compile(r"\[\[[a-z]{2,3}:[^\[\]]*\]\]")
|
| 59 |
+
_RE_LINK = re.compile(r"\[\[([^\[\]|]+)(?:\|([^\[\]]*))?\]\]")
|
| 60 |
+
_RE_EXTLINK = re.compile(r"\[(?:https?|ftp)://[^\s\]]+\s*([^\]]*)\]")
|
| 61 |
+
_RE_BOLD = re.compile(r"'{2,5}")
|
| 62 |
+
_RE_LIST = re.compile(r"^[ \t]*[*#:;]+[ \t]*", re.M)
|
| 63 |
+
_RE_WS = re.compile(r"[ \t]+")
|
| 64 |
+
_RE_BLANKLINES = re.compile(r"\n{3,}")
|
| 65 |
+
|
| 66 |
+
|
| 67 |
+
def _strip_templates(text: str) -> str:
|
| 68 |
+
"""Remove ``{{...}}`` blocks, handling nesting.
|
| 69 |
+
|
| 70 |
+
MediaWiki templates are balanced but not regular, so this walks the string
|
| 71 |
+
once with a depth counter instead of using a regex.
|
| 72 |
+
"""
|
| 73 |
+
out: list[str] = []
|
| 74 |
+
depth = 0
|
| 75 |
+
i = 0
|
| 76 |
+
n = len(text)
|
| 77 |
+
while i < n:
|
| 78 |
+
if text.startswith("{{", i):
|
| 79 |
+
depth += 1
|
| 80 |
+
i += 2
|
| 81 |
+
continue
|
| 82 |
+
if text.startswith("}}", i) and depth:
|
| 83 |
+
depth -= 1
|
| 84 |
+
i += 2
|
| 85 |
+
continue
|
| 86 |
+
if depth == 0:
|
| 87 |
+
out.append(text[i])
|
| 88 |
+
i += 1
|
| 89 |
+
return "".join(out)
|
| 90 |
+
|
| 91 |
+
|
| 92 |
+
def _drop_sections(text: str) -> str:
|
| 93 |
+
"""Drop whole sections whose heading is boilerplate (references, links)."""
|
| 94 |
+
lines = text.split("\n")
|
| 95 |
+
kept: list[str] = []
|
| 96 |
+
skip_level: Optional[int] = None
|
| 97 |
+
for line in lines:
|
| 98 |
+
m = _RE_HEADING.match(line)
|
| 99 |
+
if m:
|
| 100 |
+
level = len(m.group(1))
|
| 101 |
+
title = m.group(2).strip().lower().rstrip(":")
|
| 102 |
+
if skip_level is not None and level <= skip_level:
|
| 103 |
+
skip_level = None
|
| 104 |
+
if skip_level is None and title in BOILERPLATE_SECTIONS:
|
| 105 |
+
skip_level = level
|
| 106 |
+
continue
|
| 107 |
+
if skip_level is None:
|
| 108 |
+
kept.append(line)
|
| 109 |
+
return "\n".join(kept)
|
| 110 |
+
|
| 111 |
+
|
| 112 |
+
def clean_wikitext(text: str) -> str:
|
| 113 |
+
"""Convert MediaWiki wikitext to plain prose.
|
| 114 |
+
|
| 115 |
+
The cleaner is intentionally conservative: it removes markup that never
|
| 116 |
+
contains prose (templates, references, tables, files, categories) and
|
| 117 |
+
unwraps markup that wraps prose (links, bold, headings). It does not try
|
| 118 |
+
to be a full parser; residual markup is filtered later by the symbol-ratio
|
| 119 |
+
check.
|
| 120 |
+
|
| 121 |
+
Args:
|
| 122 |
+
text: raw wikitext of one article.
|
| 123 |
+
|
| 124 |
+
Returns:
|
| 125 |
+
Plain text with paragraphs separated by a single blank line.
|
| 126 |
+
"""
|
| 127 |
+
text = _RE_COMMENT.sub(" ", text)
|
| 128 |
+
text = _RE_REF.sub(" ", text)
|
| 129 |
+
text = _RE_TABLE.sub(" ", text)
|
| 130 |
+
text = _strip_templates(text)
|
| 131 |
+
text = _RE_MATH.sub(lambda m: " " + m.group(1).strip() + " ", text)
|
| 132 |
+
text = _drop_sections(text)
|
| 133 |
+
text = _RE_FILE.sub(" ", text)
|
| 134 |
+
text = _RE_CATEGORY.sub(" ", text)
|
| 135 |
+
text = _RE_INTERWIKI.sub(" ", text)
|
| 136 |
+
# headings -> sentences on their own line, then unwrapped
|
| 137 |
+
text = _RE_HEADING.sub(lambda m: "\n\n" + m.group(2).strip() + "\n\n", text)
|
| 138 |
+
text = _RE_LINK.sub(lambda m: (m.group(2) or m.group(1)).strip(), text)
|
| 139 |
+
text = _RE_EXTLINK.sub(lambda m: (m.group(1) or "").strip(), text)
|
| 140 |
+
text = _RE_BOLD.sub("", text)
|
| 141 |
+
text = _RE_LIST.sub("", text)
|
| 142 |
+
text = _RE_TAG.sub(" ", text)
|
| 143 |
+
text = text.replace(" ", " ").replace("&", "&")
|
| 144 |
+
text = text.replace("<", "<").replace(">", ">").replace(""", '"')
|
| 145 |
+
return normalize(text)
|
| 146 |
+
|
| 147 |
+
|
| 148 |
+
def normalize(text: str, keep_indent: bool = False) -> str:
|
| 149 |
+
"""Unicode-normalise and clean whitespace.
|
| 150 |
+
|
| 151 |
+
Args:
|
| 152 |
+
text: input text.
|
| 153 |
+
keep_indent: keep leading indentation and blank lines (used for code,
|
| 154 |
+
where indentation carries meaning).
|
| 155 |
+
|
| 156 |
+
Returns:
|
| 157 |
+
Normalised text, paragraphs separated by ``\\n\\n``.
|
| 158 |
+
"""
|
| 159 |
+
text = unicodedata.normalize("NFKC", text)
|
| 160 |
+
text = text.replace("\r\n", "\n").replace("\r", "\n")
|
| 161 |
+
text = text.replace("\u00ad", "") # soft hyphen
|
| 162 |
+
text = text.replace("\ufeff", "")
|
| 163 |
+
for space in ("\u00a0", "\u2009", "\u202f", "\u2007"):
|
| 164 |
+
text = text.replace(space, " ")
|
| 165 |
+
if keep_indent:
|
| 166 |
+
lines = [ln.rstrip() for ln in text.split("\n")]
|
| 167 |
+
out: list[str] = []
|
| 168 |
+
for ln in lines:
|
| 169 |
+
out.append(ln)
|
| 170 |
+
return _RE_BLANKLINES.sub("\n\n\n", "\n".join(out)).strip("\n")
|
| 171 |
+
lines = []
|
| 172 |
+
for ln in text.split("\n"):
|
| 173 |
+
ln = _RE_WS.sub(" ", ln).strip()
|
| 174 |
+
lines.append(ln)
|
| 175 |
+
text = "\n".join(lines)
|
| 176 |
+
text = _RE_BLANKLINES.sub("\n\n", text)
|
| 177 |
+
return text.strip()
|
| 178 |
+
|
| 179 |
+
|
| 180 |
+
# ---------------------------------------------------------------------------
|
| 181 |
+
# Language and quality filters
|
| 182 |
+
# ---------------------------------------------------------------------------
|
| 183 |
+
|
| 184 |
+
_CYRILLIC = re.compile(r"[\u0400-\u04FF]")
|
| 185 |
+
_LATIN = re.compile(r"[A-Za-z]")
|
| 186 |
+
_LETTER = re.compile(r"[^\W\d_]", re.UNICODE)
|
| 187 |
+
_DIGIT = re.compile(r"\d")
|
| 188 |
+
_SYMBOL = re.compile(r"[^\w\s]", re.UNICODE)
|
| 189 |
+
|
| 190 |
+
|
| 191 |
+
@dataclass
|
| 192 |
+
class QualityStats:
|
| 193 |
+
"""Measured properties of one document."""
|
| 194 |
+
|
| 195 |
+
chars: int
|
| 196 |
+
words: int
|
| 197 |
+
letters: int
|
| 198 |
+
digits: int
|
| 199 |
+
symbols: int
|
| 200 |
+
cyrillic: int
|
| 201 |
+
latin: int
|
| 202 |
+
repeat_ratio: float
|
| 203 |
+
compression: float
|
| 204 |
+
ok: bool
|
| 205 |
+
reasons: list[str] = field(default_factory=list)
|
| 206 |
+
|
| 207 |
+
|
| 208 |
+
def _repeat_ratio(words: Sequence[str], n: int = 5) -> float:
|
| 209 |
+
"""Fraction of ``n``-grams that occur more than once."""
|
| 210 |
+
if len(words) < n:
|
| 211 |
+
return 0.0
|
| 212 |
+
grams = [tuple(words[i:i + n]) for i in range(len(words) - n + 1)]
|
| 213 |
+
if not grams:
|
| 214 |
+
return 0.0
|
| 215 |
+
unique = len(set(grams))
|
| 216 |
+
return 1.0 - unique / len(grams)
|
| 217 |
+
|
| 218 |
+
|
| 219 |
+
def quality(text: str, language: Optional[str] = None,
|
| 220 |
+
min_chars: int = 400, max_symbol_ratio: float = 0.20,
|
| 221 |
+
max_digit_ratio: float = 0.12,
|
| 222 |
+
max_repeat_ratio: float = 0.30,
|
| 223 |
+
min_compression: float = 0.22,
|
| 224 |
+
max_compression: float = 0.80) -> QualityStats:
|
| 225 |
+
"""Score a document and decide whether it may enter the corpus.
|
| 226 |
+
|
| 227 |
+
The thresholds are calibrated on the pilot, not on theory: they remove
|
| 228 |
+
lists, tables, navigation residue and machine-generated text while keeping
|
| 229 |
+
dense prose and formulas.
|
| 230 |
+
|
| 231 |
+
Args:
|
| 232 |
+
text: normalised document.
|
| 233 |
+
language: ``"en"`` or ``"ru"``; ``None`` skips the script check.
|
| 234 |
+
min_chars: minimum document length in characters.
|
| 235 |
+
max_symbol_ratio: maximum share of non-alphanumeric characters.
|
| 236 |
+
max_digit_ratio: maximum share of digits (filters data dumps).
|
| 237 |
+
max_repeat_ratio: maximum share of repeated 5-grams.
|
| 238 |
+
min_compression: minimum zlib ratio (too uniform text is suspicious).
|
| 239 |
+
max_compression: maximum zlib ratio (too random text is suspicious).
|
| 240 |
+
|
| 241 |
+
Returns:
|
| 242 |
+
A :class:`QualityStats` with ``ok`` set to the verdict.
|
| 243 |
+
"""
|
| 244 |
+
n = len(text)
|
| 245 |
+
letters = len(_LETTER.findall(text))
|
| 246 |
+
digits = len(_DIGIT.findall(text))
|
| 247 |
+
symbols = len(_SYMBOL.findall(text))
|
| 248 |
+
cyr = len(_CYRILLIC.findall(text))
|
| 249 |
+
lat = len(_LATIN.findall(text))
|
| 250 |
+
words = text.split()
|
| 251 |
+
comp = len(zlib.compress(text.encode("utf-8"), 6)) / max(1, len(text.encode("utf-8")))
|
| 252 |
+
rep = _repeat_ratio(words)
|
| 253 |
+
|
| 254 |
+
reasons: list[str] = []
|
| 255 |
+
if n < min_chars:
|
| 256 |
+
reasons.append("short")
|
| 257 |
+
if n and symbols / n > max_symbol_ratio:
|
| 258 |
+
reasons.append("symbols")
|
| 259 |
+
if n and digits / n > max_digit_ratio:
|
| 260 |
+
reasons.append("digits")
|
| 261 |
+
if rep > max_repeat_ratio:
|
| 262 |
+
reasons.append("repetition")
|
| 263 |
+
if not (min_compression <= comp <= max_compression):
|
| 264 |
+
reasons.append("compression")
|
| 265 |
+
if language == "ru" and (cyr + lat) and cyr / (cyr + lat) < 0.60:
|
| 266 |
+
reasons.append("not_ru")
|
| 267 |
+
if language == "en" and (cyr + lat) and lat / (cyr + lat) < 0.80:
|
| 268 |
+
reasons.append("not_en")
|
| 269 |
+
|
| 270 |
+
return QualityStats(chars=n, words=len(words), letters=letters,
|
| 271 |
+
digits=digits, symbols=symbols, cyrillic=cyr,
|
| 272 |
+
latin=lat, repeat_ratio=round(rep, 4),
|
| 273 |
+
compression=round(comp, 4), ok=not reasons,
|
| 274 |
+
reasons=reasons)
|
| 275 |
+
|
| 276 |
+
|
| 277 |
+
# ---------------------------------------------------------------------------
|
| 278 |
+
# Near-duplicate detection (MinHash + LSH)
|
| 279 |
+
# ---------------------------------------------------------------------------
|
| 280 |
+
|
| 281 |
+
def _shingles(text: str, k: int = 5) -> set[int]:
|
| 282 |
+
"""Hashed word shingles of a document."""
|
| 283 |
+
words = text.lower().split()
|
| 284 |
+
if len(words) < k:
|
| 285 |
+
return {hash(tuple(words))}
|
| 286 |
+
return {hash(tuple(words[i:i + k])) for i in range(len(words) - k + 1)}
|
| 287 |
+
|
| 288 |
+
|
| 289 |
+
def minhash(text: str, permutations: int = 64, k: int = 5,
|
| 290 |
+
seed: int = 13) -> tuple[int, ...]:
|
| 291 |
+
"""MinHash signature used for near-duplicate detection."""
|
| 292 |
+
import random
|
| 293 |
+
|
| 294 |
+
rng = random.Random(seed)
|
| 295 |
+
a = [rng.randrange(1, 2**61 - 1) for _ in range(permutations)]
|
| 296 |
+
b = [rng.randrange(0, 2**61 - 1) for _ in range(permutations)]
|
| 297 |
+
sig = []
|
| 298 |
+
sh = _shingles(text, k)
|
| 299 |
+
for i in range(permutations):
|
| 300 |
+
sig.append(min(((a[i] * s + b[i]) % (2**61 - 1)) for s in sh))
|
| 301 |
+
return tuple(sig)
|
| 302 |
+
|
| 303 |
+
|
| 304 |
+
def _bands(sig: Sequence[int], bands: int) -> Iterator[tuple]:
|
| 305 |
+
step = max(1, len(sig) // bands)
|
| 306 |
+
for i in range(0, len(sig), step):
|
| 307 |
+
yield (i, tuple(sig[i:i + step]))
|
| 308 |
+
|
| 309 |
+
|
| 310 |
+
def dedup(docs: Iterable[dict], threshold: float = 0.8,
|
| 311 |
+
text_key: str = "text") -> tuple[list[dict], int]:
|
| 312 |
+
"""Remove exact and near duplicates.
|
| 313 |
+
|
| 314 |
+
Exact duplicates are dropped by SHA-256 of the normalised text, near
|
| 315 |
+
duplicates by MinHash banding with a Jaccard cutoff. The first document
|
| 316 |
+
wins, so callers should feed sources in priority order.
|
| 317 |
+
|
| 318 |
+
Args:
|
| 319 |
+
docs: documents with at least ``text`` (and ``sha256`` optionally).
|
| 320 |
+
threshold: Jaccard similarity above which two documents are duplicates.
|
| 321 |
+
text_key: key holding the text.
|
| 322 |
+
|
| 323 |
+
Returns:
|
| 324 |
+
``(kept, dropped)``.
|
| 325 |
+
"""
|
| 326 |
+
seen_exact: set[str] = set()
|
| 327 |
+
buckets: dict[tuple, list[tuple[int, ...]]] = {}
|
| 328 |
+
kept: list[dict] = []
|
| 329 |
+
dropped = 0
|
| 330 |
+
for doc in docs:
|
| 331 |
+
text = doc[text_key]
|
| 332 |
+
h = doc.get("sha256") or hashlib.sha256(text.encode("utf-8")).hexdigest()
|
| 333 |
+
if h in seen_exact:
|
| 334 |
+
dropped += 1
|
| 335 |
+
continue
|
| 336 |
+
sig = minhash(text)
|
| 337 |
+
is_dup = False
|
| 338 |
+
for band in _bands(sig, 16):
|
| 339 |
+
bucket = buckets.setdefault(band, [])
|
| 340 |
+
for other in bucket:
|
| 341 |
+
same = sum(1 for x, y in zip(sig, other) if x == y)
|
| 342 |
+
if same / len(sig) >= threshold:
|
| 343 |
+
is_dup = True
|
| 344 |
+
break
|
| 345 |
+
if is_dup:
|
| 346 |
+
break
|
| 347 |
+
if is_dup:
|
| 348 |
+
dropped += 1
|
| 349 |
+
continue
|
| 350 |
+
for band in _bands(sig, 16):
|
| 351 |
+
buckets.setdefault(band, []).append(sig)
|
| 352 |
+
seen_exact.add(h)
|
| 353 |
+
doc["sha256"] = h
|
| 354 |
+
kept.append(doc)
|
| 355 |
+
return kept, dropped
|
| 356 |
+
|
| 357 |
+
|
| 358 |
+
# ---------------------------------------------------------------------------
|
| 359 |
+
# Manifest
|
| 360 |
+
# ---------------------------------------------------------------------------
|
| 361 |
+
|
| 362 |
+
REQUIRED_FIELDS = ("source", "licence", "url", "language", "domain",
|
| 363 |
+
"sha256", "bytes")
|
| 364 |
+
|
| 365 |
+
|
| 366 |
+
@dataclass
|
| 367 |
+
class Shard:
|
| 368 |
+
"""One corpus shard and its provenance."""
|
| 369 |
+
|
| 370 |
+
source: str
|
| 371 |
+
licence: str
|
| 372 |
+
url: str
|
| 373 |
+
language: str
|
| 374 |
+
domain: str
|
| 375 |
+
sha256: str
|
| 376 |
+
bytes: int
|
| 377 |
+
docs: int = 0
|
| 378 |
+
path: str = ""
|
| 379 |
+
note: str = ""
|
| 380 |
+
|
| 381 |
+
def as_dict(self) -> dict:
|
| 382 |
+
return asdict(self)
|
| 383 |
+
|
| 384 |
+
|
| 385 |
+
def sha256_file(path: Path) -> str:
|
| 386 |
+
"""SHA-256 of a file, streamed."""
|
| 387 |
+
h = hashlib.sha256()
|
| 388 |
+
with open(path, "rb") as f:
|
| 389 |
+
for chunk in iter(lambda: f.read(1 << 20), b""):
|
| 390 |
+
h.update(chunk)
|
| 391 |
+
return h.hexdigest()
|
| 392 |
+
|
| 393 |
+
|
| 394 |
+
def record_shard(path: Path, manifest: Path, source: str, licence: str,
|
| 395 |
+
url: str, language: str, domain: str, docs: int = 0,
|
| 396 |
+
note: str = "") -> Shard:
|
| 397 |
+
"""Append one shard to the JSONL manifest.
|
| 398 |
+
|
| 399 |
+
Args:
|
| 400 |
+
path: shard file on disk.
|
| 401 |
+
manifest: JSONL manifest path.
|
| 402 |
+
source: human-readable source name.
|
| 403 |
+
licence: licence identifier; must be known.
|
| 404 |
+
url: canonical source URL.
|
| 405 |
+
language: ``en`` or ``ru``.
|
| 406 |
+
domain: ``math``, ``science``, ``logic``, ``literature``, ``code``
|
| 407 |
+
or ``encyclopedic``.
|
| 408 |
+
docs: number of documents in the shard.
|
| 409 |
+
note: free-form remark.
|
| 410 |
+
|
| 411 |
+
Raises:
|
| 412 |
+
ValueError: if the licence is unknown or a required field is empty.
|
| 413 |
+
"""
|
| 414 |
+
if licence.strip().lower() in {"", "unknown", "n/a", "none"}:
|
| 415 |
+
raise ValueError(f"shard {path} has no known licence")
|
| 416 |
+
if language not in {"en", "ru"}:
|
| 417 |
+
raise ValueError(f"unexpected language {language!r}")
|
| 418 |
+
if domain not in {"math", "science", "logic", "literature", "code",
|
| 419 |
+
"encyclopedic", "ml", "cs", "humour", "identity"}:
|
| 420 |
+
raise ValueError(f"unexpected domain {domain!r}")
|
| 421 |
+
shard = Shard(source=source, licence=licence, url=url, language=language,
|
| 422 |
+
domain=domain, sha256=sha256_file(path),
|
| 423 |
+
bytes=path.stat().st_size, docs=docs, path=str(path),
|
| 424 |
+
note=note)
|
| 425 |
+
manifest.parent.mkdir(parents=True, exist_ok=True)
|
| 426 |
+
with open(manifest, "a", encoding="utf-8") as f:
|
| 427 |
+
f.write(json.dumps(shard.as_dict(), ensure_ascii=False) + "\n")
|
| 428 |
+
return shard
|
| 429 |
+
|
| 430 |
+
|
| 431 |
+
# ---------------------------------------------------------------------------
|
| 432 |
+
# Mixing and splitting
|
| 433 |
+
# ---------------------------------------------------------------------------
|
| 434 |
+
|
| 435 |
+
DOMAIN_QUOTAS = {
|
| 436 |
+
"math": 0.25,
|
| 437 |
+
"science": 0.14,
|
| 438 |
+
"code": 0.15,
|
| 439 |
+
"logic": 0.10,
|
| 440 |
+
"literature": 0.11,
|
| 441 |
+
"encyclopedic": 0.09,
|
| 442 |
+
# Computer science, software engineering and engineering prose. A small
|
| 443 |
+
# slice: the code domain already teaches the syntax, this teaches the
|
| 444 |
+
# discourse around it.
|
| 445 |
+
"cs": 0.03,
|
| 446 |
+
"ml": 0.12,
|
| 447 |
+
# Humour, satire and jokes: short texts that teach tone rather than facts.
|
| 448 |
+
"humour": 0.01,
|
| 449 |
+
# Self-identity: synthetic first-person texts that teach the model its own
|
| 450 |
+
# name. The quota is an upper bound; the slice is sample-limited, so the
|
| 451 |
+
# mixer reports it as a shortfall rather than duplicating text.
|
| 452 |
+
"identity": 0.01,
|
| 453 |
+
}
|
| 454 |
+
|
| 455 |
+
|
| 456 |
+
def mix_corpus(shards: Sequence[Shard], total_bytes: int,
|
| 457 |
+
language_ratio: float = 0.40,
|
| 458 |
+
quotas: Optional[dict[str, float]] = None,
|
| 459 |
+
) -> dict[str, list[tuple[Shard, int]]]:
|
| 460 |
+
"""Allocate a byte budget per language and domain.
|
| 461 |
+
|
| 462 |
+
English receives ``1 - language_ratio`` of the total and Russian
|
| 463 |
+
``language_ratio``. Inside a language the domain quotas are applied to
|
| 464 |
+
whatever is available; a starved domain is simply under-filled, which the
|
| 465 |
+
caller can detect by comparing the allocation with the quota.
|
| 466 |
+
|
| 467 |
+
Args:
|
| 468 |
+
shards: available shards.
|
| 469 |
+
total_bytes: budget for the whole corpus, both languages.
|
| 470 |
+
language_ratio: share of Russian bytes (0.40 in the plan).
|
| 471 |
+
quotas: domain shares; defaults to :data:`DOMAIN_QUOTAS`.
|
| 472 |
+
|
| 473 |
+
Returns:
|
| 474 |
+
Mapping ``language -> [(shard, allocated_bytes), ...]``.
|
| 475 |
+
"""
|
| 476 |
+
quotas = quotas or DOMAIN_QUOTAS
|
| 477 |
+
plan: dict[str, list[tuple[Shard, int]]] = {"en": [], "ru": []}
|
| 478 |
+
for lang, share in (("en", 1.0 - language_ratio), ("ru", language_ratio)):
|
| 479 |
+
lang_budget = int(total_bytes * share)
|
| 480 |
+
by_domain: dict[str, list[Shard]] = {}
|
| 481 |
+
for s in shards:
|
| 482 |
+
if s.language == lang:
|
| 483 |
+
by_domain.setdefault(s.domain, []).append(s)
|
| 484 |
+
for domain, quota in quotas.items():
|
| 485 |
+
want = int(lang_budget * quota)
|
| 486 |
+
pool = by_domain.get(domain, [])
|
| 487 |
+
have = sum(s.bytes for s in pool)
|
| 488 |
+
take = min(want, have)
|
| 489 |
+
if take <= 0:
|
| 490 |
+
continue
|
| 491 |
+
for s in pool:
|
| 492 |
+
alloc = int(round(take * s.bytes / have))
|
| 493 |
+
if alloc:
|
| 494 |
+
plan[lang].append((s, min(alloc, s.bytes)))
|
| 495 |
+
return plan
|
| 496 |
+
|
| 497 |
+
|
| 498 |
+
def write_heldout(docs: Iterable[dict], out_dir: Path, per_language_bytes: int,
|
| 499 |
+
language: str) -> Path:
|
| 500 |
+
"""Write a deterministic held-out slice (first documents, fixed order)."""
|
| 501 |
+
out_dir.mkdir(parents=True, exist_ok=True)
|
| 502 |
+
path = out_dir / f"heldout_{language}.txt"
|
| 503 |
+
written = 0
|
| 504 |
+
with open(path, "w", encoding="utf-8") as f:
|
| 505 |
+
for doc in docs:
|
| 506 |
+
if written >= per_language_bytes:
|
| 507 |
+
break
|
| 508 |
+
text = doc["text"]
|
| 509 |
+
f.write(text + "\n\n")
|
| 510 |
+
written += len(text.encode("utf-8"))
|
| 511 |
+
return path
|
| 512 |
+
|
| 513 |
+
|
| 514 |
+
def quota_mix(docs: Sequence[dict], budget_per_language: int,
|
| 515 |
+
quotas: Optional[dict[str, float]] = None,
|
| 516 |
+
heldout_bytes: int = 0,
|
| 517 |
+
budgets: Optional[dict[str, int]] = None) -> dict:
|
| 518 |
+
"""Split a held-out slice and apply the domain quotas.
|
| 519 |
+
|
| 520 |
+
Fetching is deliberately generous and uneven; this is where the plan is
|
| 521 |
+
enforced. Both the held-out reservation and the quota selection walk
|
| 522 |
+
documents in SHA-256 order, so the same input always yields the same
|
| 523 |
+
corpus.
|
| 524 |
+
|
| 525 |
+
Args:
|
| 526 |
+
docs: deduplicated documents with ``language``, ``domain``, ``sha256``
|
| 527 |
+
and ``text``.
|
| 528 |
+
budget_per_language: maximum training bytes per language.
|
| 529 |
+
quotas: domain shares inside a language; defaults to
|
| 530 |
+
:data:`DOMAIN_QUOTAS`.
|
| 531 |
+
heldout_bytes: total held-out budget, split evenly across
|
| 532 |
+
``2 * len(quotas)`` buckets.
|
| 533 |
+
budgets: per-language override of ``budget_per_language``; used to
|
| 534 |
+
enforce the 60/40 byte ratio when the fetched volumes differ.
|
| 535 |
+
|
| 536 |
+
Returns:
|
| 537 |
+
``{"train": {lang: {domain: [docs]}}, "heldout": {lang: [docs]},
|
| 538 |
+
"shortfalls": [str], "bytes": {lang: {domain: int}}}``.
|
| 539 |
+
"""
|
| 540 |
+
quotas = quotas or DOMAIN_QUOTAS
|
| 541 |
+
ordered = sorted(docs, key=lambda d: d.get("sha256") or "")
|
| 542 |
+
buckets: dict[tuple, list[dict]] = {}
|
| 543 |
+
for doc in ordered:
|
| 544 |
+
buckets.setdefault((doc["language"], doc["domain"]), []).append(doc)
|
| 545 |
+
|
| 546 |
+
per_bucket = int(heldout_bytes / max(1, 2 * len(quotas)))
|
| 547 |
+
heldout: dict[str, list[dict]] = {"en": [], "ru": []}
|
| 548 |
+
pool: dict[tuple, list[dict]] = {}
|
| 549 |
+
for key, items in buckets.items():
|
| 550 |
+
bucket_bytes = sum(len(d["text"].encode("utf-8")) for d in items)
|
| 551 |
+
# A bucket smaller than 5 MB is a deliberate small slice (identity,
|
| 552 |
+
# rare domains). Holding out a share of it would remove it from
|
| 553 |
+
# training entirely, which is the opposite of the intent.
|
| 554 |
+
budget = per_bucket if bucket_bytes >= 5_000_000 else 0
|
| 555 |
+
taken = 0
|
| 556 |
+
n = 0
|
| 557 |
+
while n < len(items) and taken < budget:
|
| 558 |
+
taken += len(items[n]["text"].encode("utf-8"))
|
| 559 |
+
n += 1
|
| 560 |
+
heldout.setdefault(key[0], []).extend(items[:n])
|
| 561 |
+
pool[key] = items[n:]
|
| 562 |
+
|
| 563 |
+
train: dict[str, dict[str, list[dict]]] = {"en": {}, "ru": {}}
|
| 564 |
+
sizes: dict[str, dict[str, int]] = {"en": {}, "ru": {}}
|
| 565 |
+
shortfalls: list[str] = []
|
| 566 |
+
for lang in ("en", "ru"):
|
| 567 |
+
available = sum(len(d["text"].encode("utf-8"))
|
| 568 |
+
for key, items in pool.items() if key[0] == lang
|
| 569 |
+
for d in items)
|
| 570 |
+
lang_budget = int((budgets or {}).get(lang, budget_per_language))
|
| 571 |
+
budget = min(available, lang_budget)
|
| 572 |
+
for domain, quota in quotas.items():
|
| 573 |
+
want = int(budget * quota)
|
| 574 |
+
items = pool.get((lang, domain), [])
|
| 575 |
+
chosen: list[dict] = []
|
| 576 |
+
taken = 0
|
| 577 |
+
for doc in items:
|
| 578 |
+
if taken >= want:
|
| 579 |
+
break
|
| 580 |
+
chosen.append(doc)
|
| 581 |
+
taken += len(doc["text"].encode("utf-8"))
|
| 582 |
+
if not chosen:
|
| 583 |
+
shortfalls.append(f"{lang}/{domain}: empty")
|
| 584 |
+
continue
|
| 585 |
+
if taken < want * 0.8:
|
| 586 |
+
shortfalls.append(f"{lang}/{domain}: "
|
| 587 |
+
f"{taken/1e6:.1f} MB of {want/1e6:.1f} MB")
|
| 588 |
+
train[lang][domain] = chosen
|
| 589 |
+
sizes[lang][domain] = taken
|
| 590 |
+
return {"train": train, "heldout": heldout, "shortfalls": shortfalls,
|
| 591 |
+
"bytes": sizes}
|
| 592 |
+
|
| 593 |
+
|
| 594 |
+
def iter_jsonl(path: Path) -> Iterator[dict]:
|
| 595 |
+
"""Stream documents from a JSONL file."""
|
| 596 |
+
with open(path, encoding="utf-8") as f:
|
| 597 |
+
for line in f:
|
| 598 |
+
line = line.strip()
|
| 599 |
+
if line:
|
| 600 |
+
yield json.loads(line)
|
| 601 |
+
|
| 602 |
+
|
| 603 |
+
def append_jsonl(path: Path, docs: Iterable[dict]) -> int:
|
| 604 |
+
"""Append documents to a JSONL file and return how many were written."""
|
| 605 |
+
path.parent.mkdir(parents=True, exist_ok=True)
|
| 606 |
+
n = 0
|
| 607 |
+
with open(path, "a", encoding="utf-8") as f:
|
| 608 |
+
for doc in docs:
|
| 609 |
+
f.write(json.dumps(doc, ensure_ascii=False) + "\n")
|
| 610 |
+
n += 1
|
| 611 |
+
return n
|
| 612 |
+
|
| 613 |
+
|
| 614 |
+
def corpus_report(shards: Sequence[Shard], out_path: Path) -> dict:
|
| 615 |
+
"""Summarise a manifest into a JSON report (bytes, docs, licences)."""
|
| 616 |
+
report: dict = {"total_bytes": 0, "total_docs": 0, "by_language": {},
|
| 617 |
+
"by_domain": {}, "by_licence": {}, "shards": len(shards)}
|
| 618 |
+
for s in shards:
|
| 619 |
+
report["total_bytes"] += s.bytes
|
| 620 |
+
report["total_docs"] += s.docs
|
| 621 |
+
lang = report["by_language"].setdefault(
|
| 622 |
+
s.language, {"bytes": 0, "docs": 0, "domains": {}})
|
| 623 |
+
lang["bytes"] += s.bytes
|
| 624 |
+
lang["docs"] += s.docs
|
| 625 |
+
lang["domains"][s.domain] = lang["domains"].get(s.domain, 0) + s.bytes
|
| 626 |
+
dom = report["by_domain"].setdefault(s.domain, {"bytes": 0, "docs": 0})
|
| 627 |
+
dom["bytes"] += s.bytes
|
| 628 |
+
dom["docs"] += s.docs
|
| 629 |
+
report["by_licence"][s.licence] = (
|
| 630 |
+
report["by_licence"].get(s.licence, 0) + s.bytes)
|
| 631 |
+
total = max(1, report["total_bytes"])
|
| 632 |
+
report["language_share"] = {
|
| 633 |
+
k: round(v["bytes"] / total, 4)
|
| 634 |
+
for k, v in report["by_language"].items()}
|
| 635 |
+
report["domain_share"] = {
|
| 636 |
+
k: round(v["bytes"] / total, 4)
|
| 637 |
+
for k, v in report["by_domain"].items()}
|
| 638 |
+
out_path.parent.mkdir(parents=True, exist_ok=True)
|
| 639 |
+
out_path.write_text(json.dumps(report, indent=2, ensure_ascii=False),
|
| 640 |
+
encoding="utf-8")
|
| 641 |
+
return report
|
| 642 |
+
|
| 643 |
+
|
| 644 |
+
def read_manifest(path: Path) -> list[Shard]:
|
| 645 |
+
"""Load a JSONL manifest into :class:`Shard` objects."""
|
| 646 |
+
out = []
|
| 647 |
+
if not path.exists():
|
| 648 |
+
return out
|
| 649 |
+
for line in path.read_text(encoding="utf-8").splitlines():
|
| 650 |
+
line = line.strip()
|
| 651 |
+
if line:
|
| 652 |
+
out.append(Shard(**json.loads(line)))
|
| 653 |
+
return out
|
| 654 |
+
|
| 655 |
+
|
| 656 |
+
def pack_shard(docs: Iterable[dict], path: Path, compress: bool = True) -> int:
|
| 657 |
+
"""Write documents to one shard file, optionally gzip-compressed.
|
| 658 |
+
|
| 659 |
+
Shards are newline-delimited JSON so that they can be streamed without
|
| 660 |
+
loading the whole corpus into memory.
|
| 661 |
+
|
| 662 |
+
Returns:
|
| 663 |
+
Number of documents written.
|
| 664 |
+
"""
|
| 665 |
+
path.parent.mkdir(parents=True, exist_ok=True)
|
| 666 |
+
opener = gzip.open if compress else open
|
| 667 |
+
n = 0
|
| 668 |
+
with opener(path, "wt", encoding="utf-8") as f:
|
| 669 |
+
for doc in docs:
|
| 670 |
+
f.write(json.dumps(doc, ensure_ascii=False) + "\n")
|
| 671 |
+
n += 1
|
| 672 |
+
return n
|
kronllm/cli.py
ADDED
|
@@ -0,0 +1,282 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""Command line interface.
|
| 2 |
+
|
| 3 |
+
Examples
|
| 4 |
+
--------
|
| 5 |
+
python -m kronllm.cli train --exp E2
|
| 6 |
+
python -m kronllm.cli train --exp E2 --steps 300 --run-dir runs/smoke
|
| 7 |
+
python -m kronllm.cli generate --run runs/E2 --prompt "Once upon a time"
|
| 8 |
+
python -m kronllm.cli eval --run runs/E2
|
| 9 |
+
python -m kronllm.cli analyze --run runs/E2
|
| 10 |
+
"""
|
| 11 |
+
from __future__ import annotations
|
| 12 |
+
|
| 13 |
+
import argparse
|
| 14 |
+
import json
|
| 15 |
+
import sys
|
| 16 |
+
from pathlib import Path
|
| 17 |
+
from typing import Any, Dict, List, Optional
|
| 18 |
+
|
| 19 |
+
import torch
|
| 20 |
+
|
| 21 |
+
|
| 22 |
+
# ---------------------------------------------------------------------------
|
| 23 |
+
def _load_model(run_dir: str | Path, checkpoint: str = "best.pt"):
|
| 24 |
+
from kronllm.config import load_config
|
| 25 |
+
from kronllm.model import KronTransformer
|
| 26 |
+
from kronllm.quantization import dequantize_state_dict
|
| 27 |
+
|
| 28 |
+
run_dir = Path(run_dir)
|
| 29 |
+
path = run_dir / checkpoint
|
| 30 |
+
if not path.exists():
|
| 31 |
+
path = run_dir / "last.pt"
|
| 32 |
+
if not path.exists():
|
| 33 |
+
raise FileNotFoundError(f"no checkpoint in {run_dir}")
|
| 34 |
+
ck = torch.load(path, map_location="cpu", weights_only=False)
|
| 35 |
+
cfg = ck.get("cfg") or load_config(run_dir / "config.yaml")
|
| 36 |
+
if ck.get("quantized"):
|
| 37 |
+
state = dequantize_state_dict(ck["state"], int(ck["bits"]))
|
| 38 |
+
else:
|
| 39 |
+
state = ck["model"]
|
| 40 |
+
model = KronTransformer(cfg)
|
| 41 |
+
model.load_state_dict(state)
|
| 42 |
+
model.eval()
|
| 43 |
+
return model, cfg, ck
|
| 44 |
+
|
| 45 |
+
|
| 46 |
+
def _device(name: Optional[str] = None) -> torch.device:
|
| 47 |
+
if name:
|
| 48 |
+
return torch.device(name)
|
| 49 |
+
return torch.device("cuda" if torch.cuda.is_available() else "cpu")
|
| 50 |
+
|
| 51 |
+
|
| 52 |
+
# ---------------------------------------------------------------------------
|
| 53 |
+
def cmd_train(args: argparse.Namespace) -> None:
|
| 54 |
+
from kronllm.config import experiment_config, load_config
|
| 55 |
+
from kronllm.train import train
|
| 56 |
+
|
| 57 |
+
if args.exp:
|
| 58 |
+
cfg = experiment_config(args.exp, args.config, args.override)
|
| 59 |
+
else:
|
| 60 |
+
cfg = load_config(args.config, args.override)
|
| 61 |
+
|
| 62 |
+
if args.steps is not None:
|
| 63 |
+
cfg["training"]["max_steps"] = args.steps
|
| 64 |
+
if args.epochs is not None:
|
| 65 |
+
cfg["training"]["epochs"] = args.epochs
|
| 66 |
+
if args.batch_size is not None:
|
| 67 |
+
cfg["training"]["batch_size"] = args.batch_size
|
| 68 |
+
if args.lr is not None:
|
| 69 |
+
cfg["training"]["lr"] = args.lr
|
| 70 |
+
if args.device is not None:
|
| 71 |
+
cfg["training"]["device"] = args.device
|
| 72 |
+
if args.no_amp:
|
| 73 |
+
cfg["training"]["amp"] = "none"
|
| 74 |
+
if args.data_dir is not None:
|
| 75 |
+
cfg["data"]["data_dir"] = args.data_dir
|
| 76 |
+
if args.wandb:
|
| 77 |
+
cfg["logging"]["wandb"] = True
|
| 78 |
+
|
| 79 |
+
run_dir = Path(args.run_dir or (Path(cfg["training"]["out_dir"])
|
| 80 |
+
/ (args.exp or "custom")))
|
| 81 |
+
results = train(cfg, run_dir, resume=args.resume)
|
| 82 |
+
print(json.dumps({k: results[k] for k in
|
| 83 |
+
("experiment", "n_params", "val_ppl", "best_val_ppl",
|
| 84 |
+
"val_bpb", "time_per_epoch_sec", "peak_vram_gb")}, indent=2))
|
| 85 |
+
|
| 86 |
+
|
| 87 |
+
def cmd_generate(args: argparse.Namespace) -> None:
|
| 88 |
+
from kronllm.generate import generate_texts
|
| 89 |
+
from kronllm.tokenizer import load_tokenizer
|
| 90 |
+
|
| 91 |
+
model, cfg, _ = _load_model(args.run, args.checkpoint)
|
| 92 |
+
device = _device(args.device)
|
| 93 |
+
model.to(device)
|
| 94 |
+
texts = generate_texts(
|
| 95 |
+
model, [args.prompt], max_new_tokens=args.max_new,
|
| 96 |
+
temperature=args.temperature, top_k=args.top_k,
|
| 97 |
+
n_samples=args.samples, seed=args.seed, device=device,
|
| 98 |
+
tokenizer=load_tokenizer(cfg))
|
| 99 |
+
for text in texts:
|
| 100 |
+
print("-" * 70)
|
| 101 |
+
print(text)
|
| 102 |
+
|
| 103 |
+
|
| 104 |
+
def cmd_eval(args: argparse.Namespace) -> None:
|
| 105 |
+
from kronllm.data import open_dataset
|
| 106 |
+
from kronllm.diagnostics import (comparison_report, sigma_theta_report)
|
| 107 |
+
from kronllm.train import diagnostics, evaluate
|
| 108 |
+
|
| 109 |
+
model, cfg, ck = _load_model(args.run, args.checkpoint)
|
| 110 |
+
device = _device(args.device)
|
| 111 |
+
model.to(device)
|
| 112 |
+
ds = open_dataset(cfg, "val")
|
| 113 |
+
out: Dict[str, Any] = {"checkpoint": str(args.run), "step": ck.get("step"),
|
| 114 |
+
**comparison_report(model, int(cfg["data"]["seq_len"]))}
|
| 115 |
+
out.update(evaluate(model, ds, 32, device, args.batches, cfg["training"]["amp"]))
|
| 116 |
+
dg = diagnostics(model, ds, 32, device, spectral=True,
|
| 117 |
+
amp=cfg["training"]["amp"])
|
| 118 |
+
out.update({k: v for k, v in dg.items() if isinstance(v, (int, float))})
|
| 119 |
+
print(json.dumps(out, indent=2, default=float))
|
| 120 |
+
|
| 121 |
+
|
| 122 |
+
def cmd_analyze(args: argparse.Namespace) -> None:
|
| 123 |
+
scripts_dir = Path(__file__).resolve().parents[1] / "scripts"
|
| 124 |
+
if str(scripts_dir) not in sys.path:
|
| 125 |
+
sys.path.insert(0, str(scripts_dir))
|
| 126 |
+
import analyze as analyze_mod
|
| 127 |
+
|
| 128 |
+
argv = ["--run", str(args.run), "--checkpoint", args.checkpoint,
|
| 129 |
+
"--samples", str(args.samples)]
|
| 130 |
+
if args.skip_generation:
|
| 131 |
+
argv.append("--skip-generation")
|
| 132 |
+
old = sys.argv
|
| 133 |
+
try:
|
| 134 |
+
sys.argv = ["analyze.py"] + argv
|
| 135 |
+
analyze_mod.main()
|
| 136 |
+
finally:
|
| 137 |
+
sys.argv = old
|
| 138 |
+
|
| 139 |
+
|
| 140 |
+
# ---------------------------------------------------------------------------
|
| 141 |
+
def cmd_quantize(args: argparse.Namespace) -> None:
|
| 142 |
+
"""Quantize a checkpoint to int8/int4 and report the quality delta."""
|
| 143 |
+
from kronllm.data import open_dataset
|
| 144 |
+
from kronllm.quantization import quantize_checkpoint
|
| 145 |
+
from kronllm.tokenizer import bits_per_byte, load_tokenizer
|
| 146 |
+
from kronllm.train import evaluate
|
| 147 |
+
|
| 148 |
+
run_dir = Path(args.run)
|
| 149 |
+
model, cfg, _ = _load_model(run_dir, args.checkpoint)
|
| 150 |
+
device = _device(args.device)
|
| 151 |
+
model.to(device)
|
| 152 |
+
dataset = open_dataset(cfg, "val")
|
| 153 |
+
bpt = load_tokenizer(cfg).bytes_per_token
|
| 154 |
+
amp = cfg["training"]["amp"]
|
| 155 |
+
|
| 156 |
+
base = evaluate(model, dataset, args.batch_size, device, args.batches, amp)
|
| 157 |
+
base_bpb = bits_per_byte(base["val_loss"], bpt)
|
| 158 |
+
print(f"[fp32] loss {base['val_loss']:.4f} | bpb {base_bpb:.4f}")
|
| 159 |
+
|
| 160 |
+
src = run_dir / args.checkpoint
|
| 161 |
+
if not src.exists():
|
| 162 |
+
src = run_dir / "last.pt"
|
| 163 |
+
reports = []
|
| 164 |
+
for bits in args.bits:
|
| 165 |
+
destination = run_dir / f"model_int{bits}.pt"
|
| 166 |
+
report = quantize_checkpoint(src, destination, bits=bits)
|
| 167 |
+
quantized_model, _, _ = _load_model(run_dir, destination.name)
|
| 168 |
+
quantized_model.to(device)
|
| 169 |
+
result = evaluate(quantized_model, dataset, args.batch_size, device,
|
| 170 |
+
args.batches, amp)
|
| 171 |
+
result_bpb = bits_per_byte(result["val_loss"], bpt)
|
| 172 |
+
report.update({"val_loss": result["val_loss"], "val_bpb": result_bpb,
|
| 173 |
+
"delta_bpb": result_bpb - base_bpb})
|
| 174 |
+
print(f"[int{bits}] loss {result['val_loss']:.4f} | bpb {result_bpb:.4f} "
|
| 175 |
+
f"| delta {report['delta_bpb']:+.4f} | "
|
| 176 |
+
f"{report['size_after_mb']:.2f} MB ({report['compression']}x)")
|
| 177 |
+
reports.append(report)
|
| 178 |
+
del quantized_model
|
| 179 |
+
if device.type == "cuda":
|
| 180 |
+
torch.cuda.empty_cache()
|
| 181 |
+
|
| 182 |
+
payload = {"checkpoint": str(src), "fp32_loss": base["val_loss"],
|
| 183 |
+
"fp32_bpb": base_bpb, "reports": reports}
|
| 184 |
+
out = run_dir / "quantization.json"
|
| 185 |
+
out.write_text(json.dumps(payload, indent=2), encoding="utf-8")
|
| 186 |
+
print(f"wrote {out}")
|
| 187 |
+
|
| 188 |
+
|
| 189 |
+
# ---------------------------------------------------------------------------
|
| 190 |
+
def cmd_budget(args: argparse.Namespace) -> None:
|
| 191 |
+
"""Print the parameter budget of the E-series (all models with ``--all``)."""
|
| 192 |
+
from .config import EXPERIMENTS, experiment_config
|
| 193 |
+
from .model import KronTransformer
|
| 194 |
+
|
| 195 |
+
limit = int(args.limit)
|
| 196 |
+
if args.all:
|
| 197 |
+
ids = list(EXPERIMENTS)
|
| 198 |
+
else:
|
| 199 |
+
ids = [exp_id for exp_id in EXPERIMENTS if exp_id.startswith("E")]
|
| 200 |
+
print(f"{'E':>4} {'total':>10} {'matrix':>10} {'<=1M':>6} description")
|
| 201 |
+
worst = 0
|
| 202 |
+
for exp_id in ids:
|
| 203 |
+
model = KronTransformer(experiment_config(exp_id))
|
| 204 |
+
worst = max(worst, model.n_params)
|
| 205 |
+
ok = "ok" if model.n_params <= limit else "FAIL"
|
| 206 |
+
print(f"{exp_id:>4} {model.n_params:>10,} {model.n_params_matrix:>10,} "
|
| 207 |
+
f"{ok:>6} {EXPERIMENTS[exp_id]['desc']}")
|
| 208 |
+
print(f"\nmax {worst:,} / {limit:,} ({worst / limit:.1%})")
|
| 209 |
+
if worst > limit:
|
| 210 |
+
raise SystemExit(1)
|
| 211 |
+
|
| 212 |
+
|
| 213 |
+
def build_parser() -> argparse.ArgumentParser:
|
| 214 |
+
p = argparse.ArgumentParser(prog="kronllm")
|
| 215 |
+
sub = p.add_subparsers(dest="command", required=True)
|
| 216 |
+
|
| 217 |
+
tr = sub.add_parser("train", help="train one model / experiment")
|
| 218 |
+
tr.add_argument("--exp", default=None, help="E0..E12")
|
| 219 |
+
tr.add_argument("--config", default=None)
|
| 220 |
+
tr.add_argument("--run-dir", default=None)
|
| 221 |
+
tr.add_argument("--resume", default=None)
|
| 222 |
+
tr.add_argument("--steps", type=int, default=None)
|
| 223 |
+
tr.add_argument("--epochs", type=int, default=None)
|
| 224 |
+
tr.add_argument("--batch-size", type=int, default=None)
|
| 225 |
+
tr.add_argument("--lr", type=float, default=None)
|
| 226 |
+
tr.add_argument("--device", default=None)
|
| 227 |
+
tr.add_argument("--data-dir", default=None)
|
| 228 |
+
tr.add_argument("--no-amp", action="store_true")
|
| 229 |
+
tr.add_argument("--wandb", action="store_true")
|
| 230 |
+
tr.add_argument("--override", action="append", default=[])
|
| 231 |
+
tr.set_defaults(func=cmd_train)
|
| 232 |
+
|
| 233 |
+
ge = sub.add_parser("generate", help="sample from a checkpoint")
|
| 234 |
+
ge.add_argument("--run", required=True)
|
| 235 |
+
ge.add_argument("--checkpoint", default="best.pt")
|
| 236 |
+
ge.add_argument("--prompt", default="Once upon a time")
|
| 237 |
+
ge.add_argument("--max-new", type=int, default=200)
|
| 238 |
+
ge.add_argument("--temperature", type=float, default=1.0)
|
| 239 |
+
ge.add_argument("--top-k", type=int, default=8)
|
| 240 |
+
ge.add_argument("--samples", type=int, default=1)
|
| 241 |
+
ge.add_argument("--seed", type=int, default=None)
|
| 242 |
+
ge.add_argument("--device", default=None)
|
| 243 |
+
ge.set_defaults(func=cmd_generate)
|
| 244 |
+
|
| 245 |
+
ev = sub.add_parser("eval", help="evaluate a checkpoint")
|
| 246 |
+
ev.add_argument("--run", required=True)
|
| 247 |
+
ev.add_argument("--checkpoint", default="best.pt")
|
| 248 |
+
ev.add_argument("--batches", type=int, default=64)
|
| 249 |
+
ev.add_argument("--device", default=None)
|
| 250 |
+
ev.set_defaults(func=cmd_eval)
|
| 251 |
+
|
| 252 |
+
an = sub.add_parser("analyze", help="spectra / heatmaps / samples")
|
| 253 |
+
an.add_argument("--run", required=True)
|
| 254 |
+
an.add_argument("--checkpoint", default="best.pt")
|
| 255 |
+
an.add_argument("--samples", type=int, default=10)
|
| 256 |
+
an.add_argument("--skip-generation", action="store_true")
|
| 257 |
+
an.set_defaults(func=cmd_analyze)
|
| 258 |
+
|
| 259 |
+
qt = sub.add_parser("quantize", help="int8/int4 checkpoint quantization")
|
| 260 |
+
qt.add_argument("--run", required=True)
|
| 261 |
+
qt.add_argument("--checkpoint", default="best.pt")
|
| 262 |
+
qt.add_argument("--bits", type=int, nargs="+", default=[8, 4])
|
| 263 |
+
qt.add_argument("--batches", type=int, default=32)
|
| 264 |
+
qt.add_argument("--batch-size", type=int, default=32)
|
| 265 |
+
qt.add_argument("--device", default=None)
|
| 266 |
+
qt.set_defaults(func=cmd_quantize)
|
| 267 |
+
|
| 268 |
+
bd = sub.add_parser("budget", help="print the parameter budget of E0..E13")
|
| 269 |
+
bd.add_argument("--limit", type=int, default=1_000_000)
|
| 270 |
+
bd.add_argument("--all", action="store_true",
|
| 271 |
+
help="also print Q1-Q3, D1 and S1 (deliberately above 1M)")
|
| 272 |
+
bd.set_defaults(func=cmd_budget)
|
| 273 |
+
return p
|
| 274 |
+
|
| 275 |
+
|
| 276 |
+
def main(argv: Optional[List[str]] = None) -> None:
|
| 277 |
+
args = build_parser().parse_args(argv)
|
| 278 |
+
args.func(args)
|
| 279 |
+
|
| 280 |
+
|
| 281 |
+
if __name__ == "__main__":
|
| 282 |
+
main()
|
kronllm/config.py
ADDED
|
@@ -0,0 +1,735 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""Configuration handling for KronLLM.
|
| 2 |
+
|
| 3 |
+
Single source of truth for defaults is ``DEFAULTS`` below. Experiments are
|
| 4 |
+
expressed as *overrides* (dotted keys) merged on top of the defaults, so that
|
| 5 |
+
every E-run is guaranteed to share the same seed / data / schedule unless it
|
| 6 |
+
explicitly says otherwise.
|
| 7 |
+
"""
|
| 8 |
+
from __future__ import annotations
|
| 9 |
+
|
| 10 |
+
import copy
|
| 11 |
+
import json
|
| 12 |
+
from pathlib import Path
|
| 13 |
+
from typing import Any, Dict, Iterable, Mapping
|
| 14 |
+
|
| 15 |
+
import yaml
|
| 16 |
+
|
| 17 |
+
DEFAULTS: Dict[str, Any] = {
|
| 18 |
+
"project": "KronLLM-v2",
|
| 19 |
+
"seed": 42,
|
| 20 |
+
"data": {
|
| 21 |
+
"dataset": "roneneldan/TinyStories",
|
| 22 |
+
"tokenizer": "byte_level",
|
| 23 |
+
"vocab_size": 256,
|
| 24 |
+
"seq_len": 256,
|
| 25 |
+
"val_split": 0.05,
|
| 26 |
+
"data_dir": "data",
|
| 27 |
+
"eot_id": 255,
|
| 28 |
+
"num_proc": 8,
|
| 29 |
+
"max_stories": 200000,
|
| 30 |
+
"max_val_stories": 10000,
|
| 31 |
+
"force": False,
|
| 32 |
+
},
|
| 33 |
+
"model": {
|
| 34 |
+
"d_model": 128,
|
| 35 |
+
"n_layers": 12,
|
| 36 |
+
# Recursive weight sharing: use `n_unique_blocks` distinct blocks and
|
| 37 |
+
# apply the stack `recurse` times. Effective depth =
|
| 38 |
+
# n_unique_blocks * recurse; parameters are counted once.
|
| 39 |
+
"n_unique_blocks": None,
|
| 40 |
+
"recurse": 1,
|
| 41 |
+
"n_heads": 8,
|
| 42 |
+
"n_kv_heads": None, # None -> MHA
|
| 43 |
+
"rope_theta": 10000.0,
|
| 44 |
+
"norm": "rmsnorm",
|
| 45 |
+
"norm_eps": 1e-6,
|
| 46 |
+
"tie_embeddings": True,
|
| 47 |
+
"ffn_silu": True, # E9 is now the *linear* FFN ablation
|
| 48 |
+
"ffn_gated": False, # E11: SwiGLU-style gated FFN
|
| 49 |
+
# Feature interaction in the FFN: "kron" is the linear composition,
|
| 50 |
+
# "glu" is the bilinear (W_a x) * (W_b x) term (Q9/Q9k).
|
| 51 |
+
"ffn_mix": "kron",
|
| 52 |
+
"ffn_mix_rank": 16,
|
| 53 |
+
"ffn_mix_kron": False,
|
| 54 |
+
"use_kron_qkv": True,
|
| 55 |
+
"use_kron_ffn": True,
|
| 56 |
+
"dense_ffn_mult": 1.0, # only used when use_kron_ffn == False
|
| 57 |
+
# ALBERT-style factorised (tied) embedding: V x r then r x d.
|
| 58 |
+
# 0 -> plain V x d table. Keeps the embedding ~12% of the budget.
|
| 59 |
+
"embed_rank": 0,
|
| 60 |
+
# "auto" -> factorised when embed_rank > 0, dense otherwise.
|
| 61 |
+
# "routed" -> top-k mixture of a dense (v_eff, d) bank (Q3).
|
| 62 |
+
# "kron" -> the V x r table is a sum of Kronecker products
|
| 63 |
+
# (embed_kron); lookup and the tied head are unchanged.
|
| 64 |
+
"embed_mode": "auto",
|
| 65 |
+
"embed_kron": {
|
| 66 |
+
# E = sum_k A_k (x) B_k, V = v1*v2, r = r1*r2.
|
| 67 |
+
# Cost: K*(v1*r1 + v2*r2) instead of V*r.
|
| 68 |
+
"v1": 64, "v2": 64,
|
| 69 |
+
"r1": 16, "r2": 8,
|
| 70 |
+
"K": 32,
|
| 71 |
+
},
|
| 72 |
+
"routed": {
|
| 73 |
+
"v_eff": 5000,
|
| 74 |
+
"route_dim": 6,
|
| 75 |
+
"top_k": 10,
|
| 76 |
+
"temperature": 1.0,
|
| 77 |
+
},
|
| 78 |
+
# Intra-layer RMSNorm on q/k per head: removes logit drift and lets the
|
| 79 |
+
# model use a higher lr at this scale.
|
| 80 |
+
"qk_norm": False,
|
| 81 |
+
# SDPA kernel: "auto" (best fused kernel), or "flash",
|
| 82 |
+
# "mem_efficient", "math" to pin a backend for benchmarking. Only the
|
| 83 |
+
# attention kernel changes; the Kronecker maps are untouched.
|
| 84 |
+
"attn_impl": "auto",
|
| 85 |
+
# Fuse all composes of the model into a handful of batched GEMMs.
|
| 86 |
+
# See kronllm/kron.py:KronBank for the launch-count rationale.
|
| 87 |
+
"batched_prelude": True,
|
| 88 |
+
"kron": {
|
| 89 |
+
"L": 4,
|
| 90 |
+
"K": 2,
|
| 91 |
+
"K_win": 2,
|
| 92 |
+
"n1": 8,
|
| 93 |
+
"n2": 16,
|
| 94 |
+
"stop_gradient": True, # E8 flips this
|
| 95 |
+
"composition": "product", # E14 uses "sum": sI + sum_l alpha_l D_l
|
| 96 |
+
"sum_alpha_init": 1.0,
|
| 97 |
+
# E15/E16: input-dependent softmax weights over the L sum terms.
|
| 98 |
+
# None (static alpha) | "dense" (d x L linear) | "kron" (Kron gate).
|
| 99 |
+
"dynamic_gate": None,
|
| 100 |
+
# E17: grouped composition -- sum inside `groups` blocks, product
|
| 101 |
+
# across blocks. G=1 is the sum, G=L is the product.
|
| 102 |
+
"groups": 2,
|
| 103 |
+
# E18: alternating composition -- one operation per micro-depth
|
| 104 |
+
# step: sum, product, sum, product... starting from `sI`.
|
| 105 |
+
"alternate_start": "sum",
|
| 106 |
+
# E23/E24: elementwise nonlinearity between composition steps.
|
| 107 |
+
"nonlinearity": "silu",
|
| 108 |
+
"norm_between": False,
|
| 109 |
+
"nonlinear_every": 1,
|
| 110 |
+
"nonlinear_mode": "replace",
|
| 111 |
+
"anchor": True, # E4 turns the s*I anchor off
|
| 112 |
+
# Per-role anchor, interpreted as the *effective* (post L-fold)
|
| 113 |
+
# value: s = s_init ** (1/L). See README "Deviations".
|
| 114 |
+
"s_init": {"q": 1.0, "k": 1.0, "v": 1.0, "o": 1.0, "ffn": 1.0},
|
| 115 |
+
"s_init_effective": True,
|
| 116 |
+
"s_learnable": False, # E3 turns this on
|
| 117 |
+
# Orthogonal factor gain. With normalize_D the gain is normalised
|
| 118 |
+
# away in the forward pass, so it only matters for the ablation
|
| 119 |
+
# table (E12). "spec" is the literal TZ recipe 1/sqrt(d).
|
| 120 |
+
"init_gain": "unit",
|
| 121 |
+
# RMS-normalise D before it enters the anchor and the recurrence.
|
| 122 |
+
# This is what makes the innovation *visible* to the gradient:
|
| 123 |
+
# without it ||I|| ~ 1/d and the model is numerically the identity.
|
| 124 |
+
"normalize_D": True,
|
| 125 |
+
"innovation_scale": 0.1, # tau init: RMS of every D_l
|
| 126 |
+
"gate_init": 0.1, # ReZero gamma init
|
| 127 |
+
"gate": True, # E12 keeps tau but drops the normaliser
|
| 128 |
+
"theta_std": 0.1,
|
| 129 |
+
# Depth-recurrence strength. "learned" = sigmoid(theta) (default);
|
| 130 |
+
# "fixed" pins it to sigma_value and freezes theta; "off" removes
|
| 131 |
+
# the carry, leaving independent factors. Ablations R1/R2.
|
| 132 |
+
"sigma_mode": "learned",
|
| 133 |
+
"sigma_value": 0.5,
|
| 134 |
+
"rezero_eps": 0.0,
|
| 135 |
+
},
|
| 136 |
+
},
|
| 137 |
+
"training": {
|
| 138 |
+
"batch_size": 128,
|
| 139 |
+
"seq_len": 256,
|
| 140 |
+
"grad_accum": 1,
|
| 141 |
+
"epochs": 3,
|
| 142 |
+
"optimizer": "adamw",
|
| 143 |
+
"betas": [0.9, 0.95],
|
| 144 |
+
"lr": 3e-4,
|
| 145 |
+
"lr_s": 3e-3,
|
| 146 |
+
# theta is 4 scalars per module; its gradient is ~400x smaller than the
|
| 147 |
+
# factor gradients because D is RMS-normalised (a scale-free mixing
|
| 148 |
+
# coefficient). 10x the old value gives it a chance to move at all --
|
| 149 |
+
# H3 ("fast/slow channels") is otherwise not even testable.
|
| 150 |
+
"lr_theta": 1e-2,
|
| 151 |
+
"warmup_steps": 500,
|
| 152 |
+
"schedule": "cosine",
|
| 153 |
+
"min_lr": 3e-5,
|
| 154 |
+
"weight_decay": 0.1,
|
| 155 |
+
"grad_clip": 1.0,
|
| 156 |
+
"amp": "none", # fp16 autocast is slower than fp32 on the reference GPU
|
| 157 |
+
"log_every": 100,
|
| 158 |
+
"eval_every": 500,
|
| 159 |
+
# Periodic checkpoint for crash / GPU-quota resilience. ``last.pt``
|
| 160 |
+
# is rewritten every ``save_every`` steps; ``best.pt`` follows the
|
| 161 |
+
# validation curve. `kronllm.cli train --resume runs/<run>/last.pt`
|
| 162 |
+
# continues from the step counter stored in the file.
|
| 163 |
+
"save_every": 1000,
|
| 164 |
+
"eval_batches": 64,
|
| 165 |
+
"early_stop_patience": 4,
|
| 166 |
+
# Recursive weight sharing: sample r ~ U{1..recurse} per training step
|
| 167 |
+
# so one model works at several depths (adaptive compute).
|
| 168 |
+
"recurse_random": False,
|
| 169 |
+
"num_workers": 0,
|
| 170 |
+
"device": "cuda",
|
| 171 |
+
"compile": False,
|
| 172 |
+
"out_dir": "runs",
|
| 173 |
+
},
|
| 174 |
+
"metrics": {
|
| 175 |
+
"primary": ["val_ppl", "val_loss"],
|
| 176 |
+
"diagnostic": [
|
| 177 |
+
"sigma_theta_mean",
|
| 178 |
+
"sigma_theta_var",
|
| 179 |
+
"D_norm",
|
| 180 |
+
"D_rms",
|
| 181 |
+
"tau_mean",
|
| 182 |
+
"gate_mean",
|
| 183 |
+
"h_norm_ratio",
|
| 184 |
+
"sigma_min_W",
|
| 185 |
+
"eff_rank_W",
|
| 186 |
+
"eff_rank_D",
|
| 187 |
+
"rank_excess",
|
| 188 |
+
"attn_entropy",
|
| 189 |
+
"grad_norm",
|
| 190 |
+
],
|
| 191 |
+
"comparison": [
|
| 192 |
+
"n_params",
|
| 193 |
+
"n_params_matrix",
|
| 194 |
+
"flops_per_token",
|
| 195 |
+
"time_per_epoch",
|
| 196 |
+
"peak_vram",
|
| 197 |
+
],
|
| 198 |
+
"spectral_every": 2000,
|
| 199 |
+
},
|
| 200 |
+
"logging": {"tensorboard": True, "wandb": False, "wandb_project": "kronllm"},
|
| 201 |
+
}
|
| 202 |
+
|
| 203 |
+
# E-series from section 6.3 of the spec. ``overrides`` are dotted paths.
|
| 204 |
+
# NOTE: SiLU on the FFN output is part of the *base* model now (it was E9 in
|
| 205 |
+
# the TZ), so E9 tests the opposite direction -- a purely linear FFN. E11/E12
|
| 206 |
+
# are additions motivated by the diagnostics of the first training runs; see
|
| 207 |
+
# README "Deviations" for the full table.
|
| 208 |
+
EXPERIMENTS: Dict[str, Dict[str, Any]] = {
|
| 209 |
+
"E0": {
|
| 210 |
+
"desc": "Dense baseline (MHA + dense FFN, ~0.97M params)",
|
| 211 |
+
"overrides": {"model.use_kron_qkv": False, "model.use_kron_ffn": False,
|
| 212 |
+
# dense FFN hidden = 48 keeps the *control* under the
|
| 213 |
+
# project-wide 1M ceiling (it is 5x the Kron budget)
|
| 214 |
+
"model.dense_ffn_mult": 0.375},
|
| 215 |
+
},
|
| 216 |
+
"E1": {
|
| 217 |
+
"desc": "Dense MHA + Kron FFN",
|
| 218 |
+
"overrides": {"model.use_kron_qkv": False, "model.use_kron_ffn": True},
|
| 219 |
+
},
|
| 220 |
+
"E2": {
|
| 221 |
+
"desc": "Full Kron (base)",
|
| 222 |
+
"overrides": {},
|
| 223 |
+
},
|
| 224 |
+
"E3": {
|
| 225 |
+
"desc": "E2 + learnable anchor s",
|
| 226 |
+
"overrides": {"model.kron.s_learnable": True},
|
| 227 |
+
},
|
| 228 |
+
"E4": {
|
| 229 |
+
"desc": "E2 without anchor, K_qkv=8 (pure innovation product)",
|
| 230 |
+
"overrides": {
|
| 231 |
+
"model.kron.anchor": False,
|
| 232 |
+
"model.kron.K_qkv": 8,
|
| 233 |
+
# without the identity carrier the product of L innovations has
|
| 234 |
+
# norm ~ tau**L, so tau must be O(1) for the branch to be alive
|
| 235 |
+
"model.kron.innovation_scale": 1.0,
|
| 236 |
+
},
|
| 237 |
+
},
|
| 238 |
+
"E5": {
|
| 239 |
+
"desc": "E2 + wider/deeper (d=192, N=16)",
|
| 240 |
+
"overrides": {
|
| 241 |
+
"model.d_model": 192,
|
| 242 |
+
"model.n_layers": 16,
|
| 243 |
+
"model.kron.n1": 12,
|
| 244 |
+
"model.kron.n2": 16,
|
| 245 |
+
},
|
| 246 |
+
},
|
| 247 |
+
"E6": {
|
| 248 |
+
"desc": "E2 + L=6",
|
| 249 |
+
"overrides": {"model.kron.L": 6},
|
| 250 |
+
},
|
| 251 |
+
"E7": {
|
| 252 |
+
"desc": "E2 + K=4",
|
| 253 |
+
"overrides": {"model.kron.K": 4},
|
| 254 |
+
},
|
| 255 |
+
"E8": {
|
| 256 |
+
"desc": "E2 + full backprop (no stop-grad)",
|
| 257 |
+
"overrides": {"model.kron.stop_gradient": False},
|
| 258 |
+
},
|
| 259 |
+
"E9": {
|
| 260 |
+
"desc": "E2 with a linear FFN (SiLU ablation)",
|
| 261 |
+
"overrides": {"model.ffn_silu": False},
|
| 262 |
+
},
|
| 263 |
+
"E10": {
|
| 264 |
+
"desc": "E2 + GQA (n_kv=2)",
|
| 265 |
+
"overrides": {"model.n_kv_heads": 2},
|
| 266 |
+
},
|
| 267 |
+
"E11": {
|
| 268 |
+
"desc": "E2 + gated FFN: (W_a x) * silu(W_b x)",
|
| 269 |
+
"overrides": {"model.ffn_gated": True},
|
| 270 |
+
},
|
| 271 |
+
"E12": {
|
| 272 |
+
"desc": "E2 without D normalisation (raw innovation scale)",
|
| 273 |
+
"overrides": {
|
| 274 |
+
"model.kron.normalize_D": False,
|
| 275 |
+
"model.kron.innovation_scale": 1.0,
|
| 276 |
+
},
|
| 277 |
+
},
|
| 278 |
+
"E13": {
|
| 279 |
+
"desc": "E2 + K=8 (budget-scaling point, ~651K params)",
|
| 280 |
+
"overrides": {"model.kron.K": 8},
|
| 281 |
+
},
|
| 282 |
+
"E14": {
|
| 283 |
+
"desc": "E2 + sum composition (sI + sum_l alpha_l D_l), full backprop",
|
| 284 |
+
"overrides": {
|
| 285 |
+
"model.kron.composition": "sum",
|
| 286 |
+
"model.kron.stop_gradient": False,
|
| 287 |
+
},
|
| 288 |
+
},
|
| 289 |
+
# E15/E16: the same sum, but the channel weights are generated from the
|
| 290 |
+
# input by a softmax gate instead of being learned constants. "dense" uses
|
| 291 |
+
# a d x L linear layer; "kron" uses a factorised gate whose output is a
|
| 292 |
+
# g1 x g2 Kronecker grid (g1*g2 == L), matching the project's style.
|
| 293 |
+
"E15": {
|
| 294 |
+
"desc": "E14 + input-dependent softmax gate (dense d x L)",
|
| 295 |
+
"overrides": {
|
| 296 |
+
"model.kron.composition": "sum",
|
| 297 |
+
"model.kron.stop_gradient": False,
|
| 298 |
+
"model.kron.dynamic_gate": "dense",
|
| 299 |
+
},
|
| 300 |
+
},
|
| 301 |
+
"E16": {
|
| 302 |
+
"desc": "E14 + input-dependent softmax gate (Kron g1 x g2)",
|
| 303 |
+
"overrides": {
|
| 304 |
+
"model.kron.composition": "sum",
|
| 305 |
+
"model.kron.stop_gradient": False,
|
| 306 |
+
"model.kron.dynamic_gate": "kron",
|
| 307 |
+
},
|
| 308 |
+
},
|
| 309 |
+
# ------------------------------------------------------------------
|
| 310 |
+
# Q-series: quality first. Same Kron skeleton, but the budget is spent
|
| 311 |
+
# where it pays off: a 3-bytes/token BPE vocabulary, more depth/width and
|
| 312 |
+
# more Kronecker terms per matrix, with a factorised embedding so the
|
| 313 |
+
# lookup table does not swallow the budget.
|
| 314 |
+
# ------------------------------------------------------------------
|
| 315 |
+
"Q1": {
|
| 316 |
+
"desc": "BPE-1024, d=192/N=20, K=5, qk-norm (~926K params)",
|
| 317 |
+
"overrides": {
|
| 318 |
+
"data.tokenizer": "bpe",
|
| 319 |
+
"data.vocab_size": 1024,
|
| 320 |
+
"data.eot_id": 1023,
|
| 321 |
+
"data.seq_len": 192,
|
| 322 |
+
"model.d_model": 192,
|
| 323 |
+
"model.n_layers": 20,
|
| 324 |
+
"model.n_heads": 8,
|
| 325 |
+
"model.embed_rank": 96,
|
| 326 |
+
"model.qk_norm": True,
|
| 327 |
+
"model.kron.n1": 12,
|
| 328 |
+
"model.kron.n2": 16,
|
| 329 |
+
"model.kron.K": 5,
|
| 330 |
+
"training.seq_len": 192,
|
| 331 |
+
# batch 96 is too large for the reference GPU at d=192/N=20;
|
| 332 |
+
# 64 x 192 = 12.3k tokens per step.
|
| 333 |
+
"training.batch_size": 64,
|
| 334 |
+
"training.lr": 5e-4,
|
| 335 |
+
"training.min_lr": 5e-5,
|
| 336 |
+
"training.warmup_steps": 300,
|
| 337 |
+
"training.eval_every": 500,
|
| 338 |
+
"training.early_stop_patience": 99,
|
| 339 |
+
# 10000 x 64 x 192 = 123M tokens ~ 2.1 epochs of the BPE corpus
|
| 340 |
+
"training.max_steps": 10000,
|
| 341 |
+
},
|
| 342 |
+
},
|
| 343 |
+
# ------------------------------------------------------------------
|
| 344 |
+
# Controls: is the Kron parameterisation actually *worth* it at equal
|
| 345 |
+
# parameters? D1 is E0's dense recipe on the Q tokenizer, so the pair
|
| 346 |
+
# D1 vs Q1 differs in exactly one thing: dense matrices vs Kron ones.
|
| 347 |
+
# ------------------------------------------------------------------
|
| 348 |
+
"D1": {
|
| 349 |
+
"desc": "Dense+BPE control: E0 recipe, 1024-token BPE (~1.07M)",
|
| 350 |
+
"overrides": {
|
| 351 |
+
"data.tokenizer": "bpe",
|
| 352 |
+
"data.vocab_size": 1024,
|
| 353 |
+
"data.eot_id": 1023,
|
| 354 |
+
"data.seq_len": 192,
|
| 355 |
+
"model.use_kron_qkv": False,
|
| 356 |
+
"model.use_kron_ffn": False,
|
| 357 |
+
"model.dense_ffn_mult": 0.375,
|
| 358 |
+
"model.qk_norm": True,
|
| 359 |
+
"training.seq_len": 192,
|
| 360 |
+
"training.batch_size": 64,
|
| 361 |
+
"training.lr": 5e-4,
|
| 362 |
+
"training.min_lr": 5e-5,
|
| 363 |
+
"training.warmup_steps": 300,
|
| 364 |
+
"training.eval_every": 500,
|
| 365 |
+
"training.early_stop_patience": 99,
|
| 366 |
+
# Exactly Q1's token budget: 4200 x 64 x 192 = 51.6M tokens.
|
| 367 |
+
"training.max_steps": 4200,
|
| 368 |
+
},
|
| 369 |
+
},
|
| 370 |
+
# Q1 was stopped at step ~4200 (val bpb 0.976) to spend the remaining time
|
| 371 |
+
# on a 2.2x larger model; the design goal is quality, not the 1M budget.
|
| 372 |
+
"Q2": {
|
| 373 |
+
"desc": "BPE-1024, d=320/N=24, K=6, qk-norm (~2.08M params)",
|
| 374 |
+
"overrides": {
|
| 375 |
+
"data.tokenizer": "bpe",
|
| 376 |
+
"data.vocab_size": 1024,
|
| 377 |
+
"data.eot_id": 1023,
|
| 378 |
+
"data.seq_len": 192,
|
| 379 |
+
"model.d_model": 320,
|
| 380 |
+
"model.n_layers": 24,
|
| 381 |
+
"model.n_heads": 8, # d_head = 40
|
| 382 |
+
"model.embed_rank": 128,
|
| 383 |
+
"model.qk_norm": True,
|
| 384 |
+
"model.kron.n1": 16,
|
| 385 |
+
"model.kron.n2": 20, # n1 * n2 == d_model == 320
|
| 386 |
+
"model.kron.K": 6,
|
| 387 |
+
"training.seq_len": 192,
|
| 388 |
+
# N*d*bs is held at Q1's level (24*320*32 == 20*192*64), so the
|
| 389 |
+
# activation footprint stays the same as Q1's.
|
| 390 |
+
"training.batch_size": 32,
|
| 391 |
+
"training.lr": 4e-4,
|
| 392 |
+
"training.min_lr": 4e-5,
|
| 393 |
+
"training.warmup_steps": 300,
|
| 394 |
+
"training.eval_every": 500,
|
| 395 |
+
"training.early_stop_patience": 99,
|
| 396 |
+
# 10000 x 32 x 192 = 61M tokens ~ 1 epoch of the BPE corpus
|
| 397 |
+
"training.max_steps": 10000,
|
| 398 |
+
},
|
| 399 |
+
},
|
| 400 |
+
}
|
| 401 |
+
|
| 402 |
+
# Q3: dense routed embeddings on top of Q2. Each token has a 6-d routing code;
|
| 403 |
+
# a linear router selects the top-10 rows of a 5000 x 320 bank and mixes them
|
| 404 |
+
# with softmax weights. The effective V x d table is reused by the tied head,
|
| 405 |
+
# so the head stays parameter-free. ~3.54M parameters.
|
| 406 |
+
EXPERIMENTS["Q3"] = {
|
| 407 |
+
"desc": "Q2 + routed dense embeddings (v_eff=5000, top-k=10, ~3.54M params)",
|
| 408 |
+
"overrides": {
|
| 409 |
+
**EXPERIMENTS["Q2"]["overrides"],
|
| 410 |
+
"model.embed_mode": "routed",
|
| 411 |
+
"model.embed_rank": 0,
|
| 412 |
+
"model.routed.v_eff": 5000,
|
| 413 |
+
"model.routed.route_dim": 6,
|
| 414 |
+
"model.routed.top_k": 10,
|
| 415 |
+
"model.routed.temperature": 1.0,
|
| 416 |
+
"training.lr": 3e-4,
|
| 417 |
+
"training.min_lr": 3e-5,
|
| 418 |
+
},
|
| 419 |
+
}
|
| 420 |
+
|
| 421 |
+
# Supervised fine-tuning on top of Q2. The corpus is written by
|
| 422 |
+
# ``scripts/prepare_sft.py`` from Dolly-15k; only the model weights carry over,
|
| 423 |
+
# so the run gets a fresh optimizer and its own short cosine schedule.
|
| 424 |
+
EXPERIMENTS["S1"] = {
|
| 425 |
+
"desc": "Q2 + supervised fine-tuning on Dolly-15k Q/A (~2.08M params)",
|
| 426 |
+
"overrides": {
|
| 427 |
+
**EXPERIMENTS["Q2"]["overrides"],
|
| 428 |
+
"data.train_bin": "sft_train.bpe.bin",
|
| 429 |
+
"data.val_bin": "sft_val.bpe.bin",
|
| 430 |
+
"training.init_from": "runs/Q2/best.pt",
|
| 431 |
+
"training.max_steps": 400,
|
| 432 |
+
"training.lr": 1e-4,
|
| 433 |
+
"training.min_lr": 1e-5,
|
| 434 |
+
"training.warmup_steps": 40,
|
| 435 |
+
"training.eval_every": 50,
|
| 436 |
+
"training.early_stop_patience": 12,
|
| 437 |
+
},
|
| 438 |
+
}
|
| 439 |
+
|
| 440 |
+
# Q4: test the E8 win at quality scale. Q2 plus full backprop through the
|
| 441 |
+
# depth recurrence; same parameter count, tokenizer and activation footprint.
|
| 442 |
+
EXPERIMENTS["Q4"] = {
|
| 443 |
+
"desc": "Q2 + full backprop (no stop-grad, ~2.08M params)",
|
| 444 |
+
"overrides": {
|
| 445 |
+
**EXPERIMENTS["Q2"]["overrides"],
|
| 446 |
+
"model.kron.stop_gradient": False,
|
| 447 |
+
},
|
| 448 |
+
}
|
| 449 |
+
|
| 450 |
+
# Q5: scale toward the 10M ceiling without increasing the activation
|
| 451 |
+
# footprint. K=24 raises the matrix budget from 1.89M to 7.56M parameters
|
| 452 |
+
# while N*d*batch*seq stays at Q2's level, so it still targets a 6 GB card.
|
| 453 |
+
EXPERIMENTS["Q5"] = {
|
| 454 |
+
"desc": "Q4 + K=24 (~7.75M params), same activation footprint",
|
| 455 |
+
"overrides": {
|
| 456 |
+
**EXPERIMENTS["Q4"]["overrides"],
|
| 457 |
+
"model.kron.K": 24,
|
| 458 |
+
# Same lr schedule as Q4: the first attempt used 3e-4/2k steps and is
|
| 459 |
+
# not a clean K ablation.
|
| 460 |
+
"training.max_steps": 4000,
|
| 461 |
+
"training.eval_every": 500,
|
| 462 |
+
},
|
| 463 |
+
}
|
| 464 |
+
|
| 465 |
+
# Q6: a wider tokenizer on top of Q4. BPE-1024 gives ~3.07 bytes/token; a
|
| 466 |
+
# 4096-token vocabulary raises the compression rate and the generation
|
| 467 |
+
# bytes/s. The embedding table is factorised (V x rank), so the extra
|
| 468 |
+
# vocabulary costs only ~393k parameters. Artifacts are prepared with
|
| 469 |
+
# `python scripts/prepare_bpe.py --vocab 4096 --suffix 4096` and live next to
|
| 470 |
+
# the shipped BPE-1024 files.
|
| 471 |
+
EXPERIMENTS["Q6"] = {
|
| 472 |
+
"desc": "Q4 + BPE-4096 tokenizer (~2.47M params)",
|
| 473 |
+
"overrides": {
|
| 474 |
+
**EXPERIMENTS["Q4"]["overrides"],
|
| 475 |
+
"data.vocab_size": 4096,
|
| 476 |
+
"data.eot_id": 4095,
|
| 477 |
+
"data.train_bin": "train.bpe4096.bin",
|
| 478 |
+
"data.val_bin": "val.bpe4096.bin",
|
| 479 |
+
"data.meta_file": "tokenizer_meta4096.json",
|
| 480 |
+
"model.loss_chunk": 1024,
|
| 481 |
+
"training.max_steps": 4000,
|
| 482 |
+
},
|
| 483 |
+
}
|
| 484 |
+
|
| 485 |
+
# Q7: Kronecker-structured embedding on top of Q6. The V x r table becomes
|
| 486 |
+
# E = sum_k A_k (x) B_k with V = v1*v2 and r = r1*r2, so the embedding budget
|
| 487 |
+
# scales as K*(v1*r1 + v2*r2) instead of V*r: 49,152 parameters instead of
|
| 488 |
+
# 524,288 at the same rank ceiling (r1*r2 = r). Lookup and the tied head are
|
| 489 |
+
# unchanged; only the table storage is compressed.
|
| 490 |
+
EXPERIMENTS["Q7"] = {
|
| 491 |
+
"desc": "Q6 + kron embedding (49K vs 524K embed params)",
|
| 492 |
+
"overrides": {
|
| 493 |
+
**EXPERIMENTS["Q6"]["overrides"],
|
| 494 |
+
"model.embed_mode": "kron",
|
| 495 |
+
"model.embed_kron": {"v1": 64, "v2": 64, "r1": 16, "r2": 8, "K": 32},
|
| 496 |
+
},
|
| 497 |
+
}
|
| 498 |
+
|
| 499 |
+
# Wikitext probes: the product-vs-sum question on a harder corpus than
|
| 500 |
+
# TinyStories, at the E2/E8 geometry (byte level, seq 256, batch 128).
|
| 501 |
+
EXPERIMENTS["E8w"] = {
|
| 502 |
+
"desc": "E8 (full backprop) on WikiText-103 bytes",
|
| 503 |
+
"overrides": {**EXPERIMENTS["E8"]["overrides"],
|
| 504 |
+
"data.data_dir": "data/wikitext"},
|
| 505 |
+
}
|
| 506 |
+
EXPERIMENTS["E14w"] = {
|
| 507 |
+
"desc": "E14 (sum composition) on WikiText-103 bytes",
|
| 508 |
+
"overrides": {**EXPERIMENTS["E14"]["overrides"],
|
| 509 |
+
"data.data_dir": "data/wikitext"},
|
| 510 |
+
}
|
| 511 |
+
EXPERIMENTS["E17w"] = {
|
| 512 |
+
"desc": "E8 + grouped composition (G=2) on WikiText-103 bytes",
|
| 513 |
+
"overrides": {**EXPERIMENTS["E8"]["overrides"],
|
| 514 |
+
"data.data_dir": "data/wikitext",
|
| 515 |
+
"model.kron.composition": "grouped",
|
| 516 |
+
"model.kron.groups": 2},
|
| 517 |
+
}
|
| 518 |
+
EXPERIMENTS["E18w"] = {
|
| 519 |
+
"desc": "E8 + alternating sum/product composition on WikiText-103 bytes",
|
| 520 |
+
"overrides": {**EXPERIMENTS["E8"]["overrides"],
|
| 521 |
+
"data.data_dir": "data/wikitext",
|
| 522 |
+
"model.kron.composition": "alternating",
|
| 523 |
+
"model.kron.alternate_start": "sum"},
|
| 524 |
+
}
|
| 525 |
+
|
| 526 |
+
# E19/E20: recursive weight sharing (Universal-Transformer style). E19 keeps
|
| 527 |
+
# the effective depth at 12 with 6 unique blocks (half the parameters, same
|
| 528 |
+
# FLOPs); E20 keeps E2's parameter count and doubles the effective depth.
|
| 529 |
+
EXPERIMENTS["E19"] = {
|
| 530 |
+
"desc": "E2 + weight sharing: 6 unique blocks x2 (12 effective, ~half params)",
|
| 531 |
+
"overrides": {"model.n_unique_blocks": 6, "model.recurse": 2},
|
| 532 |
+
}
|
| 533 |
+
EXPERIMENTS["E20"] = {
|
| 534 |
+
"desc": "E2 + weight sharing: 12 unique blocks x2 (24 effective, same params)",
|
| 535 |
+
"overrides": {"model.n_unique_blocks": 12, "model.recurse": 2,
|
| 536 |
+
"training.batch_size": 64},
|
| 537 |
+
}
|
| 538 |
+
EXPERIMENTS["E21"] = {
|
| 539 |
+
"desc": "E19 + randomized recursion r~U{1,2} + pass embedding",
|
| 540 |
+
"overrides": {**EXPERIMENTS["E19"]["overrides"],
|
| 541 |
+
"training.recurse_random": True,
|
| 542 |
+
"model.pass_embedding": True},
|
| 543 |
+
}
|
| 544 |
+
EXPERIMENTS["E22"] = {
|
| 545 |
+
"desc": "6 unique blocks x4 (24 effective, ~111K params)",
|
| 546 |
+
"overrides": {"model.n_unique_blocks": 6, "model.recurse": 4,
|
| 547 |
+
"training.batch_size": 64},
|
| 548 |
+
}
|
| 549 |
+
|
| 550 |
+
# E23/E24: nonlinear composition at the E2 geometry.
|
| 551 |
+
# CLOSED (2026-10-24): every variant lost to the linear product
|
| 552 |
+
# (E25 1.519, E26 1.568, E26b 1.591 vs E8 1.479). Kept for
|
| 553 |
+
# reproducibility only; do not schedule new runs.
|
| 554 |
+
EXPERIMENTS["E23"] = {
|
| 555 |
+
"desc": "E2 + SiLU between composition steps",
|
| 556 |
+
"overrides": {"model.kron.composition": "nonlinear"},
|
| 557 |
+
}
|
| 558 |
+
EXPERIMENTS["E24"] = {
|
| 559 |
+
"desc": "E23 + RMSNorm between composition steps",
|
| 560 |
+
"overrides": {"model.kron.composition": "nonlinear",
|
| 561 |
+
"model.kron.norm_between": True},
|
| 562 |
+
}
|
| 563 |
+
EXPERIMENTS["E25"] = {
|
| 564 |
+
"desc": "nonlinear composition every 2 factors (2 fused GEMMs/token)",
|
| 565 |
+
"overrides": {"model.kron.composition": "nonlinear",
|
| 566 |
+
"model.kron.nonlinear_every": 2},
|
| 567 |
+
}
|
| 568 |
+
EXPERIMENTS["E26"] = {
|
| 569 |
+
"desc": "nonlinearity after the whole composition (1 GEMM, almost free)",
|
| 570 |
+
"overrides": {"model.kron.composition": "nonlinear",
|
| 571 |
+
"model.kron.nonlinear_every": 4},
|
| 572 |
+
}
|
| 573 |
+
EXPERIMENTS["E26b"] = {
|
| 574 |
+
"desc": "E26 but additive: keeps the linear product skeleton",
|
| 575 |
+
"overrides": {"model.kron.composition": "nonlinear",
|
| 576 |
+
"model.kron.nonlinear_every": 4,
|
| 577 |
+
"model.kron.nonlinear_mode": "add"},
|
| 578 |
+
}
|
| 579 |
+
EXPERIMENTS["E25b"] = {
|
| 580 |
+
"desc": "E25 (k=2) but additive: linear skeleton + 2 nonlinear branches",
|
| 581 |
+
"overrides": {"model.kron.composition": "nonlinear",
|
| 582 |
+
"model.kron.nonlinear_every": 2,
|
| 583 |
+
"model.kron.nonlinear_mode": "add"},
|
| 584 |
+
}
|
| 585 |
+
# E27-E30: depth-recurrence ablation (2026-10-25).
|
| 586 |
+
# Motivated by an external review: with K_win=2 the state only crosses pairs of
|
| 587 |
+
# layers, sigma(theta) is one scalar per module, and no run isolated the carry
|
| 588 |
+
# itself. All four share the E8 geometry and parameter count (theta stays in
|
| 589 |
+
# the state dict, so only 4 scalars per module are inert).
|
| 590 |
+
# E8r is the control: E8 re-run after the sigma_mode plumbing, so the four
|
| 591 |
+
# ablations are compared against a number produced by the same code path.
|
| 592 |
+
EXPERIMENTS["E8r"] = {
|
| 593 |
+
"desc": "E8 re-run as the control for the E27-E30 sigma/K_win ablation",
|
| 594 |
+
"overrides": {**EXPERIMENTS["E8"]["overrides"]},
|
| 595 |
+
}
|
| 596 |
+
EXPERIMENTS["E27"] = {
|
| 597 |
+
"desc": "E8 without the depth carry: independent factors (sigma = 0)",
|
| 598 |
+
"overrides": {**EXPERIMENTS["E8"]["overrides"],
|
| 599 |
+
"model.kron.sigma_mode": "off"},
|
| 600 |
+
}
|
| 601 |
+
EXPERIMENTS["E28"] = {
|
| 602 |
+
"desc": "E8 with sigma frozen at 0.5 (the reviewer's ablation)",
|
| 603 |
+
"overrides": {**EXPERIMENTS["E8"]["overrides"],
|
| 604 |
+
"model.kron.sigma_mode": "fixed",
|
| 605 |
+
"model.kron.sigma_value": 0.5},
|
| 606 |
+
}
|
| 607 |
+
EXPERIMENTS["E29"] = {
|
| 608 |
+
"desc": "E8 with K_win=4: state crosses four layers instead of two",
|
| 609 |
+
"overrides": {**EXPERIMENTS["E8"]["overrides"],
|
| 610 |
+
"model.kron.K_win": 4},
|
| 611 |
+
}
|
| 612 |
+
EXPERIMENTS["E30"] = {
|
| 613 |
+
"desc": "E8 with K_win=0: state crosses the whole 12-layer stack",
|
| 614 |
+
"overrides": {**EXPERIMENTS["E8"]["overrides"],
|
| 615 |
+
"model.kron.K_win": 0},
|
| 616 |
+
}
|
| 617 |
+
# Q8g: gated FFN at quality scale (Q6 + SwiGLU).
|
| 618 |
+
EXPERIMENTS["Q8g"] = {
|
| 619 |
+
"desc": "Q6 + gated FFN (SwiGLU) (~2.85M params, batch 24)",
|
| 620 |
+
"overrides": {**EXPERIMENTS["Q6"]["overrides"],
|
| 621 |
+
"model.ffn_gated": True,
|
| 622 |
+
"training.batch_size": 24},
|
| 623 |
+
}
|
| 624 |
+
# Q8w: parameter-matched control for Q8g. 20 layers x 6 matrices is the same
|
| 625 |
+
# block count and the same parameter count as Q6's 24 layers x 5 matrices, so
|
| 626 |
+
# the only difference from Q6 is the gated FFN, not extra capacity.
|
| 627 |
+
EXPERIMENTS["Q8w"] = {
|
| 628 |
+
"desc": "Q8g with 20 layers: same params as Q6 (2.47M), gated FFN",
|
| 629 |
+
"overrides": {**EXPERIMENTS["Q8g"]["overrides"],
|
| 630 |
+
"model.n_layers": 20},
|
| 631 |
+
}
|
| 632 |
+
# Q9/Q9k: multiplicative feature interaction. The bilinear term is ~one Kron
|
| 633 |
+
# block in parameters (3*d*r = 15,360 vs 15,753), so both runs stay at ~2.47M.
|
| 634 |
+
EXPERIMENTS["Q9"] = {
|
| 635 |
+
"desc": "Q6 with a bilinear (GLU) FFN instead of the linear Kron FFN (~2.46M)",
|
| 636 |
+
"overrides": {**EXPERIMENTS["Q6"]["overrides"],
|
| 637 |
+
"model.ffn_mix": "glu",
|
| 638 |
+
"model.ffn_mix_rank": 16,
|
| 639 |
+
"training.batch_size": 24},
|
| 640 |
+
}
|
| 641 |
+
EXPERIMENTS["Q9k"] = {
|
| 642 |
+
"desc": "Q9 + the Kron map as a non-interacting term (20 layers, ~2.47M)",
|
| 643 |
+
"overrides": {**EXPERIMENTS["Q9"]["overrides"],
|
| 644 |
+
"model.ffn_mix_kron": True,
|
| 645 |
+
"model.n_layers": 20},
|
| 646 |
+
}
|
| 647 |
+
|
| 648 |
+
|
| 649 |
+
def deep_merge(base: Mapping[str, Any], override: Mapping[str, Any]) -> Dict[str, Any]:
|
| 650 |
+
"""Recursively merge ``override`` into a copy of ``base``."""
|
| 651 |
+
out = copy.deepcopy(dict(base))
|
| 652 |
+
for key, value in override.items():
|
| 653 |
+
if isinstance(value, Mapping) and isinstance(out.get(key), Mapping):
|
| 654 |
+
out[key] = deep_merge(out[key], value)
|
| 655 |
+
else:
|
| 656 |
+
out[key] = copy.deepcopy(value)
|
| 657 |
+
return out
|
| 658 |
+
|
| 659 |
+
|
| 660 |
+
def set_dotted(cfg: Dict[str, Any], dotted: str, value: Any) -> None:
|
| 661 |
+
parts = dotted.split(".")
|
| 662 |
+
node = cfg
|
| 663 |
+
for part in parts[:-1]:
|
| 664 |
+
if part not in node or not isinstance(node[part], dict):
|
| 665 |
+
node[part] = {}
|
| 666 |
+
node = node[part]
|
| 667 |
+
node[parts[-1]] = value
|
| 668 |
+
|
| 669 |
+
|
| 670 |
+
def get_dotted(cfg: Mapping[str, Any], dotted: str, default: Any = None) -> Any:
|
| 671 |
+
node: Any = cfg
|
| 672 |
+
for part in dotted.split("."):
|
| 673 |
+
if not isinstance(node, Mapping) or part not in node:
|
| 674 |
+
return default
|
| 675 |
+
node = node[part]
|
| 676 |
+
return node
|
| 677 |
+
|
| 678 |
+
|
| 679 |
+
def load_config(path: str | Path | None = None,
|
| 680 |
+
overrides: Iterable[str] | Mapping[str, Any] | None = None) -> Dict[str, Any]:
|
| 681 |
+
"""Load YAML config on top of DEFAULTS, then apply overrides.
|
| 682 |
+
|
| 683 |
+
``overrides`` accepts either a mapping of dotted keys or an iterable of
|
| 684 |
+
``key=value`` strings (CLI style).
|
| 685 |
+
"""
|
| 686 |
+
cfg = copy.deepcopy(DEFAULTS)
|
| 687 |
+
if path is not None:
|
| 688 |
+
with open(path, "r", encoding="utf-8") as fh:
|
| 689 |
+
cfg = deep_merge(cfg, yaml.safe_load(fh) or {})
|
| 690 |
+
|
| 691 |
+
if overrides is None:
|
| 692 |
+
return cfg
|
| 693 |
+
|
| 694 |
+
if isinstance(overrides, Mapping):
|
| 695 |
+
for key, value in overrides.items():
|
| 696 |
+
set_dotted(cfg, key, value)
|
| 697 |
+
return cfg
|
| 698 |
+
|
| 699 |
+
for item in overrides:
|
| 700 |
+
if "=" not in item:
|
| 701 |
+
raise ValueError(f"override must be key=value, got {item!r}")
|
| 702 |
+
key, _, raw = item.partition("=")
|
| 703 |
+
set_dotted(cfg, key.strip(), _parse_scalar(raw.strip()))
|
| 704 |
+
return cfg
|
| 705 |
+
|
| 706 |
+
|
| 707 |
+
def _parse_scalar(raw: str) -> Any:
|
| 708 |
+
try:
|
| 709 |
+
return yaml.safe_load(raw)
|
| 710 |
+
except yaml.YAMLError:
|
| 711 |
+
return raw
|
| 712 |
+
|
| 713 |
+
|
| 714 |
+
def experiment_config(exp_id: str, base_path: str | Path | None = None,
|
| 715 |
+
extra: Iterable[str] | None = None) -> Dict[str, Any]:
|
| 716 |
+
"""Build the full config for one E-run."""
|
| 717 |
+
if exp_id not in EXPERIMENTS:
|
| 718 |
+
raise KeyError(f"unknown experiment {exp_id!r}; known: {sorted(EXPERIMENTS)}")
|
| 719 |
+
overrides: Dict[str, Any] = dict(EXPERIMENTS[exp_id]["overrides"])
|
| 720 |
+
cfg = load_config(base_path, overrides)
|
| 721 |
+
cfg["experiment"] = exp_id
|
| 722 |
+
cfg["experiment_desc"] = EXPERIMENTS[exp_id]["desc"]
|
| 723 |
+
if extra:
|
| 724 |
+
for item in extra:
|
| 725 |
+
key, _, raw = item.partition("=")
|
| 726 |
+
set_dotted(cfg, key.strip(), _parse_scalar(raw.strip()))
|
| 727 |
+
return cfg
|
| 728 |
+
|
| 729 |
+
|
| 730 |
+
def save_config(cfg: Mapping[str, Any], path: str | Path) -> None:
|
| 731 |
+
path = Path(path)
|
| 732 |
+
path.parent.mkdir(parents=True, exist_ok=True)
|
| 733 |
+
with open(path, "w", encoding="utf-8") as fh:
|
| 734 |
+
yaml.safe_dump(json.loads(json.dumps(cfg, default=str)), fh,
|
| 735 |
+
sort_keys=False, allow_unicode=True)
|
kronllm/data.py
ADDED
|
@@ -0,0 +1,193 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""Byte-level data pipeline (TinyStories -> uint8 memmaps).
|
| 2 |
+
|
| 3 |
+
TinyStories is stored as UTF-8 bytes (no BPE): ``V = 256``. Stories are
|
| 4 |
+
separated by ``0xFF`` which can never occur inside valid UTF-8, so it is a safe
|
| 5 |
+
end-of-text marker.
|
| 6 |
+
"""
|
| 7 |
+
from __future__ import annotations
|
| 8 |
+
|
| 9 |
+
import os
|
| 10 |
+
from pathlib import Path
|
| 11 |
+
from typing import Any, Iterator, Mapping, Optional, Tuple
|
| 12 |
+
|
| 13 |
+
import numpy as np
|
| 14 |
+
import torch
|
| 15 |
+
from torch.utils.data import Dataset
|
| 16 |
+
|
| 17 |
+
EOT = 255
|
| 18 |
+
|
| 19 |
+
|
| 20 |
+
# ---------------------------------------------------------------------------
|
| 21 |
+
# dataset
|
| 22 |
+
# ---------------------------------------------------------------------------
|
| 23 |
+
class ByteDataset(Dataset):
|
| 24 |
+
"""Random-access token blocks backed by a memmap.
|
| 25 |
+
|
| 26 |
+
``dtype=np.uint8`` is the byte-level corpus (V=256); ``np.uint16`` is the
|
| 27 |
+
BPE corpus (V<=65535). The class name is historical.
|
| 28 |
+
"""
|
| 29 |
+
|
| 30 |
+
def __init__(self, path: str | Path, seq_len: int,
|
| 31 |
+
dtype: Any = np.uint8) -> None:
|
| 32 |
+
self.path = Path(path)
|
| 33 |
+
if not self.path.exists():
|
| 34 |
+
raise FileNotFoundError(
|
| 35 |
+
f"{self.path} not found -- run `python scripts/prepare_data.py` first")
|
| 36 |
+
self.seq_len = int(seq_len)
|
| 37 |
+
self.dtype = np.dtype(dtype)
|
| 38 |
+
self.data = np.memmap(self.path, dtype=self.dtype, mode="r")
|
| 39 |
+
if len(self.data) < self.seq_len + 1:
|
| 40 |
+
raise ValueError(f"{self.path} is too small ({len(self.data)} tokens)")
|
| 41 |
+
|
| 42 |
+
def __len__(self) -> int:
|
| 43 |
+
return max(1, (len(self.data) - 1) // self.seq_len)
|
| 44 |
+
|
| 45 |
+
def __getitem__(self, i: int) -> Tuple[torch.Tensor, torch.Tensor]:
|
| 46 |
+
start = (i * self.seq_len) % (len(self.data) - self.seq_len - 1)
|
| 47 |
+
chunk = np.asarray(self.data[start:start + self.seq_len + 1], dtype=np.int64)
|
| 48 |
+
x = torch.from_numpy(chunk[:-1])
|
| 49 |
+
y = torch.from_numpy(chunk[1:])
|
| 50 |
+
return x, y
|
| 51 |
+
|
| 52 |
+
def get_batch(self, batch_size: int, device: torch.device | str = "cpu",
|
| 53 |
+
generator: Optional[torch.Generator] = None
|
| 54 |
+
) -> Tuple[torch.Tensor, torch.Tensor]:
|
| 55 |
+
n = len(self.data)
|
| 56 |
+
ix = torch.randint(0, n - self.seq_len - 1, (batch_size,), generator=generator)
|
| 57 |
+
x = torch.stack([torch.from_numpy(
|
| 58 |
+
np.asarray(self.data[i:i + self.seq_len], dtype=np.int64)) for i in ix])
|
| 59 |
+
y = torch.stack([torch.from_numpy(
|
| 60 |
+
np.asarray(self.data[i + 1:i + 1 + self.seq_len], dtype=np.int64)) for i in ix])
|
| 61 |
+
return x.to(device, non_blocking=True), y.to(device, non_blocking=True)
|
| 62 |
+
|
| 63 |
+
def iter_blocks(self, batch_size: int, device: torch.device | str = "cpu"
|
| 64 |
+
) -> Iterator[Tuple[torch.Tensor, torch.Tensor]]:
|
| 65 |
+
"""Deterministic sequential pass (used for validation)."""
|
| 66 |
+
n_blocks = len(self)
|
| 67 |
+
for start in range(0, n_blocks, batch_size):
|
| 68 |
+
idxs = range(start, min(start + batch_size, n_blocks))
|
| 69 |
+
xs, ys = [], []
|
| 70 |
+
for b in idxs:
|
| 71 |
+
x, y = self[b]
|
| 72 |
+
xs.append(x)
|
| 73 |
+
ys.append(y)
|
| 74 |
+
yield torch.stack(xs).to(device), torch.stack(ys).to(device)
|
| 75 |
+
|
| 76 |
+
|
| 77 |
+
def make_synthetic_bin(path: str | Path, n_bytes: int = 1 << 20,
|
| 78 |
+
seed: int = 0) -> Path:
|
| 79 |
+
"""Random bytes with a learnable first-order structure (smoke tests)."""
|
| 80 |
+
path = Path(path)
|
| 81 |
+
path.parent.mkdir(parents=True, exist_ok=True)
|
| 82 |
+
rng = np.random.default_rng(seed)
|
| 83 |
+
alphabet = np.arange(32, 127, dtype=np.uint8)
|
| 84 |
+
data = rng.choice(alphabet, size=n_bytes)
|
| 85 |
+
# inject simple Markov structure so the model has something to learn
|
| 86 |
+
for i in range(1, n_bytes):
|
| 87 |
+
if rng.random() < 0.6:
|
| 88 |
+
data[i] = min(255, data[i - 1] + 1)
|
| 89 |
+
data.tofile(path)
|
| 90 |
+
return path
|
| 91 |
+
|
| 92 |
+
|
| 93 |
+
# ---------------------------------------------------------------------------
|
| 94 |
+
# preparation
|
| 95 |
+
# ---------------------------------------------------------------------------
|
| 96 |
+
SEP = b"\xff"
|
| 97 |
+
|
| 98 |
+
|
| 99 |
+
def dataset_paths(cfg: Mapping[str, Any], data_dir: str | Path | None = None
|
| 100 |
+
) -> dict:
|
| 101 |
+
"""Resolve corpus paths + dtype from the tokenizer choice.
|
| 102 |
+
|
| 103 |
+
``bpe`` -> ``data/{train,val}.bpe.bin`` as ``uint16``
|
| 104 |
+
``byte`` -> ``data/{train,val}.bin`` as ``uint8``
|
| 105 |
+
"""
|
| 106 |
+
dc = cfg["data"]
|
| 107 |
+
base = Path(data_dir or dc.get("data_dir", "data"))
|
| 108 |
+
kind = str(dc.get("tokenizer", "byte")).lower()
|
| 109 |
+
if kind in ("byte", "bytes", "byte_level"):
|
| 110 |
+
return {"train": base / dc.get("train_bin", "train.bin"),
|
| 111 |
+
"val": base / dc.get("val_bin", "val.bin"),
|
| 112 |
+
"dtype": np.uint8, "tokenizer": "byte"}
|
| 113 |
+
if kind == "bpe":
|
| 114 |
+
return {"train": base / dc.get("train_bin", "train.bpe.bin"),
|
| 115 |
+
"val": base / dc.get("val_bin", "val.bpe.bin"),
|
| 116 |
+
"dtype": np.uint16, "tokenizer": "bpe"}
|
| 117 |
+
raise ValueError(f"unknown tokenizer {kind!r}")
|
| 118 |
+
|
| 119 |
+
|
| 120 |
+
def open_dataset(cfg: Mapping[str, Any], split: str = "train",
|
| 121 |
+
data_dir: str | Path | None = None,
|
| 122 |
+
seq_len: int | None = None) -> "ByteDataset":
|
| 123 |
+
paths = dataset_paths(cfg, data_dir)
|
| 124 |
+
n = int(seq_len if seq_len is not None else cfg["data"]["seq_len"])
|
| 125 |
+
return ByteDataset(paths[split], n, dtype=paths["dtype"])
|
| 126 |
+
|
| 127 |
+
|
| 128 |
+
def _encode_texts(texts, eot: int = EOT, budget: Optional[int] = None) -> bytes:
|
| 129 |
+
sep = bytes([eot])
|
| 130 |
+
out = bytearray()
|
| 131 |
+
for text in texts:
|
| 132 |
+
out += text.encode("utf-8", errors="ignore")
|
| 133 |
+
out += sep
|
| 134 |
+
if budget is not None and len(out) >= budget:
|
| 135 |
+
break
|
| 136 |
+
return bytes(out)
|
| 137 |
+
|
| 138 |
+
|
| 139 |
+
def prepare_tinystories(cfg: Mapping[str, Any]) -> dict:
|
| 140 |
+
"""Download/encode TinyStories into ``{data_dir}/train.bin`` and ``val.bin``."""
|
| 141 |
+
from datasets import load_dataset
|
| 142 |
+
|
| 143 |
+
dc = cfg["data"]
|
| 144 |
+
data_dir = Path(dc.get("data_dir", "data"))
|
| 145 |
+
data_dir.mkdir(parents=True, exist_ok=True)
|
| 146 |
+
eot = int(dc.get("eot_id", EOT))
|
| 147 |
+
max_stories = dc.get("max_stories", 200_000)
|
| 148 |
+
max_val = dc.get("max_val_stories", 10_000)
|
| 149 |
+
|
| 150 |
+
train_path = data_dir / "train.bin"
|
| 151 |
+
val_path = data_dir / "val.bin"
|
| 152 |
+
meta: dict = {"dataset": dc["dataset"], "eot": eot}
|
| 153 |
+
|
| 154 |
+
if train_path.exists() and val_path.exists() and not dc.get("force", False):
|
| 155 |
+
meta["train_bytes"] = train_path.stat().st_size
|
| 156 |
+
meta["val_bytes"] = val_path.stat().st_size
|
| 157 |
+
meta["cached"] = True
|
| 158 |
+
return meta
|
| 159 |
+
|
| 160 |
+
ds = load_dataset(dc["dataset"])
|
| 161 |
+
tr_key = "train" if "train" in ds else list(ds.keys())[0]
|
| 162 |
+
va_key = "validation" if "validation" in ds else (
|
| 163 |
+
"valid" if "valid" in ds else None)
|
| 164 |
+
del ds
|
| 165 |
+
|
| 166 |
+
n_tr = int(max_stories) if max_stories else None
|
| 167 |
+
tr_split = f"{tr_key}[:{n_tr}]" if n_tr else tr_key
|
| 168 |
+
print(f"[data] encoding train split {tr_split} ...")
|
| 169 |
+
tr_ds = load_dataset(dc["dataset"], split=tr_split)
|
| 170 |
+
tr_bytes = _encode_texts(tr_ds["text"], eot)
|
| 171 |
+
train_path.write_bytes(tr_bytes)
|
| 172 |
+
del tr_ds
|
| 173 |
+
|
| 174 |
+
if va_key is not None:
|
| 175 |
+
n_va = int(max_val) if max_val else None
|
| 176 |
+
va_split = f"{va_key}[:{n_va}]" if n_va else va_key
|
| 177 |
+
print(f"[data] encoding validation split {va_split} ...")
|
| 178 |
+
va_ds = load_dataset(dc["dataset"], split=va_split)
|
| 179 |
+
va_bytes = _encode_texts(va_ds["text"], eot)
|
| 180 |
+
del va_ds
|
| 181 |
+
else:
|
| 182 |
+
n = len(tr_bytes)
|
| 183 |
+
split_at = int(n * (1.0 - float(dc.get("val_split", 0.05))))
|
| 184 |
+
va_bytes = tr_bytes[split_at:]
|
| 185 |
+
tr_bytes = tr_bytes[:split_at]
|
| 186 |
+
train_path.write_bytes(tr_bytes)
|
| 187 |
+
val_path.write_bytes(va_bytes)
|
| 188 |
+
|
| 189 |
+
meta["train_bytes"] = len(tr_bytes)
|
| 190 |
+
meta["val_bytes"] = len(va_bytes)
|
| 191 |
+
meta["train_stories"] = n_tr
|
| 192 |
+
meta["cached"] = False
|
| 193 |
+
return meta
|
kronllm/diagnostics.py
ADDED
|
@@ -0,0 +1,386 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""Diagnostic and comparison metrics (sections 5.1 - 5.3 of the spec).
|
| 2 |
+
|
| 3 |
+
Everything here is measurement only: no gradients are created and no module
|
| 4 |
+
state is mutated.
|
| 5 |
+
"""
|
| 6 |
+
from __future__ import annotations
|
| 7 |
+
|
| 8 |
+
import math
|
| 9 |
+
from typing import Any, Dict, List, Mapping, Optional, Sequence, Tuple
|
| 10 |
+
|
| 11 |
+
import torch
|
| 12 |
+
import torch.nn as nn
|
| 13 |
+
|
| 14 |
+
from .kron import KronCompose, effective_rank
|
| 15 |
+
|
| 16 |
+
ROLES = ("q", "k", "v", "o", "f")
|
| 17 |
+
ALL_ROLES = ("q", "k", "v", "o", "f", "fg")
|
| 18 |
+
|
| 19 |
+
|
| 20 |
+
# ---------------------------------------------------------------------------
|
| 21 |
+
# helpers
|
| 22 |
+
# ---------------------------------------------------------------------------
|
| 23 |
+
def _module(block: nn.Module, role: str) -> Optional[KronCompose]:
|
| 24 |
+
if role == "f":
|
| 25 |
+
mod = block.ffn
|
| 26 |
+
elif role == "fg":
|
| 27 |
+
mod = getattr(block, "ffn_g", None)
|
| 28 |
+
else:
|
| 29 |
+
mod = getattr(block, role, None)
|
| 30 |
+
return mod if isinstance(mod, KronCompose) else None
|
| 31 |
+
|
| 32 |
+
|
| 33 |
+
def spearman(x: Sequence[float], y: Sequence[float]) -> float:
|
| 34 |
+
"""Spearman rank correlation without scipy."""
|
| 35 |
+
n = len(x)
|
| 36 |
+
if n < 3:
|
| 37 |
+
return float("nan")
|
| 38 |
+
|
| 39 |
+
def ranks(v: Sequence[float]) -> List[float]:
|
| 40 |
+
order = sorted(range(n), key=lambda i: v[i])
|
| 41 |
+
r = [0.0] * n
|
| 42 |
+
i = 0
|
| 43 |
+
while i < n:
|
| 44 |
+
j = i
|
| 45 |
+
while j + 1 < n and v[order[j + 1]] == v[order[i]]:
|
| 46 |
+
j += 1
|
| 47 |
+
avg = (i + j) / 2.0 + 1.0
|
| 48 |
+
for t in range(i, j + 1):
|
| 49 |
+
r[order[t]] = avg
|
| 50 |
+
i = j + 1
|
| 51 |
+
return r
|
| 52 |
+
|
| 53 |
+
rx, ry = ranks(x), ranks(y)
|
| 54 |
+
mx, my = sum(rx) / n, sum(ry) / n
|
| 55 |
+
num = sum((a - mx) * (b - my) for a, b in zip(rx, ry))
|
| 56 |
+
dx = math.sqrt(sum((a - mx) ** 2 for a in rx))
|
| 57 |
+
dy = math.sqrt(sum((b - my) ** 2 for b in ry))
|
| 58 |
+
return num / (dx * dy) if dx > 0 and dy > 0 else float("nan")
|
| 59 |
+
|
| 60 |
+
|
| 61 |
+
# ---------------------------------------------------------------------------
|
| 62 |
+
# sigma(theta) -- D1.1, D1.2, H3
|
| 63 |
+
# ---------------------------------------------------------------------------
|
| 64 |
+
@torch.no_grad()
|
| 65 |
+
def sigma_theta_report(model: nn.Module) -> Dict[str, Any]:
|
| 66 |
+
per_type: Dict[str, List[List[float]]] = {r: [] for r in ALL_ROLES}
|
| 67 |
+
for block in model.blocks:
|
| 68 |
+
for role in ALL_ROLES:
|
| 69 |
+
mod = _module(block, role)
|
| 70 |
+
if mod is None:
|
| 71 |
+
continue
|
| 72 |
+
per_type[role].append(torch.sigmoid(mod.theta).tolist())
|
| 73 |
+
|
| 74 |
+
flat = [v for rows in per_type.values() for row in rows for v in row]
|
| 75 |
+
out: Dict[str, Any] = {
|
| 76 |
+
"sigma_theta_mean": sum(flat) / len(flat) if flat else float("nan"),
|
| 77 |
+
"sigma_theta_per_type": per_type,
|
| 78 |
+
"sigma_theta_heatmap": per_type.get("q", []),
|
| 79 |
+
}
|
| 80 |
+
# variance across the L memory channels, per layer/type (H3)
|
| 81 |
+
variances = [torch.tensor(rows).var(unbiased=False).item()
|
| 82 |
+
for rows in per_type.values() for row in rows if len(row) > 1]
|
| 83 |
+
out["sigma_theta_var"] = sum(variances) / len(variances) if variances else 0.0
|
| 84 |
+
out["sigma_theta_var_max"] = max(variances) if variances else 0.0
|
| 85 |
+
return out
|
| 86 |
+
|
| 87 |
+
|
| 88 |
+
@torch.no_grad()
|
| 89 |
+
def theta_grad_norm(model: nn.Module) -> float:
|
| 90 |
+
total = 0.0
|
| 91 |
+
for block in model.blocks:
|
| 92 |
+
for role in ALL_ROLES:
|
| 93 |
+
mod = _module(block, role)
|
| 94 |
+
if mod is not None and mod.theta.grad is not None:
|
| 95 |
+
total += float(mod.theta.grad.detach().pow(2).sum())
|
| 96 |
+
return math.sqrt(total)
|
| 97 |
+
|
| 98 |
+
|
| 99 |
+
# ---------------------------------------------------------------------------
|
| 100 |
+
# spectral analysis of W_{i,l} -- D2.1, D2.4, D3.x, H2
|
| 101 |
+
# ---------------------------------------------------------------------------
|
| 102 |
+
@torch.no_grad()
|
| 103 |
+
def routed_report(model: nn.Module) -> Dict[str, float]:
|
| 104 |
+
"""Measure router usage and effective-table diversity for a routed model.
|
| 105 |
+
|
| 106 |
+
The report is intentionally cheap enough to run at every validation point:
|
| 107 |
+
it materialises the ``(vocab_size, d_model)`` effective table once and does
|
| 108 |
+
one small SVD. Non-routed models return an empty dict, so the same training
|
| 109 |
+
loop can log both families.
|
| 110 |
+
|
| 111 |
+
Args:
|
| 112 |
+
model: A ``KronTransformer`` or compatible module.
|
| 113 |
+
|
| 114 |
+
Returns:
|
| 115 |
+
A mapping with router usage perplexity, used-row counts, the fraction of
|
| 116 |
+
tokens whose own bank row is top-1, the effective rank of the table, and
|
| 117 |
+
the router-logit standard deviation. Empty for a factorised model.
|
| 118 |
+
"""
|
| 119 |
+
routed = getattr(model, "routed", None)
|
| 120 |
+
if routed is None:
|
| 121 |
+
return {}
|
| 122 |
+
|
| 123 |
+
logits = routed.route_logits()
|
| 124 |
+
top = torch.topk(logits, k=routed.top_k, dim=-1).indices
|
| 125 |
+
counts = torch.bincount(top.flatten(), minlength=routed.v_eff).float()
|
| 126 |
+
probs = counts / counts.sum()
|
| 127 |
+
nz = probs[probs > 0]
|
| 128 |
+
usage_ppl = torch.exp(-(nz * nz.log()).sum())
|
| 129 |
+
|
| 130 |
+
top1 = top[:, 0]
|
| 131 |
+
own = (top1[:routed.vocab_size] ==
|
| 132 |
+
torch.arange(routed.vocab_size, device=top1.device)).float().mean()
|
| 133 |
+
|
| 134 |
+
effective = routed.effective_embeddings()
|
| 135 |
+
table_rank = effective_rank(torch.linalg.svdvals(effective))
|
| 136 |
+
return {
|
| 137 |
+
"route_usage_ppl": float(usage_ppl),
|
| 138 |
+
"route_unique_rows": int((counts > 0).sum()),
|
| 139 |
+
"route_top1_unique": int(torch.unique(top1).numel()),
|
| 140 |
+
"route_own_top1": float(own),
|
| 141 |
+
"route_eff_rank": float(table_rank),
|
| 142 |
+
"route_logit_std": float(logits.std()),
|
| 143 |
+
}
|
| 144 |
+
|
| 145 |
+
|
| 146 |
+
@torch.no_grad()
|
| 147 |
+
def spectral_report(model: nn.Module, trace: List[Dict[str, Any]],
|
| 148 |
+
max_layers: Optional[int] = None) -> Dict[str, Any]:
|
| 149 |
+
"""SVD of every ``D_{i,l}`` and of the composed map ``P_i``.
|
| 150 |
+
|
| 151 |
+
Two rank numbers matter and they answer different questions:
|
| 152 |
+
|
| 153 |
+
* ``eff_rank_D`` -- rank of the *innovation* of one channel. This is what
|
| 154 |
+
the Kronecker parameterisation can actually control and what H2 ("does
|
| 155 |
+
expressivity grow with depth") is about. A single term has
|
| 156 |
+
``rank <= min(n1^2, n2^2) = 64``.
|
| 157 |
+
* ``eff_rank_W`` -- rank of the materialised factor ``s Id + D``. With the
|
| 158 |
+
anchor on it is *always* ``d`` (the identity carries the missing rank),
|
| 159 |
+
which is why the original spec's "full rank W" statement was vacuous.
|
| 160 |
+
|
| 161 |
+
``eff_rank_excess`` is the rank of ``P - s^L Id``, i.e. the part of the
|
| 162 |
+
composition that is not the anchor.
|
| 163 |
+
"""
|
| 164 |
+
rows: List[Dict[str, Any]] = []
|
| 165 |
+
prev: Dict[Tuple[str, int], torch.Tensor] = {}
|
| 166 |
+
prev_p: Dict[str, torch.Tensor] = {}
|
| 167 |
+
n = len(trace) if max_layers is None else min(max_layers, len(trace))
|
| 168 |
+
|
| 169 |
+
for i in range(n):
|
| 170 |
+
entry = trace[i]
|
| 171 |
+
block = model.blocks[i]
|
| 172 |
+
for role in ALL_ROLES:
|
| 173 |
+
mod = _module(block, role)
|
| 174 |
+
D = entry["D"].get(role)
|
| 175 |
+
if mod is None or D is None:
|
| 176 |
+
continue
|
| 177 |
+
s = mod.s
|
| 178 |
+
s_val = float(s) if s is not None else float("nan")
|
| 179 |
+
tau_val = float(mod.tau.mean())
|
| 180 |
+
gate_val = float(mod.gate) if mod.gate is not None else float("nan")
|
| 181 |
+
|
| 182 |
+
# composition-level numbers
|
| 183 |
+
P = mod.compose(D).float()
|
| 184 |
+
sv_p = torch.linalg.svdvals(P)
|
| 185 |
+
eff_p = float(effective_rank(sv_p))
|
| 186 |
+
if s is not None:
|
| 187 |
+
eye = torch.eye(P.shape[0], device=P.device, dtype=P.dtype)
|
| 188 |
+
P_exc = P - float(mod.anchor_term) * eye
|
| 189 |
+
else:
|
| 190 |
+
P_exc = P
|
| 191 |
+
eff_exc = float(effective_rank(torch.linalg.svdvals(P_exc)))
|
| 192 |
+
# Rank of the sum-composition alternative on the same trained D_l.
|
| 193 |
+
# This is a diagnostic, not a trained model: it isolates how much of
|
| 194 |
+
# the product's rank loss is due to multiplication itself. For an
|
| 195 |
+
# already-sum model this reduces to the trained sum map.
|
| 196 |
+
if mod.composition == "sum":
|
| 197 |
+
w = (mod._last_w_mean if getattr(mod, "dynamic_gate", None)
|
| 198 |
+
else mod.alpha)
|
| 199 |
+
S = (w.float().view(-1, 1, 1) * D.float()).sum(dim=0)
|
| 200 |
+
else:
|
| 201 |
+
S = D.float().sum(dim=0)
|
| 202 |
+
if s is not None:
|
| 203 |
+
S = S + float(s) * torch.eye(S.shape[0], device=S.device,
|
| 204 |
+
dtype=S.dtype)
|
| 205 |
+
eff_sum = float(effective_rank(torch.linalg.svdvals(S)))
|
| 206 |
+
cos_p = float("nan")
|
| 207 |
+
if role in prev_p:
|
| 208 |
+
a, b = P.flatten(), prev_p[role].flatten()
|
| 209 |
+
cos_p = float(torch.dot(a, b) / (a.norm() * b.norm()).clamp_min(1e-12))
|
| 210 |
+
prev_p[role] = P.clone()
|
| 211 |
+
|
| 212 |
+
for l in range(D.shape[0]):
|
| 213 |
+
Dl = D[l].float()
|
| 214 |
+
sv_d = torch.linalg.svdvals(Dl)
|
| 215 |
+
eff_d = float(effective_rank(sv_d))
|
| 216 |
+
d_rms = float(Dl.pow(2).mean().sqrt())
|
| 217 |
+
d_norm = float(Dl.norm())
|
| 218 |
+
s_abs = abs(s_val) if math.isfinite(s_val) else float("nan")
|
| 219 |
+
d_over_s = d_rms / s_abs if s_abs > 0 else float("nan")
|
| 220 |
+
fro_over_s = d_norm / s_abs if s_abs > 0 else float("nan")
|
| 221 |
+
sigma_max_d = float(sv_d.max())
|
| 222 |
+
smax_d_over_s = sigma_max_d / s_abs if s_abs > 0 else float("nan")
|
| 223 |
+
W = Dl if s is None else float(s) * torch.eye(
|
| 224 |
+
Dl.shape[0], device=Dl.device, dtype=Dl.dtype) + Dl
|
| 225 |
+
sv_w = torch.linalg.svdvals(W)
|
| 226 |
+
smin, smax = float(sv_w.min()), float(sv_w.max())
|
| 227 |
+
cos = float("nan")
|
| 228 |
+
key = (role, l)
|
| 229 |
+
if key in prev:
|
| 230 |
+
a, b = W.flatten(), prev[key].flatten()
|
| 231 |
+
cos = float(torch.dot(a, b) / (a.norm() * b.norm()).clamp_min(1e-12))
|
| 232 |
+
prev[key] = W.clone()
|
| 233 |
+
rows.append({
|
| 234 |
+
"layer": i, "role": role, "channel": l,
|
| 235 |
+
"s": s_val, "tau": tau_val, "gate": gate_val,
|
| 236 |
+
"D_rms": d_rms,
|
| 237 |
+
"D_norm": d_norm,
|
| 238 |
+
"D_over_s": d_over_s,
|
| 239 |
+
"D_fro_over_s": fro_over_s,
|
| 240 |
+
"sigma_max_D": sigma_max_d,
|
| 241 |
+
"sigma_max_D_over_s": smax_d_over_s,
|
| 242 |
+
"eff_rank_D": eff_d,
|
| 243 |
+
"eff_rank_W": float(effective_rank(sv_w)),
|
| 244 |
+
"rank_excess": float(effective_rank(sv_w)) - eff_d,
|
| 245 |
+
"sigma_min": smin, "sigma_max": smax,
|
| 246 |
+
"cond": smax / max(smin, 1e-12),
|
| 247 |
+
"cos_prev": cos,
|
| 248 |
+
"eff_rank_P": eff_p,
|
| 249 |
+
"eff_rank_excess": eff_exc,
|
| 250 |
+
"eff_rank_sum": eff_sum,
|
| 251 |
+
"sum_over_prod": eff_sum / max(eff_p, 1e-9),
|
| 252 |
+
"cos_prev_P": cos_p,
|
| 253 |
+
})
|
| 254 |
+
|
| 255 |
+
summary: Dict[str, Any] = {}
|
| 256 |
+
if rows:
|
| 257 |
+
mean = lambda key: sum(r[key] for r in rows) / len(rows) # noqa: E731
|
| 258 |
+
summary["sigma_min_W"] = min(r["sigma_min"] for r in rows)
|
| 259 |
+
summary["sigma_max_W"] = max(r["sigma_max"] for r in rows)
|
| 260 |
+
summary["cond_max"] = max(r["cond"] for r in rows)
|
| 261 |
+
summary["eff_rank_mean"] = mean("eff_rank_W")
|
| 262 |
+
summary["eff_rank_W"] = mean("eff_rank_W")
|
| 263 |
+
summary["eff_rank_D"] = mean("eff_rank_D")
|
| 264 |
+
summary["rank_excess"] = mean("rank_excess")
|
| 265 |
+
summary["eff_rank_P"] = mean("eff_rank_P")
|
| 266 |
+
summary["eff_rank_excess"] = mean("eff_rank_excess")
|
| 267 |
+
summary["eff_rank_sum"] = mean("eff_rank_sum")
|
| 268 |
+
summary["sum_over_prod"] = mean("sum_over_prod")
|
| 269 |
+
summary["D_norm_mean"] = mean("D_norm")
|
| 270 |
+
summary["D_rms"] = mean("D_rms")
|
| 271 |
+
finite = lambda key: [r[key] for r in rows if math.isfinite(r[key])] # noqa: E731
|
| 272 |
+
mean_finite = lambda key: (sum(finite(key)) / len(finite(key)) # noqa: E731
|
| 273 |
+
if finite(key) else float("nan"))
|
| 274 |
+
summary["D_over_s"] = mean_finite("D_over_s")
|
| 275 |
+
summary["D_fro_over_s"] = mean_finite("D_fro_over_s")
|
| 276 |
+
summary["sigma_max_D_over_s"] = mean_finite("sigma_max_D_over_s")
|
| 277 |
+
summary["tau_mean"] = mean("tau")
|
| 278 |
+
summary["gate_mean"] = mean("gate")
|
| 279 |
+
coss = [r["cos_prev"] for r in rows if not math.isnan(r["cos_prev"])]
|
| 280 |
+
summary["cos_prev_max"] = max(coss) if coss else float("nan")
|
| 281 |
+
coss_p = [r["cos_prev_P"] for r in rows if not math.isnan(r["cos_prev_P"])]
|
| 282 |
+
summary["cos_prev_P_max"] = max(coss_p) if coss_p else float("nan")
|
| 283 |
+
summary["cos_prev_P_mean"] = (sum(coss_p) / len(coss_p)) if coss_p else float("nan")
|
| 284 |
+
# H2: does the expressivity of the innovation grow with depth?
|
| 285 |
+
layers = sorted({r["layer"] for r in rows})
|
| 286 |
+
per_layer = [sum(r["eff_rank_D"] for r in rows if r["layer"] == i)
|
| 287 |
+
/ max(1, sum(1 for r in rows if r["layer"] == i))
|
| 288 |
+
for i in layers]
|
| 289 |
+
per_layer_p = [sum(r["eff_rank_excess"] for r in rows if r["layer"] == i)
|
| 290 |
+
/ max(1, sum(1 for r in rows if r["layer"] == i))
|
| 291 |
+
for i in layers]
|
| 292 |
+
summary["eff_rank_per_layer"] = per_layer
|
| 293 |
+
summary["eff_rank_spearman"] = spearman(layers, per_layer)
|
| 294 |
+
summary["eff_rank_D_spearman"] = spearman(layers, per_layer)
|
| 295 |
+
summary["eff_rank_excess_per_layer"] = per_layer_p
|
| 296 |
+
summary["eff_rank_excess_spearman"] = spearman(layers, per_layer_p)
|
| 297 |
+
summary["eff_rank_first"] = per_layer[0]
|
| 298 |
+
summary["eff_rank_last"] = per_layer[-1]
|
| 299 |
+
summary["eff_rank_growth"] = per_layer[-1] / max(per_layer[0], 1e-9)
|
| 300 |
+
return {"rows": rows, "summary": summary}
|
| 301 |
+
|
| 302 |
+
|
| 303 |
+
def eff_rank_heatmap(spectral: Mapping[str, Any], role: str = "q",
|
| 304 |
+
n_layers: Optional[int] = None,
|
| 305 |
+
L: Optional[int] = None) -> torch.Tensor:
|
| 306 |
+
"""(n_layers, L) matrix of effective ranks for plotting."""
|
| 307 |
+
rows = [r for r in spectral["rows"] if r["role"] == role]
|
| 308 |
+
if not rows:
|
| 309 |
+
return torch.zeros(0, 0)
|
| 310 |
+
nl = n_layers or (max(r["layer"] for r in rows) + 1)
|
| 311 |
+
nc = L or (max(r["channel"] for r in rows) + 1)
|
| 312 |
+
m = torch.full((nl, nc), float("nan"))
|
| 313 |
+
for r in rows:
|
| 314 |
+
m[r["layer"], r["channel"]] = r["eff_rank_D"]
|
| 315 |
+
return m
|
| 316 |
+
|
| 317 |
+
|
| 318 |
+
def sigma_heatmap(sigma: Mapping[str, Any], role: str = "q") -> torch.Tensor:
|
| 319 |
+
rows = sigma["sigma_theta_per_type"].get(role, [])
|
| 320 |
+
return torch.tensor(rows) if rows else torch.zeros(0, 0)
|
| 321 |
+
|
| 322 |
+
|
| 323 |
+
# ---------------------------------------------------------------------------
|
| 324 |
+
# gradients -- D1.3, D2.3
|
| 325 |
+
# ---------------------------------------------------------------------------
|
| 326 |
+
@torch.no_grad()
|
| 327 |
+
def grad_report(model: nn.Module) -> Dict[str, float]:
|
| 328 |
+
groups: Dict[str, float] = {}
|
| 329 |
+
total = 0.0
|
| 330 |
+
for name, p in model.named_parameters():
|
| 331 |
+
if p.grad is None:
|
| 332 |
+
continue
|
| 333 |
+
g = float(p.grad.detach().pow(2).sum())
|
| 334 |
+
total += g
|
| 335 |
+
key = ("theta" if name.endswith(".theta")
|
| 336 |
+
else "tau" if name.endswith(".tau")
|
| 337 |
+
else "gate" if name.endswith(".gate")
|
| 338 |
+
else "s" if name.endswith(".s")
|
| 339 |
+
else "norm" if "norm" in name
|
| 340 |
+
else "embed" if ("embed" in name or "lm_head" in name)
|
| 341 |
+
else "factors")
|
| 342 |
+
groups[key] = groups.get(key, 0.0) + g
|
| 343 |
+
out = {f"grad_{k}": math.sqrt(v) for k, v in groups.items()}
|
| 344 |
+
out["grad_norm"] = math.sqrt(total)
|
| 345 |
+
return out
|
| 346 |
+
|
| 347 |
+
|
| 348 |
+
# ---------------------------------------------------------------------------
|
| 349 |
+
# comparison metrics -- C1..C5
|
| 350 |
+
# ---------------------------------------------------------------------------
|
| 351 |
+
def comparison_report(model: nn.Module, seq_len: int,
|
| 352 |
+
batch_size: int = 1) -> Dict[str, float]:
|
| 353 |
+
"""``flops_per_token`` is amortised over ``seq_len * batch_size`` tokens.
|
| 354 |
+
|
| 355 |
+
The Kron prelude evaluates ``L`` products of ``d x d`` matrices once per
|
| 356 |
+
forward pass, so its cost per token shrinks with the batch. Reporting it
|
| 357 |
+
at ``batch_size=1`` overstates the Kron cost by ~2x.
|
| 358 |
+
"""
|
| 359 |
+
return {
|
| 360 |
+
"n_params": model.n_params,
|
| 361 |
+
"n_params_matrix": model.n_params_matrix,
|
| 362 |
+
"flops_per_token": model.flops_per_token(seq_len, batch_size=batch_size),
|
| 363 |
+
"flops_per_token_train": model.flops_per_token(
|
| 364 |
+
seq_len, training=True, batch_size=batch_size),
|
| 365 |
+
}
|
| 366 |
+
|
| 367 |
+
|
| 368 |
+
def aggregate_eval_stats(stats: Mapping[str, float]) -> Dict[str, float]:
|
| 369 |
+
"""Collapse per-layer diagnostics into scalars for logging."""
|
| 370 |
+
ent = [v for k, v in stats.items() if k.endswith("attn_entropy")]
|
| 371 |
+
qmin = [v for k, v in stats.items() if k.endswith("q_sigma_min")]
|
| 372 |
+
kmin = [v for k, v in stats.items() if k.endswith("k_sigma_min")]
|
| 373 |
+
drift = [v for k, v in stats.items() if k.endswith("q_drift")]
|
| 374 |
+
out = {
|
| 375 |
+
"attn_entropy_mean": sum(ent) / len(ent) if ent else float("nan"),
|
| 376 |
+
"attn_entropy_min": min(ent) if ent else float("nan"),
|
| 377 |
+
"attn_entropy_max": max(ent) if ent else float("nan"),
|
| 378 |
+
"q_sigma_min": min(qmin) if qmin else float("nan"),
|
| 379 |
+
"k_sigma_min": min(kmin) if kmin else float("nan"),
|
| 380 |
+
"q_drift_mean": sum(drift) / len(drift) if drift else float("nan"),
|
| 381 |
+
"q_drift_last": drift[-1] if drift else float("nan"),
|
| 382 |
+
"h_norm_ratio_min": stats.get("h_norm_ratio_min", float("nan")),
|
| 383 |
+
"h_norm_ratio_max": stats.get("h_norm_ratio_max", float("nan")),
|
| 384 |
+
"h_norm_ratio_mean": stats.get("h_norm_ratio_mean", float("nan")),
|
| 385 |
+
}
|
| 386 |
+
return out
|
kronllm/generate.py
ADDED
|
@@ -0,0 +1,43 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""Text generation helpers (byte-level and BPE)."""
|
| 2 |
+
from __future__ import annotations
|
| 3 |
+
|
| 4 |
+
from typing import Iterable, List, Optional, Sequence
|
| 5 |
+
|
| 6 |
+
import torch
|
| 7 |
+
|
| 8 |
+
from .model import KronTransformer
|
| 9 |
+
from .tokenizer import ByteTokenizer
|
| 10 |
+
|
| 11 |
+
|
| 12 |
+
def encode(text: str) -> List[int]:
|
| 13 |
+
return list(text.encode("utf-8", errors="ignore"))
|
| 14 |
+
|
| 15 |
+
|
| 16 |
+
def decode(ids: Sequence[int]) -> str:
|
| 17 |
+
return bytes(int(i) & 0xFF for i in ids).decode("utf-8", errors="replace")
|
| 18 |
+
|
| 19 |
+
|
| 20 |
+
@torch.no_grad()
|
| 21 |
+
def generate_texts(model: KronTransformer, prompts: Iterable[str],
|
| 22 |
+
max_new_tokens: int = 200, temperature: float = 1.0,
|
| 23 |
+
top_k: Optional[int] = 8, n_samples: int = 1,
|
| 24 |
+
seed: Optional[int] = None,
|
| 25 |
+
device: torch.device | str | None = None,
|
| 26 |
+
stop_at_eot: bool = True,
|
| 27 |
+
tokenizer=None) -> List[str]:
|
| 28 |
+
"""Sample from ``model``; ``tokenizer`` defaults to the byte vocabulary."""
|
| 29 |
+
tok = tokenizer if tokenizer is not None else ByteTokenizer()
|
| 30 |
+
device = torch.device(device) if device is not None else model.device
|
| 31 |
+
prompts = [p if p else "\n" for p in prompts] or ["Once upon a time"]
|
| 32 |
+
if seed is not None:
|
| 33 |
+
torch.manual_seed(int(seed))
|
| 34 |
+
texts: List[str] = []
|
| 35 |
+
for i in range(max(1, n_samples)):
|
| 36 |
+
prompt = prompts[i % len(prompts)]
|
| 37 |
+
ids = tok.encode(prompt) or [tok.eot_id]
|
| 38 |
+
idx = torch.tensor([ids], dtype=torch.long, device=device)
|
| 39 |
+
out = model.generate(idx, max_new_tokens=max_new_tokens,
|
| 40 |
+
temperature=temperature, top_k=top_k,
|
| 41 |
+
eot_id=tok.eot_id, stop_at_eot=stop_at_eot)
|
| 42 |
+
texts.append(tok.decode(out[0].tolist()))
|
| 43 |
+
return texts
|
kronllm/hf.py
ADDED
|
@@ -0,0 +1,229 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""Hugging Face ``transformers`` integration for KronLLM.
|
| 2 |
+
|
| 3 |
+
The wrapper exposes the custom Kronecker model through the standard
|
| 4 |
+
``PreTrainedModel`` / ``PreTrainedConfig`` interfaces, so that it can be saved,
|
| 5 |
+
loaded, moved between devices and sampled with ``GenerationMixin.generate``::
|
| 6 |
+
|
| 7 |
+
from kronllm.hf import KronForCausalLM
|
| 8 |
+
|
| 9 |
+
model = KronForCausalLM.from_kron_checkpoint("runs/Q2/best.pt")
|
| 10 |
+
model.generate(input_ids, max_new_tokens=32, do_sample=True, top_k=8)
|
| 11 |
+
|
| 12 |
+
It also registers ``KronConfig`` and ``KronForCausalLM`` with the ``AutoClass``
|
| 13 |
+
registry, which makes ``AutoModelForCausalLM.from_pretrained`` work after this
|
| 14 |
+
module has been imported.
|
| 15 |
+
|
| 16 |
+
Two deliberate limitations are inherited from the base model:
|
| 17 |
+
|
| 18 |
+
* there is no KV cache, so every generated token is a full forward pass;
|
| 19 |
+
* ``attention_mask`` is accepted for API compatibility but only an all-ones
|
| 20 |
+
mask is supported. Batch generation with left/right padding would attend to
|
| 21 |
+
pad tokens. Use one sequence per call if exact masking matters.
|
| 22 |
+
|
| 23 |
+
Tokenisation is not part of this wrapper: KronLLM uses a byte-level BPE stored
|
| 24 |
+
in ``data/tokenizer.json`` (see :func:`kronllm.tokenizer.load_tokenizer`), which
|
| 25 |
+
is not a ``PreTrainedTokenizer``. Encode text with that tokenizer, call
|
| 26 |
+
``generate``, then decode the result.
|
| 27 |
+
"""
|
| 28 |
+
from __future__ import annotations
|
| 29 |
+
|
| 30 |
+
import copy
|
| 31 |
+
from pathlib import Path
|
| 32 |
+
from typing import Any, Dict, Optional
|
| 33 |
+
|
| 34 |
+
import torch
|
| 35 |
+
import torch.nn.functional as F
|
| 36 |
+
|
| 37 |
+
from transformers import (AutoConfig, AutoModelForCausalLM, PretrainedConfig,
|
| 38 |
+
PreTrainedModel)
|
| 39 |
+
from transformers.generation import GenerationMixin
|
| 40 |
+
from transformers.modeling_outputs import CausalLMOutputWithPast
|
| 41 |
+
|
| 42 |
+
from .model import KronTransformer
|
| 43 |
+
from .quantization import load_any_checkpoint
|
| 44 |
+
|
| 45 |
+
__all__ = ["KronConfig", "KronForCausalLM"]
|
| 46 |
+
|
| 47 |
+
|
| 48 |
+
class KronConfig(PretrainedConfig):
|
| 49 |
+
"""``PretrainedConfig`` carrying the nested KronLLM configuration.
|
| 50 |
+
|
| 51 |
+
Args:
|
| 52 |
+
kron_config: The dictionary returned by
|
| 53 |
+
:func:`kronllm.config.experiment_config` or stored inside a
|
| 54 |
+
checkpoint. It is deep-copied under the ``kron_config`` key and
|
| 55 |
+
converted back by :meth:`to_kron_config`.
|
| 56 |
+
**kwargs: Standard ``PretrainedConfig`` keyword arguments.
|
| 57 |
+
"""
|
| 58 |
+
|
| 59 |
+
model_type = "kronllm"
|
| 60 |
+
|
| 61 |
+
def __init__(self, kron_config: Optional[Dict[str, Any]] = None, **kwargs: Any) -> None:
|
| 62 |
+
super().__init__(**kwargs)
|
| 63 |
+
# ``PretrainedConfig.to_diff_dict`` instantiates the class without
|
| 64 |
+
# arguments, so an empty config must be constructible; the model
|
| 65 |
+
# wrapper rejects it later with a clearer error.
|
| 66 |
+
self.kron_config = copy.deepcopy(kron_config) if kron_config else {}
|
| 67 |
+
|
| 68 |
+
data = self.kron_config.get("data", {})
|
| 69 |
+
eot = data.get("eot_id")
|
| 70 |
+
if getattr(self, "vocab_size", None) is None:
|
| 71 |
+
self.vocab_size = data.get("vocab_size")
|
| 72 |
+
if self.eos_token_id is None:
|
| 73 |
+
self.eos_token_id = eot
|
| 74 |
+
if self.pad_token_id is None:
|
| 75 |
+
self.pad_token_id = eot
|
| 76 |
+
self.max_position_embeddings = int(data.get("seq_len", 2048))
|
| 77 |
+
self.architectures = ["KronForCausalLM"]
|
| 78 |
+
# The model has no KV cache; telling GenerationMixin this keeps the
|
| 79 |
+
# sampling loop from passing past_key_values it cannot use.
|
| 80 |
+
self.use_cache = False
|
| 81 |
+
|
| 82 |
+
@classmethod
|
| 83 |
+
def from_kron_config(cls, cfg: Dict[str, Any], **kwargs: Any) -> "KronConfig":
|
| 84 |
+
"""Build a config from a plain nested KronLLM config dictionary."""
|
| 85 |
+
return cls(kron_config=cfg, **kwargs)
|
| 86 |
+
|
| 87 |
+
def to_kron_config(self) -> Dict[str, Any]:
|
| 88 |
+
"""Return a deep copy of the nested config expected by the model.
|
| 89 |
+
|
| 90 |
+
Raises:
|
| 91 |
+
ValueError: If the config was created without a ``kron_config``.
|
| 92 |
+
"""
|
| 93 |
+
if not self.kron_config:
|
| 94 |
+
raise ValueError("KronConfig carries no 'kron_config' dictionary")
|
| 95 |
+
return copy.deepcopy(self.kron_config)
|
| 96 |
+
|
| 97 |
+
|
| 98 |
+
class KronForCausalLM(PreTrainedModel, GenerationMixin):
|
| 99 |
+
"""A ``PreTrainedModel`` wrapper around :class:`kronllm.model.KronTransformer`.
|
| 100 |
+
|
| 101 |
+
The wrapped network lives in the ``kron`` attribute, so a plain checkpoint
|
| 102 |
+
saved by ``kronllm.train`` can be loaded with
|
| 103 |
+
:meth:`from_kron_checkpoint`. Weights saved with ``save_pretrained`` keep
|
| 104 |
+
the same ``kron.*`` key prefix and reload through the usual
|
| 105 |
+
``from_pretrained`` path.
|
| 106 |
+
"""
|
| 107 |
+
|
| 108 |
+
config_class = KronConfig
|
| 109 |
+
base_model_prefix = "kron"
|
| 110 |
+
# KronTransformer ties the output head to the embedding when the embedding
|
| 111 |
+
# is a plain table; declare it so save_pretrained drops the duplicate.
|
| 112 |
+
_tied_weights_keys = ["kron.lm_head.weight"]
|
| 113 |
+
# ... and do not warn when that duplicate is absent from a checkpoint.
|
| 114 |
+
_keys_to_ignore_on_load_missing = ["kron.lm_head.weight"]
|
| 115 |
+
|
| 116 |
+
def __init__(self, config: KronConfig) -> None:
|
| 117 |
+
super().__init__(config)
|
| 118 |
+
if not config.kron_config:
|
| 119 |
+
raise ValueError("KronForCausalLM requires a KronConfig with 'kron_config'")
|
| 120 |
+
self.kron = KronTransformer(config.to_kron_config())
|
| 121 |
+
self.vocab_size = self.kron.vocab_size
|
| 122 |
+
# The inner model already initialises every module (including the
|
| 123 |
+
# orthogonal Kronecker factors), so HF's default initialiser must not
|
| 124 |
+
# touch the weights a second time.
|
| 125 |
+
self.post_init()
|
| 126 |
+
|
| 127 |
+
def _init_weights(self, module: torch.nn.Module) -> None:
|
| 128 |
+
"""Leave the weights as initialised by :class:`KronTransformer`."""
|
| 129 |
+
|
| 130 |
+
def tie_weights(self) -> None:
|
| 131 |
+
"""Re-tie the output head after Hugging Face replaces parameters.
|
| 132 |
+
|
| 133 |
+
``from_pretrained`` may assign loaded tensors with ``assign=True``,
|
| 134 |
+
which breaks the shared storage created in ``KronTransformer.__init__``
|
| 135 |
+
for the plain (non-factorised) embedding. Restoring the link keeps the
|
| 136 |
+
head and the embedding identical, as in training.
|
| 137 |
+
"""
|
| 138 |
+
kron = getattr(self, "kron", None)
|
| 139 |
+
if kron is None or getattr(kron, "embed_up", None) is not None:
|
| 140 |
+
return
|
| 141 |
+
if getattr(kron, "lm_head", None) is not None:
|
| 142 |
+
kron.lm_head.weight = kron.embed.weight
|
| 143 |
+
|
| 144 |
+
@property
|
| 145 |
+
def seq_len(self) -> int:
|
| 146 |
+
"""Maximum context length used by the sliding-window generation loop."""
|
| 147 |
+
return self.kron.seq_len
|
| 148 |
+
|
| 149 |
+
def can_generate(self) -> bool:
|
| 150 |
+
"""KronLLM supports ``generate`` even though it has no KV cache."""
|
| 151 |
+
return True
|
| 152 |
+
|
| 153 |
+
def forward(
|
| 154 |
+
self,
|
| 155 |
+
input_ids: Optional[torch.Tensor] = None,
|
| 156 |
+
attention_mask: Optional[torch.Tensor] = None,
|
| 157 |
+
labels: Optional[torch.Tensor] = None,
|
| 158 |
+
**kwargs: Any,
|
| 159 |
+
) -> CausalLMOutputWithPast:
|
| 160 |
+
"""Run the model and optionally compute the next-token loss.
|
| 161 |
+
|
| 162 |
+
Args:
|
| 163 |
+
input_ids: ``(batch, seq)`` integer token ids.
|
| 164 |
+
attention_mask: Accepted for API compatibility; only an all-ones
|
| 165 |
+
mask is supported (the model has no padding-aware attention).
|
| 166 |
+
labels: ``(batch, seq)`` next-token targets. Positions equal to
|
| 167 |
+
``-100`` are ignored. The labels are shifted internally, as in
|
| 168 |
+
every Hugging Face causal language model.
|
| 169 |
+
|
| 170 |
+
Returns:
|
| 171 |
+
A :class:`CausalLMOutputWithPast` with ``logits`` of shape
|
| 172 |
+
``(batch, seq, vocab_size)`` and, when ``labels`` is given, the
|
| 173 |
+
scalar cross-entropy ``loss``.
|
| 174 |
+
"""
|
| 175 |
+
if input_ids is None:
|
| 176 |
+
raise ValueError("input_ids is required")
|
| 177 |
+
|
| 178 |
+
out = self.kron(input_ids)
|
| 179 |
+
logits = out["logits"]
|
| 180 |
+
|
| 181 |
+
loss = None
|
| 182 |
+
if labels is not None:
|
| 183 |
+
shift_logits = logits[:, :-1, :].contiguous()
|
| 184 |
+
shift_labels = labels[:, 1:].contiguous()
|
| 185 |
+
loss = F.cross_entropy(
|
| 186 |
+
shift_logits.view(-1, shift_logits.size(-1)),
|
| 187 |
+
shift_labels.view(-1),
|
| 188 |
+
ignore_index=-100,
|
| 189 |
+
)
|
| 190 |
+
|
| 191 |
+
return CausalLMOutputWithPast(loss=loss, logits=logits)
|
| 192 |
+
|
| 193 |
+
def prepare_inputs_for_generation(self, input_ids: torch.Tensor,
|
| 194 |
+
**kwargs: Any) -> Dict[str, torch.Tensor]:
|
| 195 |
+
"""Recompute from a window of at most ``seq_len`` tokens.
|
| 196 |
+
|
| 197 |
+
KronLLM has no KV cache and was trained on fixed-length blocks, so the
|
| 198 |
+
sampling loop mirrors :meth:`KronTransformer.generate` and slides a
|
| 199 |
+
window over the context.
|
| 200 |
+
"""
|
| 201 |
+
if input_ids.shape[1] > self.seq_len:
|
| 202 |
+
input_ids = input_ids[:, -self.seq_len:]
|
| 203 |
+
return {"input_ids": input_ids}
|
| 204 |
+
|
| 205 |
+
@classmethod
|
| 206 |
+
def from_kron_checkpoint(cls, path: str | Path,
|
| 207 |
+
**kwargs: Any) -> "KronForCausalLM":
|
| 208 |
+
"""Load a checkpoint written by ``kronllm.train`` or ``quantize``.
|
| 209 |
+
|
| 210 |
+
Args:
|
| 211 |
+
path: Path to ``best.pt`` / ``last.pt`` / ``model_int8.pt``.
|
| 212 |
+
**kwargs: Extra arguments for ``KronConfig``.
|
| 213 |
+
|
| 214 |
+
Returns:
|
| 215 |
+
An evaluation-mode :class:`KronForCausalLM` on CPU; call ``.to()``
|
| 216 |
+
to move it to a GPU.
|
| 217 |
+
"""
|
| 218 |
+
cfg, state = load_any_checkpoint(path)
|
| 219 |
+
if cfg is None:
|
| 220 |
+
raise ValueError(f"{path} carries no config; pass a KronConfig explicitly")
|
| 221 |
+
model = cls(KronConfig.from_kron_config(cfg, **kwargs))
|
| 222 |
+
model.kron.load_state_dict(state)
|
| 223 |
+
model.eval()
|
| 224 |
+
return model
|
| 225 |
+
|
| 226 |
+
|
| 227 |
+
# Make the AutoClasses work as soon as this module is imported.
|
| 228 |
+
AutoConfig.register(KronConfig.model_type, KronConfig)
|
| 229 |
+
AutoModelForCausalLM.register(KronConfig, KronForCausalLM)
|
kronllm/kron.py
ADDED
|
@@ -0,0 +1,854 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""KronCompose: the Kronecker-parameterised linear map with depth recurrence.
|
| 2 |
+
|
| 3 |
+
Core object of the project. A ``KronCompose`` layer implements
|
| 4 |
+
|
| 5 |
+
I_{i,l} = sum_{k=1..K} A_{i,l,k} (x) B_{i,l,k} (innovation)
|
| 6 |
+
D_{i,l} = I_{i,l} + sigma(theta_{i,l}) * sg(D_{i-1,l}) (depth recurrence)
|
| 7 |
+
W_{i,l} = s * Id + tau * D_{i,l} (anchor)
|
| 8 |
+
P_i = W_{i,L} ( ... ( W_{i,1} ) ) (L-fold composition)
|
| 9 |
+
y = gamma_i * P_i x (ReZero gate)
|
| 10 |
+
|
| 11 |
+
where ``i`` is the *model depth* (transformer layer index), ``l`` the
|
| 12 |
+
micro-composition channel and ``sg`` the stop-gradient operator.
|
| 13 |
+
|
| 14 |
+
Scale handling (deviation from the original TZ, see README)
|
| 15 |
+
-----------------------------------------------------------
|
| 16 |
+
The TZ fixed the orthogonal gain of ``A, B`` to ``1/sqrt(n1*n2) = 1/sqrt(d)``.
|
| 17 |
+
That makes ``||I|| ~ 1/d`` -- 128x below the ``s*Id`` anchor at d=128 -- so
|
| 18 |
+
``dL/dA``, ``dL/dB`` and ``dL/dtheta`` all vanish and the model is numerically
|
| 19 |
+
the identity map (measured: ``theta_grad = 0``, ``eff_rank(W) = d``,
|
| 20 |
+
``cos(W_i, W_{i-1}) = 1``), i.e. a context-free unigram model.
|
| 21 |
+
|
| 22 |
+
Here the innovation is **normalised** before it is used
|
| 23 |
+
(``D <- tau * D / rms(D)``), which makes the gain irrelevant and replaces it
|
| 24 |
+
with two explicit, learnable magnitudes:
|
| 25 |
+
|
| 26 |
+
* ``tau`` -- per-channel innovation scale (init ``innovation_scale``),
|
| 27 |
+
* ``gamma`` -- per-compose ReZero gate (init ``gate_init``), so a branch can
|
| 28 |
+
start as a small perturbation of the residual stream instead of forcing
|
| 29 |
+
``||W|| ~ 1`` at initialisation.
|
| 30 |
+
|
| 31 |
+
Kronecker convention
|
| 32 |
+
--------------------
|
| 33 |
+
``A`` is ``(n1_out, n1_in)``, ``B`` is ``(n2_out, n2_in)`` and ``x`` is viewed
|
| 34 |
+
as ``(n1_in, n2_in)``. Then
|
| 35 |
+
|
| 36 |
+
(A (x) B) x == A @ X @ B.T
|
| 37 |
+
|
| 38 |
+
with ``X = x.reshape(n1_in, n2_in)`` and row-major flattening of the result.
|
| 39 |
+
This is what ``kron_apply`` implements and what test T1 checks.
|
| 40 |
+
"""
|
| 41 |
+
from __future__ import annotations
|
| 42 |
+
|
| 43 |
+
import math
|
| 44 |
+
from typing import Any, Dict, List, Optional, Sequence, Tuple
|
| 45 |
+
|
| 46 |
+
import torch
|
| 47 |
+
import torch.nn as nn
|
| 48 |
+
import torch.nn.functional as F
|
| 49 |
+
from torch.utils.checkpoint import checkpoint
|
| 50 |
+
|
| 51 |
+
|
| 52 |
+
# ---------------------------------------------------------------------------
|
| 53 |
+
# primitives
|
| 54 |
+
# ---------------------------------------------------------------------------
|
| 55 |
+
def kron_apply(A: torch.Tensor, B: torch.Tensor, x: torch.Tensor) -> torch.Tensor:
|
| 56 |
+
"""Apply ``(A (x) B)`` to ``x`` without materialising the Kronecker product.
|
| 57 |
+
|
| 58 |
+
Parameters
|
| 59 |
+
----------
|
| 60 |
+
A : (n1_out, n1_in)
|
| 61 |
+
B : (n2_out, n2_in)
|
| 62 |
+
x : (..., n1_in, n2_in)
|
| 63 |
+
|
| 64 |
+
Returns
|
| 65 |
+
-------
|
| 66 |
+
(..., n1_out, n2_out)
|
| 67 |
+
"""
|
| 68 |
+
if A.shape[0] == A.shape[1] and B.shape[0] == B.shape[1]:
|
| 69 |
+
return torch.matmul(torch.matmul(A, x), B.transpose(-1, -2))
|
| 70 |
+
return torch.einsum("ij,...jl,ml->...im", A, x, B)
|
| 71 |
+
|
| 72 |
+
|
| 73 |
+
def kron_materialize(A: torch.Tensor, B: torch.Tensor) -> torch.Tensor:
|
| 74 |
+
"""Dense matrix of ``A (x) B`` with row-major index order."""
|
| 75 |
+
n1o, n1i = A.shape
|
| 76 |
+
n2o, n2i = B.shape
|
| 77 |
+
return torch.einsum("ij,ml->imjl", A, B).reshape(n1o * n2o, n1i * n2i)
|
| 78 |
+
|
| 79 |
+
|
| 80 |
+
def effective_rank(singular_values: torch.Tensor, eps: float = 1e-12) -> torch.Tensor:
|
| 81 |
+
"""``exp(entropy(sigma / sum(sigma)))`` -- a smooth rank proxy."""
|
| 82 |
+
s = singular_values.clamp_min(eps)
|
| 83 |
+
p = s / s.sum()
|
| 84 |
+
return torch.exp(-(p * p.log()).sum())
|
| 85 |
+
|
| 86 |
+
|
| 87 |
+
def rope_frequencies(seq_len: int, d_head: int, theta: float,
|
| 88 |
+
device: Optional[torch.device] = None,
|
| 89 |
+
dtype: torch.dtype = torch.float32
|
| 90 |
+
) -> Tuple[torch.Tensor, torch.Tensor]:
|
| 91 |
+
inv_freq = 1.0 / (theta ** (torch.arange(0, d_head, 2, device=device,
|
| 92 |
+
dtype=torch.float32) / d_head))
|
| 93 |
+
pos = torch.arange(seq_len, device=device, dtype=torch.float32)
|
| 94 |
+
freqs = torch.outer(pos, inv_freq) # (T, d_head/2)
|
| 95 |
+
return freqs.cos().to(dtype), freqs.sin().to(dtype)
|
| 96 |
+
|
| 97 |
+
|
| 98 |
+
def apply_rope(x: torch.Tensor, cos: torch.Tensor, sin: torch.Tensor) -> torch.Tensor:
|
| 99 |
+
"""Rotary positional embedding. ``x``: (B, H, T, d_head)."""
|
| 100 |
+
x1, x2 = x[..., 0::2], x[..., 1::2]
|
| 101 |
+
out = torch.stack((x1 * cos - x2 * sin, x1 * sin + x2 * cos), dim=-1)
|
| 102 |
+
return out.flatten(-2)
|
| 103 |
+
|
| 104 |
+
|
| 105 |
+
_EYE_CACHE: dict = {}
|
| 106 |
+
_STATE0_CACHE: dict = {}
|
| 107 |
+
|
| 108 |
+
|
| 109 |
+
def cached_eye(d: int, device, dtype) -> torch.Tensor:
|
| 110 |
+
key = (d, str(device), dtype)
|
| 111 |
+
eye = _EYE_CACHE.get(key)
|
| 112 |
+
if eye is None:
|
| 113 |
+
eye = torch.eye(d, device=device, dtype=dtype)
|
| 114 |
+
_EYE_CACHE[key] = eye
|
| 115 |
+
return eye
|
| 116 |
+
|
| 117 |
+
|
| 118 |
+
def rms_norm_matrix(x: torch.Tensor, eps: float = 1e-6) -> torch.Tensor:
|
| 119 |
+
"""Normalise every matrix in the trailing two dims to unit RMS."""
|
| 120 |
+
r = x.pow(2).mean(dim=(-2, -1), keepdim=True).clamp_min(eps).sqrt()
|
| 121 |
+
return x / r
|
| 122 |
+
|
| 123 |
+
|
| 124 |
+
def _init_gain_value(mode: "str | float", K: int, d_in: int) -> float:
|
| 125 |
+
if mode == "unit":
|
| 126 |
+
return float(K) ** -0.25
|
| 127 |
+
if mode == "spec":
|
| 128 |
+
return 1.0 / math.sqrt(max(d_in, 1))
|
| 129 |
+
return float(mode)
|
| 130 |
+
|
| 131 |
+
|
| 132 |
+
# ---------------------------------------------------------------------------
|
| 133 |
+
# KronCompose
|
| 134 |
+
# ---------------------------------------------------------------------------
|
| 135 |
+
def _largest_factor(n: int) -> int:
|
| 136 |
+
"""Largest divisor of ``n`` not exceeding ``sqrt(n)`` (≥ 1)."""
|
| 137 |
+
for k in range(int(n ** 0.5), 0, -1):
|
| 138 |
+
if n % k == 0:
|
| 139 |
+
return k
|
| 140 |
+
return 1
|
| 141 |
+
|
| 142 |
+
|
| 143 |
+
class KronCompose(nn.Module):
|
| 144 |
+
"""Kronecker-parameterised linear layer with depth recurrence.
|
| 145 |
+
|
| 146 |
+
The depth state ``D`` is *not* owned by the module: it is passed in and
|
| 147 |
+
returned so that the same parameters can be reused at every transformer
|
| 148 |
+
layer while ``D`` keeps evolving along depth (one state per channel and
|
| 149 |
+
per matrix role).
|
| 150 |
+
"""
|
| 151 |
+
|
| 152 |
+
def __init__(
|
| 153 |
+
self,
|
| 154 |
+
d_in: int,
|
| 155 |
+
d_out: Optional[int] = None,
|
| 156 |
+
n1_in: int = 8,
|
| 157 |
+
n2_in: int = 16,
|
| 158 |
+
n1_out: Optional[int] = None,
|
| 159 |
+
n2_out: Optional[int] = None,
|
| 160 |
+
L: int = 4,
|
| 161 |
+
K: int = 2,
|
| 162 |
+
K_win: int = 2,
|
| 163 |
+
s_init: float = 1.0,
|
| 164 |
+
s_learnable: bool = False,
|
| 165 |
+
anchor: bool = True,
|
| 166 |
+
stop_gradient: bool = True,
|
| 167 |
+
theta_std: float = 0.1,
|
| 168 |
+
rezero_eps: float = 0.0,
|
| 169 |
+
s_init_effective: bool = True,
|
| 170 |
+
init_gain: "str | float" = "unit",
|
| 171 |
+
innovation_scale: float = 0.1,
|
| 172 |
+
normalize_D: bool = True,
|
| 173 |
+
gate_init: float = 0.1,
|
| 174 |
+
use_gate: bool = True,
|
| 175 |
+
composition: str = "product",
|
| 176 |
+
sum_alpha_init: float = 1.0,
|
| 177 |
+
dynamic_gate: Optional[str] = None,
|
| 178 |
+
groups: int = 1,
|
| 179 |
+
alternate_start: str = "sum",
|
| 180 |
+
nonlinearity: str = "silu",
|
| 181 |
+
norm_between: bool = False,
|
| 182 |
+
nonlinear_every: int = 1,
|
| 183 |
+
nonlinear_mode: str = "replace",
|
| 184 |
+
sigma_mode: str = "learned",
|
| 185 |
+
sigma_value: float = 0.5,
|
| 186 |
+
) -> None:
|
| 187 |
+
super().__init__()
|
| 188 |
+
d_out = d_in if d_out is None else d_out
|
| 189 |
+
n1_out = n1_in if n1_out is None else n1_out
|
| 190 |
+
n2_out = n2_in if n2_out is None else n2_out
|
| 191 |
+
|
| 192 |
+
assert n1_in * n2_in == d_in, (n1_in, n2_in, d_in)
|
| 193 |
+
assert n1_out * n2_out == d_out, (n1_out, n2_out, d_out)
|
| 194 |
+
|
| 195 |
+
self.d_in, self.d_out = d_in, d_out
|
| 196 |
+
self.n1_in, self.n2_in = n1_in, n2_in
|
| 197 |
+
self.n1_out, self.n2_out = n1_out, n2_out
|
| 198 |
+
self.L, self.K, self.K_win = L, K, K_win
|
| 199 |
+
self.init_gain = init_gain
|
| 200 |
+
self.square = d_in == d_out
|
| 201 |
+
# L-fold composition is only defined for square maps.
|
| 202 |
+
assert self.square or L == 1, "non-square KronCompose requires L == 1"
|
| 203 |
+
|
| 204 |
+
self.composition = str(composition).lower()
|
| 205 |
+
if self.composition not in {"product", "sum", "grouped", "alternating",
|
| 206 |
+
"nonlinear"}:
|
| 207 |
+
raise ValueError(
|
| 208 |
+
"composition must be 'product', 'sum', 'grouped', "
|
| 209 |
+
f"'alternating' or 'nonlinear', got {composition!r}")
|
| 210 |
+
self.nonlinearity = str(nonlinearity).lower()
|
| 211 |
+
if self.nonlinearity not in {"silu", "gelu", "tanh"}:
|
| 212 |
+
raise ValueError("nonlinearity must be silu/gelu/tanh")
|
| 213 |
+
self.norm_between = bool(norm_between)
|
| 214 |
+
self.nonlinear_every = int(nonlinear_every)
|
| 215 |
+
if not 1 <= self.nonlinear_every <= L:
|
| 216 |
+
raise ValueError("nonlinear_every must be in [1, L]")
|
| 217 |
+
self.nonlinear_mode = str(nonlinear_mode).lower()
|
| 218 |
+
if self.nonlinear_mode not in {"replace", "add"}:
|
| 219 |
+
raise ValueError("nonlinear_mode must be 'replace' or 'add'")
|
| 220 |
+
self.groups = int(groups)
|
| 221 |
+
if self.composition == "grouped":
|
| 222 |
+
if self.groups < 1 or L % self.groups != 0:
|
| 223 |
+
raise ValueError(
|
| 224 |
+
f"groups={groups} must divide L={L}")
|
| 225 |
+
else:
|
| 226 |
+
self.groups = 1
|
| 227 |
+
self.alternate_start = str(alternate_start).lower()
|
| 228 |
+
if self.alternate_start not in {"sum", "product"}:
|
| 229 |
+
raise ValueError("alternate_start must be 'sum' or 'product'")
|
| 230 |
+
self.sum_alpha_init = float(sum_alpha_init)
|
| 231 |
+
self.dynamic_gate = (str(dynamic_gate).lower() if dynamic_gate else None)
|
| 232 |
+
if self.dynamic_gate is not None:
|
| 233 |
+
if self.dynamic_gate not in {"dense", "kron"}:
|
| 234 |
+
raise ValueError(
|
| 235 |
+
"dynamic_gate must be None/'dense'/'kron', got "
|
| 236 |
+
f"{dynamic_gate!r}")
|
| 237 |
+
if self.composition != "sum":
|
| 238 |
+
raise ValueError("dynamic_gate requires composition='sum'")
|
| 239 |
+
if L < 2:
|
| 240 |
+
raise ValueError("dynamic_gate requires L >= 2")
|
| 241 |
+
|
| 242 |
+
self.anchor = bool(anchor) and self.square
|
| 243 |
+
self.stop_gradient = stop_gradient
|
| 244 |
+
self.rezero_eps = rezero_eps
|
| 245 |
+
self.normalize_D = bool(normalize_D)
|
| 246 |
+
self.innovation_scale = float(innovation_scale)
|
| 247 |
+
# Depth-recurrence strength: sigmoid(theta) by default. "off" removes
|
| 248 |
+
# the carry (independent factors), "fixed" pins it to sigma_value and
|
| 249 |
+
# freezes theta. Both exist as ablations, not as design choices.
|
| 250 |
+
self.sigma_mode = str(sigma_mode).lower()
|
| 251 |
+
if self.sigma_mode not in {"learned", "fixed", "off"}:
|
| 252 |
+
raise ValueError("sigma_mode must be 'learned', 'fixed' or 'off'")
|
| 253 |
+
self.sigma_value = float(sigma_value)
|
| 254 |
+
|
| 255 |
+
self.A = nn.Parameter(torch.empty(L, K, n1_out, n1_in))
|
| 256 |
+
self.B = nn.Parameter(torch.empty(L, K, n2_out, n2_in))
|
| 257 |
+
self.theta = nn.Parameter(torch.randn(L) * theta_std)
|
| 258 |
+
# explicit innovation magnitude (replaces the implicit factor gain)
|
| 259 |
+
self.tau = nn.Parameter(torch.full((L,), float(innovation_scale)))
|
| 260 |
+
# ReZero gate: the branch starts as a small perturbation
|
| 261 |
+
self.gate = nn.Parameter(torch.tensor(float(gate_init))) if use_gate else None
|
| 262 |
+
# Per-channel mixing coefficients for the sum composition. The
|
| 263 |
+
# product composition keeps ``alpha=None`` so old checkpoints and the
|
| 264 |
+
# parameter count of the product runs stay unchanged.
|
| 265 |
+
self.alpha = (nn.Parameter(torch.full((L,), self.sum_alpha_init))
|
| 266 |
+
if self.composition in {"sum", "grouped", "alternating"}
|
| 267 |
+
else None)
|
| 268 |
+
|
| 269 |
+
# Input-dependent softmax gate over the L sum-composition terms:
|
| 270 |
+
# w_t = L * softmax(g(x_t)), P_t = sI + sum_l w_{t,l} D_l.
|
| 271 |
+
# The L*softmax normalisation keeps the mean weight at 1, so at
|
| 272 |
+
# initialisation (zero gate) it is exactly the static alpha=1 sum.
|
| 273 |
+
if self.dynamic_gate == "dense":
|
| 274 |
+
self.w_gate = nn.Linear(d_in, L, bias=True)
|
| 275 |
+
nn.init.zeros_(self.w_gate.weight)
|
| 276 |
+
nn.init.zeros_(self.w_gate.bias)
|
| 277 |
+
elif self.dynamic_gate == "kron":
|
| 278 |
+
g1 = _largest_factor(L)
|
| 279 |
+
self.gate_shape = (g1, L // g1)
|
| 280 |
+
self.w_gate_A = nn.Parameter(torch.zeros(g1, n1_in))
|
| 281 |
+
self.w_gate_B = nn.Parameter(torch.zeros(L // g1, n2_in))
|
| 282 |
+
self.w_gate_b = nn.Parameter(torch.zeros(L))
|
| 283 |
+
if self.dynamic_gate is not None:
|
| 284 |
+
self.register_buffer("_last_w_mean", torch.ones(L),
|
| 285 |
+
persistent=False)
|
| 286 |
+
|
| 287 |
+
if self.anchor:
|
| 288 |
+
# A sum has no L-fold product: s_init is already the coefficient of
|
| 289 |
+
# the identity term. A grouped composition is a product of G sums,
|
| 290 |
+
# so s_init is also the *effective* anchor (each group gets its
|
| 291 |
+
# G-th root at composition time).
|
| 292 |
+
if self.composition in {"sum", "grouped", "alternating"}:
|
| 293 |
+
s0 = float(s_init)
|
| 294 |
+
else:
|
| 295 |
+
s0 = (float(s_init) ** (1.0 / L)
|
| 296 |
+
if s_init_effective else float(s_init))
|
| 297 |
+
if s_learnable:
|
| 298 |
+
self.s = nn.Parameter(torch.tensor(s0))
|
| 299 |
+
else:
|
| 300 |
+
self.register_buffer("s", torch.tensor(s0))
|
| 301 |
+
else:
|
| 302 |
+
self.s = None
|
| 303 |
+
|
| 304 |
+
self.reset_parameters()
|
| 305 |
+
|
| 306 |
+
# -- init -------------------------------------------------------------
|
| 307 |
+
def reset_parameters(self) -> None:
|
| 308 |
+
"""Orthogonal factors.
|
| 309 |
+
|
| 310 |
+
With ``normalize_D`` the gain is irrelevant for the forward pass (it is
|
| 311 |
+
normalised away); ``init_gain`` is kept so the TZ recipe can be
|
| 312 |
+
reproduced for the ablation table.
|
| 313 |
+
"""
|
| 314 |
+
gain = _init_gain_value(self.init_gain, self.K, self.d_in)
|
| 315 |
+
nn.init.orthogonal_(self.A, gain=gain)
|
| 316 |
+
nn.init.orthogonal_(self.B, gain=gain)
|
| 317 |
+
with torch.no_grad():
|
| 318 |
+
self.theta.normal_(0.0, 0.1)
|
| 319 |
+
self.tau.fill_(self.innovation_scale)
|
| 320 |
+
if self.alpha is not None:
|
| 321 |
+
self.alpha.fill_(self.sum_alpha_init)
|
| 322 |
+
|
| 323 |
+
def sigma_values(self) -> torch.Tensor:
|
| 324 |
+
"""Depth-carry coefficient for each composition step, shape ``(L,)``.
|
| 325 |
+
|
| 326 |
+
``learned`` uses ``sigmoid(theta)``; ``fixed`` pins the value (the
|
| 327 |
+
critic's "sigma frozen at 0.5" ablation); ``off`` removes the carry so
|
| 328 |
+
the factors become independent. ``theta`` is kept in the state dict in
|
| 329 |
+
every mode, so the parameter count is unchanged across the ablation.
|
| 330 |
+
"""
|
| 331 |
+
if self.sigma_mode == "off":
|
| 332 |
+
return torch.zeros(self.L, device=self.theta.device,
|
| 333 |
+
dtype=self.theta.dtype)
|
| 334 |
+
if self.sigma_mode == "fixed":
|
| 335 |
+
return torch.full((self.L,), self.sigma_value,
|
| 336 |
+
device=self.theta.device, dtype=self.theta.dtype)
|
| 337 |
+
return torch.sigmoid(self.theta)
|
| 338 |
+
|
| 339 |
+
@property
|
| 340 |
+
def n_params(self) -> int:
|
| 341 |
+
total = self.A.numel() + self.B.numel() + self.theta.numel() + self.tau.numel()
|
| 342 |
+
if isinstance(self.s, nn.Parameter):
|
| 343 |
+
total += 1
|
| 344 |
+
if self.gate is not None:
|
| 345 |
+
total += 1
|
| 346 |
+
if self.alpha is not None:
|
| 347 |
+
total += self.alpha.numel()
|
| 348 |
+
if self.dynamic_gate == "dense":
|
| 349 |
+
total += sum(p.numel() for p in self.w_gate.parameters())
|
| 350 |
+
elif self.dynamic_gate == "kron":
|
| 351 |
+
total += (self.w_gate_A.numel() + self.w_gate_B.numel()
|
| 352 |
+
+ self.w_gate_b.numel())
|
| 353 |
+
return total
|
| 354 |
+
|
| 355 |
+
# -- state ------------------------------------------------------------
|
| 356 |
+
def initial_state(self, device=None, dtype=torch.float32) -> torch.Tensor:
|
| 357 |
+
"""``D_{-1,l} = eps * Id`` (small start at a window boundary).
|
| 358 |
+
|
| 359 |
+
Cached: the tensor is constant and never mutated (the recurrence
|
| 360 |
+
creates a new tensor every layer).
|
| 361 |
+
"""
|
| 362 |
+
key = (self.L, self.d_out, self.d_in, self.rezero_eps,
|
| 363 |
+
str(device), dtype, self.square)
|
| 364 |
+
cached = _STATE0_CACHE.get(key)
|
| 365 |
+
if cached is None:
|
| 366 |
+
D = torch.zeros(self.L, self.d_out, self.d_in, device=device,
|
| 367 |
+
dtype=dtype)
|
| 368 |
+
if self.square and self.rezero_eps:
|
| 369 |
+
D = D + self.rezero_eps * cached_eye(self.d_in, device, dtype)
|
| 370 |
+
_STATE0_CACHE[key] = D
|
| 371 |
+
cached = D
|
| 372 |
+
return cached
|
| 373 |
+
|
| 374 |
+
def innovation(self) -> torch.Tensor:
|
| 375 |
+
"""``I_l = sum_k A_k (x) B_k`` as dense ``(L, d_out, d_in)``.
|
| 376 |
+
|
| 377 |
+
Explicit broadcast outer product instead of a 5-index einsum: the
|
| 378 |
+
contraction is tiny, but einsum's path selection made this one of the
|
| 379 |
+
most expensive ops in the forward pass.
|
| 380 |
+
"""
|
| 381 |
+
L, K, n1o, n1i = self.A.shape
|
| 382 |
+
_, _, n2o, n2i = self.B.shape
|
| 383 |
+
outer = (self.A.unsqueeze(-1).unsqueeze(-1)
|
| 384 |
+
* self.B.unsqueeze(2).unsqueeze(2)) # (L,K,n1o,n1i,n2o,n2i)
|
| 385 |
+
I4 = outer.sum(dim=1) # (L,n1o,n1i,n2o,n2i)
|
| 386 |
+
return I4.permute(0, 1, 3, 2, 4).reshape(
|
| 387 |
+
self.L, n1o * n2o, n1i * n2i)
|
| 388 |
+
|
| 389 |
+
def apply_innovation(self, x: torch.Tensor, l: int = 0) -> torch.Tensor:
|
| 390 |
+
"""Apply ``I_l = sum_k A_k (x) B_k`` to ``x`` without building ``d x d``.
|
| 391 |
+
|
| 392 |
+
Uses ``(A (x) B) vec(X) = A X B^T`` per summand, i.e.
|
| 393 |
+
``O(K * (n1_out*n1_in*n2_in + n2_out*n2_in*n1_out))`` instead of the
|
| 394 |
+
dense ``O(d_in*d_out)``. With square factors ``n1=n2=sqrt(d)`` the
|
| 395 |
+
saving per summand is ``O(d^1.5)``; with ``K`` summands the crossover is
|
| 396 |
+
``K * 2 * d^1.5 < d^2`` (K=6, d=320: ~1.5x). The depth recurrence
|
| 397 |
+
still carries a dense ``prev`` term, so the full ``D_l`` application is
|
| 398 |
+
not cheaper unless the state is factorised too -- see the benchmark in
|
| 399 |
+
docs/architecture.md.
|
| 400 |
+
"""
|
| 401 |
+
tokens = x.shape[0]
|
| 402 |
+
X = x.reshape(tokens, self.n1_in, self.n2_in)
|
| 403 |
+
y = torch.zeros(tokens, self.n1_out, self.n2_out,
|
| 404 |
+
device=x.device, dtype=x.dtype)
|
| 405 |
+
for k in range(self.K):
|
| 406 |
+
y = y + torch.einsum("ip,bpq,jq->bij", self.A[l, k], X,
|
| 407 |
+
self.B[l, k])
|
| 408 |
+
return y.reshape(tokens, self.d_out)
|
| 409 |
+
|
| 410 |
+
# -- composition ------------------------------------------------------
|
| 411 |
+
def compose_product(self, D: torch.Tensor) -> torch.Tensor:
|
| 412 |
+
"""``P = W_L ... W_1`` with ``W_l = s Id + D_l``."""
|
| 413 |
+
s = self.s
|
| 414 |
+
if s is None:
|
| 415 |
+
P = D[0]
|
| 416 |
+
for l in range(1, self.L):
|
| 417 |
+
P = torch.matmul(D[l], P)
|
| 418 |
+
return P
|
| 419 |
+
sI = s * cached_eye(self.d_in, D.device, D.dtype)
|
| 420 |
+
P = sI + D[0]
|
| 421 |
+
for l in range(1, self.L):
|
| 422 |
+
# (s Id + D_l) @ P = s*P + D_l @ P
|
| 423 |
+
P = torch.addmm(s * P, D[l], P)
|
| 424 |
+
return P
|
| 425 |
+
|
| 426 |
+
def compose_sum(self, D: torch.Tensor,
|
| 427 |
+
weights: Optional[torch.Tensor] = None) -> torch.Tensor:
|
| 428 |
+
"""``P = s Id + sum_l w_l D_l`` (sum composition).
|
| 429 |
+
|
| 430 |
+
``w`` is ``alpha`` for the static sum, or the last mean softmax gate
|
| 431 |
+
weights for the dynamic variant (diagnostics only; the forward pass
|
| 432 |
+
applies the per-token weights directly).
|
| 433 |
+
"""
|
| 434 |
+
assert self.alpha is not None
|
| 435 |
+
w = (weights if weights is not None else
|
| 436 |
+
(self._last_w_mean if self.dynamic_gate is not None else self.alpha))
|
| 437 |
+
P = w[0] * D[0]
|
| 438 |
+
for l in range(1, self.L):
|
| 439 |
+
P = P + w[l] * D[l]
|
| 440 |
+
if self.s is not None:
|
| 441 |
+
P = P + self.s * cached_eye(self.d_in, D.device, D.dtype)
|
| 442 |
+
return P
|
| 443 |
+
|
| 444 |
+
def compose_grouped(self, D: torch.Tensor) -> torch.Tensor:
|
| 445 |
+
"""Grouped composition: sum inside groups, product across groups.
|
| 446 |
+
|
| 447 |
+
With ``L`` factors split into ``G`` groups of size ``L/G``:
|
| 448 |
+
|
| 449 |
+
S_g = s^(1/G) I + sum_{l in g} alpha_l D_l
|
| 450 |
+
P = S_G ... S_1
|
| 451 |
+
|
| 452 |
+
``G=1`` is exactly the sum composition; ``G=L`` is exactly the product
|
| 453 |
+
(with ``alpha=1``), so the family interpolates between the two. The
|
| 454 |
+
multiplicative depth is ``G-1`` instead of ``L-1``: less conditioning
|
| 455 |
+
and rank collapse than the product, more interaction than the sum.
|
| 456 |
+
"""
|
| 457 |
+
G = self.groups
|
| 458 |
+
m = self.L // G
|
| 459 |
+
sg = (self.s ** (1.0 / G)) if self.s is not None else None
|
| 460 |
+
eye = (cached_eye(self.d_in, D.device, D.dtype)
|
| 461 |
+
if sg is not None else None)
|
| 462 |
+
P: Optional[torch.Tensor] = None
|
| 463 |
+
for g in range(G):
|
| 464 |
+
start = g * m
|
| 465 |
+
S = self.alpha[start] * D[start]
|
| 466 |
+
for l in range(start + 1, start + m):
|
| 467 |
+
S = S + self.alpha[l] * D[l]
|
| 468 |
+
if sg is not None:
|
| 469 |
+
S = S + sg * eye
|
| 470 |
+
P = S if P is None else S @ P
|
| 471 |
+
assert P is not None
|
| 472 |
+
return P
|
| 473 |
+
|
| 474 |
+
def compose_alternating(self, D: torch.Tensor) -> torch.Tensor:
|
| 475 |
+
"""Alternate sum and product at every micro-depth step.
|
| 476 |
+
|
| 477 |
+
Starting from ``P = sI``, each factor applies one operation:
|
| 478 |
+
|
| 479 |
+
l even (with ``alternate_start='sum'``): P <- P + alpha_l D_l
|
| 480 |
+
l odd: P <- P @ (I + alpha_l D_l)
|
| 481 |
+
|
| 482 |
+
The anchor is applied once at the start, so at ``D=0`` the map is
|
| 483 |
+
exactly ``sI``. This is a sum-product network along depth: additive
|
| 484 |
+
steps raise the rank, multiplicative steps compose, and the
|
| 485 |
+
multiplicative depth is about ``L/2`` instead of ``L-1``.
|
| 486 |
+
"""
|
| 487 |
+
P = (self.s * cached_eye(self.d_in, D.device, D.dtype)
|
| 488 |
+
if self.s is not None
|
| 489 |
+
else torch.zeros(self.d_in, self.d_in, device=D.device,
|
| 490 |
+
dtype=D.dtype))
|
| 491 |
+
eye = cached_eye(self.d_in, D.device, D.dtype)
|
| 492 |
+
for l in range(self.L):
|
| 493 |
+
is_sum = ((l % 2 == 0) == (self.alternate_start == "sum"))
|
| 494 |
+
if is_sum:
|
| 495 |
+
P = P + self.alpha[l] * D[l]
|
| 496 |
+
else:
|
| 497 |
+
P = P @ (eye + self.alpha[l] * D[l])
|
| 498 |
+
return P
|
| 499 |
+
|
| 500 |
+
def compose(self, D: torch.Tensor) -> torch.Tensor:
|
| 501 |
+
"""Compose the depth state with the configured operation."""
|
| 502 |
+
if self.composition == "sum":
|
| 503 |
+
return self.compose_sum(D)
|
| 504 |
+
if self.composition == "grouped":
|
| 505 |
+
return self.compose_grouped(D)
|
| 506 |
+
if self.composition == "alternating":
|
| 507 |
+
return self.compose_alternating(D)
|
| 508 |
+
if self.composition == "nonlinear":
|
| 509 |
+
# No single matrix exists; return the linearised product for the
|
| 510 |
+
# spectral diagnostics only.
|
| 511 |
+
return self.compose_product(D)
|
| 512 |
+
return self.compose_product(D)
|
| 513 |
+
|
| 514 |
+
@property
|
| 515 |
+
def anchor_term(self) -> Optional[torch.Tensor]:
|
| 516 |
+
"""Identity coefficient in the composed map (``s`` or ``s**L``)."""
|
| 517 |
+
if self.s is None:
|
| 518 |
+
return None
|
| 519 |
+
if self.composition == "sum":
|
| 520 |
+
return self.s
|
| 521 |
+
if self.composition in {"grouped", "alternating"}:
|
| 522 |
+
return self.s
|
| 523 |
+
return self.s ** self.L
|
| 524 |
+
|
| 525 |
+
@torch.no_grad()
|
| 526 |
+
def materialize(self, D: torch.Tensor) -> torch.Tensor:
|
| 527 |
+
"""Dense ``(L, d, d)`` factor matrices ``W_l = s Id + D_l``."""
|
| 528 |
+
if not self.anchor:
|
| 529 |
+
return D.clone()
|
| 530 |
+
return self.s * cached_eye(self.d_in, D.device, D.dtype) + D
|
| 531 |
+
|
| 532 |
+
# -- dynamic softmax gate ---------------------------------------------
|
| 533 |
+
def _gate_weights(self, xf: torch.Tensor) -> torch.Tensor:
|
| 534 |
+
"""Per-token weights ``L * softmax(g(x))``; mean 1, uniform at init."""
|
| 535 |
+
if self.dynamic_gate == "dense":
|
| 536 |
+
logits = self.w_gate(xf)
|
| 537 |
+
else:
|
| 538 |
+
xm = xf.reshape(xf.shape[0], self.n1_in, self.n2_in)
|
| 539 |
+
z = torch.einsum("bpq,ip,jq->bij", xm,
|
| 540 |
+
self.w_gate_A, self.w_gate_B)
|
| 541 |
+
logits = z.reshape(xf.shape[0], self.L) + self.w_gate_b
|
| 542 |
+
return self.L * torch.softmax(logits, dim=-1)
|
| 543 |
+
|
| 544 |
+
def _dynamic_sum(self, xf: torch.Tensor, D: torch.Tensor,
|
| 545 |
+
s: torch.Tensor) -> torch.Tensor:
|
| 546 |
+
"""``y = s x + sum_l w_l(x) (x D_l^T)`` (checkpointed unit)."""
|
| 547 |
+
w = self._gate_weights(xf)
|
| 548 |
+
y = xf * s
|
| 549 |
+
for l in range(self.L):
|
| 550 |
+
y = y + w[:, l:l + 1] * torch.matmul(xf, D[l].transpose(0, 1))
|
| 551 |
+
return y
|
| 552 |
+
|
| 553 |
+
def _nonlinear_forward(self, xf: torch.Tensor, D: torch.Tensor,
|
| 554 |
+
s: torch.Tensor) -> torch.Tensor:
|
| 555 |
+
"""Nonlinearity every ``nonlinear_every`` factors.
|
| 556 |
+
|
| 557 |
+
Linear factors inside a block are still fused into one matrix, so the
|
| 558 |
+
cost is ``ceil(L / k)`` GEMMs per token instead of ``L``. ``k=L`` is a
|
| 559 |
+
single nonlinearity after the whole composition (almost free).
|
| 560 |
+
|
| 561 |
+
``nonlinear_mode="replace"`` maps the block to ``y + act(P y - a y)``,
|
| 562 |
+
which cannot reproduce the pure linear map. ``"add"`` keeps the linear
|
| 563 |
+
skeleton and adds the activated deviation as a residual branch,
|
| 564 |
+
``P y + act(P y - a y)``, so the product composition is the limit of
|
| 565 |
+
vanishing ``D`` and the nonlinearity can only add capacity.
|
| 566 |
+
"""
|
| 567 |
+
y = xf
|
| 568 |
+
k = self.nonlinear_every
|
| 569 |
+
eye = cached_eye(self.d_in, xf.device, xf.dtype)
|
| 570 |
+
for start in range(0, self.L, k):
|
| 571 |
+
idx = list(range(start, min(start + k, self.L)))
|
| 572 |
+
b = len(idx)
|
| 573 |
+
if self.s is not None:
|
| 574 |
+
P = s * eye + D[idx[0]]
|
| 575 |
+
for l in idx[1:]:
|
| 576 |
+
P = (s * eye + D[l]) @ P
|
| 577 |
+
anchor = s ** b
|
| 578 |
+
else:
|
| 579 |
+
P = D[idx[0]]
|
| 580 |
+
for l in idx[1:]:
|
| 581 |
+
P = D[l] @ P
|
| 582 |
+
anchor = None
|
| 583 |
+
u = torch.matmul(y, P.transpose(0, 1))
|
| 584 |
+
z = u - anchor * y if anchor is not None else u
|
| 585 |
+
if self.nonlinearity == "silu":
|
| 586 |
+
z = F.silu(z)
|
| 587 |
+
elif self.nonlinearity == "gelu":
|
| 588 |
+
z = F.gelu(z)
|
| 589 |
+
else:
|
| 590 |
+
z = torch.tanh(z)
|
| 591 |
+
y = (u + z) if self.nonlinear_mode == "add" else (y + z)
|
| 592 |
+
if self.norm_between:
|
| 593 |
+
y = F.rms_norm(y, (self.d_in,), None, 1e-6)
|
| 594 |
+
return y
|
| 595 |
+
|
| 596 |
+
# -- forward ----------------------------------------------------------
|
| 597 |
+
def forward(
|
| 598 |
+
self,
|
| 599 |
+
x: torch.Tensor,
|
| 600 |
+
prev_D: Optional[torch.Tensor] = None,
|
| 601 |
+
layer_idx: int = 0,
|
| 602 |
+
mode: str = "auto",
|
| 603 |
+
) -> Tuple[torch.Tensor, torch.Tensor]:
|
| 604 |
+
"""Returns ``(y, D_new)``. ``prev_D`` is the depth state of the
|
| 605 |
+
previous transformer layer (or ``None`` at a window boundary)."""
|
| 606 |
+
B, T, _ = x.shape
|
| 607 |
+
xm = x.reshape(B * T, self.n1_in, self.n2_in)
|
| 608 |
+
|
| 609 |
+
# innovation (position independent: it is a weight, not an activation)
|
| 610 |
+
I = self.innovation() # (L, d_out, d_in)
|
| 611 |
+
|
| 612 |
+
# recurrence, truncated at window boundaries
|
| 613 |
+
in_window = prev_D is not None and not (
|
| 614 |
+
self.K_win and self.K_win > 0 and layer_idx % self.K_win == 0)
|
| 615 |
+
if in_window:
|
| 616 |
+
prev = prev_D.detach() if self.stop_gradient else prev_D
|
| 617 |
+
else:
|
| 618 |
+
prev = self.initial_state(I.device, I.dtype)
|
| 619 |
+
|
| 620 |
+
if self.normalize_D:
|
| 621 |
+
I = rms_norm_matrix(I)
|
| 622 |
+
if in_window:
|
| 623 |
+
prev = rms_norm_matrix(prev)
|
| 624 |
+
|
| 625 |
+
sig = self.sigma_values() # (L,)
|
| 626 |
+
D = I + sig.view(self.L, 1, 1) * prev # (L, d_out, d_in)
|
| 627 |
+
if self.normalize_D:
|
| 628 |
+
D = rms_norm_matrix(D)
|
| 629 |
+
D = self.tau.view(self.L, 1, 1) * D
|
| 630 |
+
|
| 631 |
+
# y = gamma * W_L ... W_1 x
|
| 632 |
+
# Two equivalent execution modes:
|
| 633 |
+
# * "sequential": L matvec products (best for single-token decoding)
|
| 634 |
+
# * "matrix": build P = W_L...W_1 once, then one GEMM. The
|
| 635 |
+
# product costs L*d^3 but is amortised over all
|
| 636 |
+
# tokens in the batch, so for B*T >> L*d it is ~L x
|
| 637 |
+
# cheaper in FLOPs than the sequential form.
|
| 638 |
+
tokens = B * T
|
| 639 |
+
if self.composition == "nonlinear":
|
| 640 |
+
xf = xm.reshape(tokens, self.d_in)
|
| 641 |
+
s = (self.s if self.s is not None
|
| 642 |
+
else torch.zeros((), device=xf.device, dtype=xf.dtype))
|
| 643 |
+
if self.training:
|
| 644 |
+
# L sequential matvecs per token; checkpoint so the L
|
| 645 |
+
# intermediate activations are recomputed, not retained.
|
| 646 |
+
y = checkpoint(self._nonlinear_forward, xf, D, s,
|
| 647 |
+
use_reentrant=False)
|
| 648 |
+
else:
|
| 649 |
+
y = self._nonlinear_forward(xf, D, s)
|
| 650 |
+
elif self.composition == "sum":
|
| 651 |
+
if self.dynamic_gate is not None:
|
| 652 |
+
xf = xm.reshape(tokens, self.d_in)
|
| 653 |
+
if self.training:
|
| 654 |
+
# The dynamic path cannot use KronBank (P depends on the
|
| 655 |
+
# token) and L weighted GEMMs otherwise retain all their
|
| 656 |
+
# activations; checkpoint recomputes them in the backward.
|
| 657 |
+
s = (self.s if self.s is not None
|
| 658 |
+
else torch.zeros((), device=xf.device, dtype=xf.dtype))
|
| 659 |
+
y = checkpoint(self._dynamic_sum, xf, D, s,
|
| 660 |
+
use_reentrant=False)
|
| 661 |
+
else:
|
| 662 |
+
s = (self.s if self.s is not None
|
| 663 |
+
else torch.zeros((), device=xf.device, dtype=xf.dtype))
|
| 664 |
+
y = self._dynamic_sum(xf, D, s)
|
| 665 |
+
with torch.no_grad():
|
| 666 |
+
self._last_w_mean = self._gate_weights(
|
| 667 |
+
xf).mean(dim=0).detach()
|
| 668 |
+
else:
|
| 669 |
+
# The sum is cheap even for a single token, so there is no
|
| 670 |
+
# reason to keep a separate sequential path.
|
| 671 |
+
P = self.compose_sum(D)
|
| 672 |
+
y = torch.matmul(xm.reshape(tokens, self.d_in), P.transpose(0, 1))
|
| 673 |
+
else:
|
| 674 |
+
if self.composition in {"grouped", "alternating"}:
|
| 675 |
+
# These compositions have no cheap per-factor sequential
|
| 676 |
+
# fallback; build the composed map and apply it once.
|
| 677 |
+
P = self.compose(D)
|
| 678 |
+
y = torch.matmul(xm.reshape(tokens, self.d_in), P.transpose(0, 1))
|
| 679 |
+
else:
|
| 680 |
+
if mode == "auto":
|
| 681 |
+
use_matrix = (self.square and self.L > 1
|
| 682 |
+
and tokens >= max(256, self.d_in))
|
| 683 |
+
else:
|
| 684 |
+
use_matrix = mode == "matrix" and self.square and self.L > 1
|
| 685 |
+
if use_matrix:
|
| 686 |
+
P = self.compose_product(D)
|
| 687 |
+
y = torch.matmul(xm.reshape(tokens, self.d_in),
|
| 688 |
+
P.transpose(0, 1))
|
| 689 |
+
elif self.anchor:
|
| 690 |
+
s = self.s
|
| 691 |
+
y = xm.reshape(tokens, self.d_in)
|
| 692 |
+
for l in range(self.L):
|
| 693 |
+
y = torch.addmm(s * y, y, D[l].transpose(0, 1))
|
| 694 |
+
else:
|
| 695 |
+
y = xm.reshape(tokens, self.d_in)
|
| 696 |
+
for l in range(self.L):
|
| 697 |
+
y = torch.matmul(y, D[l].transpose(0, 1))
|
| 698 |
+
|
| 699 |
+
y = y.reshape(B, T, self.d_out)
|
| 700 |
+
if self.gate is not None:
|
| 701 |
+
y = y * self.gate
|
| 702 |
+
return y, D
|
| 703 |
+
|
| 704 |
+
def extra_repr(self) -> str:
|
| 705 |
+
s = "None" if self.s is None else f"{float(self.s):.4g}"
|
| 706 |
+
gate = "None" if self.gate is None else f"{float(self.gate):.4g}"
|
| 707 |
+
alpha = ("" if self.alpha is None
|
| 708 |
+
else f", alpha={float(self.alpha.mean()):.4g}")
|
| 709 |
+
return (f"d={self.d_in}->{self.d_out}, L={self.L}, K={self.K}, "
|
| 710 |
+
f"K_win={self.K_win}, anchor={self.anchor}, "
|
| 711 |
+
f"composition={self.composition}, s={s}, "
|
| 712 |
+
f"tau={float(self.tau.mean()):.4g}, gate={gate}"
|
| 713 |
+
f"{alpha}"
|
| 714 |
+
+ (f", dyn_gate={self.dynamic_gate}" if self.dynamic_gate else ""))
|
| 715 |
+
|
| 716 |
+
|
| 717 |
+
# ---------------------------------------------------------------------------
|
| 718 |
+
# KronBank -- batched evaluation of many composes at once
|
| 719 |
+
# ---------------------------------------------------------------------------
|
| 720 |
+
class KronBank:
|
| 721 |
+
"""Evaluate a set of same-shaped ``KronCompose`` modules as batched ops.
|
| 722 |
+
|
| 723 |
+
The modules keep ownership of their parameters; the bank only *reads* them
|
| 724 |
+
(``torch.stack`` is differentiable), so ``state_dict``, the ablation table
|
| 725 |
+
and the reference per-module path keep working unchanged.
|
| 726 |
+
|
| 727 |
+
Why this exists
|
| 728 |
+
---------------
|
| 729 |
+
A 12-layer model has 60 composes, each with its own innovation, recurrence,
|
| 730 |
+
``L``-fold product and apply. Implemented one by one that is ~900 kernel
|
| 731 |
+
launches per forward pass; on Windows/WDDM a launch costs ~45 us, so the
|
| 732 |
+
step time was 117 ms *independently of the batch size* (measure: batch 1
|
| 733 |
+
and batch 32 took the same time). The bank collapses the weight-only part
|
| 734 |
+
(innovation, recurrence, products -- everything that does not depend on the
|
| 735 |
+
activations) into ~20 batched launches.
|
| 736 |
+
"""
|
| 737 |
+
|
| 738 |
+
def __init__(self, mods: Sequence[KronCompose], layers: Sequence[int],
|
| 739 |
+
keys: Optional[Sequence[Any]] = None) -> None:
|
| 740 |
+
assert len(mods) == len(layers) and len(mods) > 0
|
| 741 |
+
m0 = mods[0]
|
| 742 |
+
for m in mods:
|
| 743 |
+
same = (m.L == m0.L and m.K == m0.K and m.d_in == m0.d_in
|
| 744 |
+
and m.d_out == m0.d_out and m.n1_in == m0.n1_in
|
| 745 |
+
and m.n2_in == m0.n2_in and m.n1_out == m0.n1_out
|
| 746 |
+
and m.n2_out == m0.n2_out and m.anchor == m0.anchor
|
| 747 |
+
and m.normalize_D == m0.normalize_D
|
| 748 |
+
and m.stop_gradient == m0.stop_gradient
|
| 749 |
+
and m.K_win == m0.K_win
|
| 750 |
+
and m.sigma_mode == m0.sigma_mode
|
| 751 |
+
and m.sigma_value == m0.sigma_value
|
| 752 |
+
and (m.gate is None) == (m0.gate is None))
|
| 753 |
+
if not same:
|
| 754 |
+
raise ValueError("KronBank modules must share shape and flags")
|
| 755 |
+
self.mods = list(mods)
|
| 756 |
+
self.layers = list(layers)
|
| 757 |
+
self.keys = list(keys) if keys is not None else list(range(len(mods)))
|
| 758 |
+
self.M = len(mods)
|
| 759 |
+
self.L, self.K = m0.L, m0.K
|
| 760 |
+
self.d_in, self.d_out = m0.d_in, m0.d_out
|
| 761 |
+
self.n1_in, self.n2_in = m0.n1_in, m0.n2_in
|
| 762 |
+
self.n1_out, self.n2_out = m0.n1_out, m0.n2_out
|
| 763 |
+
self.K_win = m0.K_win
|
| 764 |
+
self.stop_gradient = m0.stop_gradient
|
| 765 |
+
self.normalize_D = m0.normalize_D
|
| 766 |
+
self.anchor = m0.anchor
|
| 767 |
+
self.sigma_mode = m0.sigma_mode
|
| 768 |
+
self.sigma_value = m0.sigma_value
|
| 769 |
+
self.composition = m0.composition
|
| 770 |
+
self._m0 = m0
|
| 771 |
+
assert all(m.composition == self.composition for m in mods), \
|
| 772 |
+
"KronBank requires one composition mode per group"
|
| 773 |
+
assert all(self.layers[i] <= self.layers[i + 1]
|
| 774 |
+
for i in range(len(self.layers) - 1)), "layers must be sorted"
|
| 775 |
+
# contiguous (layer, start, stop) ranges
|
| 776 |
+
self.bounds: List[Tuple[int, int, int]] = []
|
| 777 |
+
start = 0
|
| 778 |
+
for j in range(1, len(self.layers) + 1):
|
| 779 |
+
if j == len(self.layers) or self.layers[j] != self.layers[start]:
|
| 780 |
+
self.bounds.append((self.layers[start], start, j))
|
| 781 |
+
start = j
|
| 782 |
+
|
| 783 |
+
# -- batched weight-only prelude --------------------------------------
|
| 784 |
+
def prelude(self) -> Tuple[torch.Tensor, torch.Tensor]:
|
| 785 |
+
"""Return ``(P, D)``.
|
| 786 |
+
|
| 787 |
+
``P``: (M, d_out, d_in) -- the full composed map *including* the
|
| 788 |
+
ReZero gate, i.e. ``y = x @ P.T``.
|
| 789 |
+
``D``: (M, L, d_out, d_in) -- the depth state (for diagnostics).
|
| 790 |
+
"""
|
| 791 |
+
mods, M, L, K = self.mods, self.M, self.L, self.K
|
| 792 |
+
n1o, n1i, n2o, n2i = self.n1_out, self.n1_in, self.n2_out, self.n2_in
|
| 793 |
+
|
| 794 |
+
A = torch.stack([m.A for m in mods]) # (M,L,K,n1o,n1i)
|
| 795 |
+
B = torch.stack([m.B for m in mods]) # (M,L,K,n2o,n2i)
|
| 796 |
+
tau = torch.stack([m.tau for m in mods]) # (M,L)
|
| 797 |
+
|
| 798 |
+
# innovation I[m,l,i,j,m2,n2] = sum_k A[m,l,k,i,j] B[m,l,k,m2,n2]
|
| 799 |
+
a = A.reshape(M * L, K, n1o * n1i).transpose(1, 2)
|
| 800 |
+
b = B.reshape(M * L, K, n2o * n2i)
|
| 801 |
+
I = torch.bmm(a, b).reshape(M, L, n1o, n1i, n2o, n2i)
|
| 802 |
+
I = I.permute(0, 1, 2, 4, 3, 5).reshape(
|
| 803 |
+
M, L, self.d_out, self.d_in).contiguous()
|
| 804 |
+
|
| 805 |
+
if self.normalize_D:
|
| 806 |
+
I = rms_norm_matrix(I)
|
| 807 |
+
|
| 808 |
+
sig = torch.stack([m.sigma_values() for m in mods]).view(M, L, 1, 1)
|
| 809 |
+
init = self._m0.initial_state(I.device, I.dtype) # (L,d,d), cached
|
| 810 |
+
|
| 811 |
+
Ds: List[torch.Tensor] = []
|
| 812 |
+
for k, (layer, lo, hi) in enumerate(self.bounds):
|
| 813 |
+
in_window = k > 0 and not (
|
| 814 |
+
self.K_win and self.K_win > 0 and layer % self.K_win == 0)
|
| 815 |
+
if in_window:
|
| 816 |
+
prev = Ds[-1]
|
| 817 |
+
if self.stop_gradient:
|
| 818 |
+
prev = prev.detach()
|
| 819 |
+
if self.normalize_D:
|
| 820 |
+
prev = rms_norm_matrix(prev)
|
| 821 |
+
else:
|
| 822 |
+
prev = init.unsqueeze(0).expand(hi - lo, L, self.d_out,
|
| 823 |
+
self.d_in)
|
| 824 |
+
Di = I[lo:hi] + sig[lo:hi] * prev
|
| 825 |
+
if self.normalize_D:
|
| 826 |
+
Di = rms_norm_matrix(Di)
|
| 827 |
+
Di = tau[lo:hi].view(-1, L, 1, 1) * Di
|
| 828 |
+
Ds.append(Di)
|
| 829 |
+
D = torch.cat(Ds, dim=0)
|
| 830 |
+
|
| 831 |
+
if self.composition == "sum":
|
| 832 |
+
alpha = torch.stack([m.alpha for m in mods]) # (M,L)
|
| 833 |
+
P = (alpha.view(M, L, 1, 1) * D).sum(dim=1)
|
| 834 |
+
if self.anchor:
|
| 835 |
+
s = torch.stack([m.s for m in mods]) # (M,)
|
| 836 |
+
eye = cached_eye(self.d_in, D.device, D.dtype)
|
| 837 |
+
P = P + s.view(M, 1, 1) * eye
|
| 838 |
+
elif self.anchor:
|
| 839 |
+
s = torch.stack([m.s for m in mods]) # (M,)
|
| 840 |
+
eye = cached_eye(self.d_in, D.device, D.dtype)
|
| 841 |
+
sI = s.view(M, 1, 1) * eye
|
| 842 |
+
P = sI + D[:, 0]
|
| 843 |
+
for l in range(1, L):
|
| 844 |
+
P = torch.baddbmm(s.view(M, 1, 1) * P, D[:, l], P)
|
| 845 |
+
else:
|
| 846 |
+
P = D[:, 0]
|
| 847 |
+
for l in range(1, L):
|
| 848 |
+
P = torch.bmm(D[:, l], P)
|
| 849 |
+
|
| 850 |
+
gates = [m.gate for m in mods]
|
| 851 |
+
if gates[0] is not None:
|
| 852 |
+
g = torch.stack(gates).view(M, 1, 1)
|
| 853 |
+
P = P * g
|
| 854 |
+
return P, D
|
kronllm/model.py
ADDED
|
@@ -0,0 +1,1086 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""KronLLM model: a transformer whose every matrix map is Kronecker-parameterised.
|
| 2 |
+
|
| 3 |
+
Design notes (deviations from the original TZ are listed in README.md)
|
| 4 |
+
---------------------------------------------------------------------
|
| 5 |
+
* every projection is a :class:`~kronllm.kron.KronCompose` (or a dense
|
| 6 |
+
``nn.Linear`` in the E0/E1 controls);
|
| 7 |
+
* the depth-recurrence state ``D`` is threaded through the blocks explicitly
|
| 8 |
+
(``state`` dict) so it is reset at the start of every forward pass and never
|
| 9 |
+
leaks between batches;
|
| 10 |
+
* each compose carries a ReZero gate ``gamma`` (init ``gate_init``) so a branch
|
| 11 |
+
can start as a small perturbation of the residual stream;
|
| 12 |
+
* ``batched_prelude`` evaluates all composes of the model as a handful of
|
| 13 |
+
batched GEMMs (:class:`~kronllm.kron.KronBank`) instead of ~900 individual
|
| 14 |
+
kernel launches. Windows/WDDM charges ~45 us per launch, so this is the
|
| 15 |
+
difference between 120 ms and ~20 ms per training step.
|
| 16 |
+
"""
|
| 17 |
+
from __future__ import annotations
|
| 18 |
+
|
| 19 |
+
import math
|
| 20 |
+
from typing import Any, Dict, List, Mapping, Optional, Tuple
|
| 21 |
+
|
| 22 |
+
import torch
|
| 23 |
+
import torch.nn as nn
|
| 24 |
+
import torch.nn.functional as F
|
| 25 |
+
from torch.nn.attention import SDPBackend, sdpa_kernel
|
| 26 |
+
|
| 27 |
+
from .kron import KronBank, KronCompose, apply_rope, rope_frequencies
|
| 28 |
+
from torch.utils.checkpoint import checkpoint
|
| 29 |
+
|
| 30 |
+
# Roles in the order in which a block consumes them. "fg" is the gate branch
|
| 31 |
+
# of the optional gated FFN.
|
| 32 |
+
ROLES = ("q", "k", "v", "o", "f", "fg")
|
| 33 |
+
|
| 34 |
+
# Explicit SDPA backends for `model.attn_impl`; "auto" leaves the choice to
|
| 35 |
+
# PyTorch (flash on Ampere+, memory-efficient on Turing, math otherwise).
|
| 36 |
+
_ATTN_BACKENDS = {
|
| 37 |
+
"flash": SDPBackend.FLASH_ATTENTION,
|
| 38 |
+
"mem_efficient": SDPBackend.EFFICIENT_ATTENTION,
|
| 39 |
+
"math": SDPBackend.MATH,
|
| 40 |
+
}
|
| 41 |
+
|
| 42 |
+
|
| 43 |
+
class RMSNorm(nn.Module):
|
| 44 |
+
def __init__(self, d: int, eps: float = 1e-6) -> None:
|
| 45 |
+
super().__init__()
|
| 46 |
+
self.eps = eps
|
| 47 |
+
self.weight = nn.Parameter(torch.ones(d))
|
| 48 |
+
|
| 49 |
+
def forward(self, x: torch.Tensor) -> torch.Tensor:
|
| 50 |
+
return F.rms_norm(x, (x.size(-1),), self.weight, self.eps)
|
| 51 |
+
|
| 52 |
+
|
| 53 |
+
class DenseFFN(nn.Module):
|
| 54 |
+
"""Plain ``Linear -> SiLU -> Linear`` FFN (the E0/E1 control)."""
|
| 55 |
+
|
| 56 |
+
def __init__(self, d: int, mult: float = 1.0) -> None:
|
| 57 |
+
super().__init__()
|
| 58 |
+
hidden = max(1, int(round(d * mult)))
|
| 59 |
+
self.fc1 = nn.Linear(d, hidden, bias=False)
|
| 60 |
+
self.fc2 = nn.Linear(hidden, d, bias=False)
|
| 61 |
+
|
| 62 |
+
def forward(self, x: torch.Tensor) -> torch.Tensor:
|
| 63 |
+
return self.fc2(F.silu(self.fc1(x)))
|
| 64 |
+
|
| 65 |
+
|
| 66 |
+
class KronEmbedding(nn.Module):
|
| 67 |
+
"""Embedding table ``V x r`` as a sum of K Kronecker products.
|
| 68 |
+
|
| 69 |
+
The vocabulary is split as ``V = v1 * v2`` and the rank as ``r = r1 * r2``;
|
| 70 |
+
the table is
|
| 71 |
+
|
| 72 |
+
E = sum_k A_k (x) B_k, A_k: v1 x r1, B_k: v2 x r2.
|
| 73 |
+
|
| 74 |
+
Parameter cost is ``K * (v1*r1 + v2*r2)`` instead of ``V * r``, while the
|
| 75 |
+
rank stays in the same class as a dense ``V x r`` factorisation:
|
| 76 |
+
``rank(A (x) B) = rank(A) * rank(B) <= r1 * r2 = r``. With ``K`` summands
|
| 77 |
+
the table can reach rank ``min(V, r)``, i.e. the same information ceiling
|
| 78 |
+
as the dense factorised embedding it replaces, at a small fraction of the
|
| 79 |
+
parameter count for wide vocabularies. The table is materialised once per
|
| 80 |
+
forward (``K * V * r`` MACs, amortised over the batch) and the tied head
|
| 81 |
+
keeps the same ``d*r + r*V`` per-token logits cost.
|
| 82 |
+
"""
|
| 83 |
+
|
| 84 |
+
def __init__(self, vocab_size: int, rank: int, v1: int, v2: int,
|
| 85 |
+
r1: int, r2: int, K: int) -> None:
|
| 86 |
+
super().__init__()
|
| 87 |
+
if v1 * v2 != vocab_size:
|
| 88 |
+
raise ValueError(f"v1*v2={v1 * v2} != vocab_size={vocab_size}")
|
| 89 |
+
if r1 * r2 != rank:
|
| 90 |
+
raise ValueError(f"r1*r2={r1 * r2} != rank={rank}")
|
| 91 |
+
self.v1, self.v2 = int(v1), int(v2)
|
| 92 |
+
self.r1, self.r2 = int(r1), int(r2)
|
| 93 |
+
self.rank = int(rank)
|
| 94 |
+
self.K = int(K)
|
| 95 |
+
self.A = nn.ParameterList(
|
| 96 |
+
[nn.Parameter(torch.empty(self.v1, self.r1)) for _ in range(self.K)])
|
| 97 |
+
self.B = nn.ParameterList(
|
| 98 |
+
[nn.Parameter(torch.empty(self.v2, self.r2)) for _ in range(self.K)])
|
| 99 |
+
self.reset_parameters()
|
| 100 |
+
|
| 101 |
+
def reset_parameters(self) -> None:
|
| 102 |
+
for a in self.A:
|
| 103 |
+
nn.init.normal_(a, 0.0, 1.0)
|
| 104 |
+
for b in self.B:
|
| 105 |
+
nn.init.normal_(b, 0.0, 1.0)
|
| 106 |
+
# Rescale so that the composed table has entry std 1/sqrt(r), the same
|
| 107 |
+
# scale as a standard embedding initialisation. Scaling every factor
|
| 108 |
+
# by s scales every Kronecker product by s^2.
|
| 109 |
+
with torch.no_grad():
|
| 110 |
+
t = self.table()
|
| 111 |
+
target = 1.0 / (self.rank ** 0.5)
|
| 112 |
+
s = (target / max(float(t.std()), 1e-8)) ** 0.5
|
| 113 |
+
for p in self.A:
|
| 114 |
+
p.mul_(s)
|
| 115 |
+
for p in self.B:
|
| 116 |
+
p.mul_(s)
|
| 117 |
+
|
| 118 |
+
def table(self) -> torch.Tensor:
|
| 119 |
+
"""Materialise the ``(V, r)`` table (cached by the caller in eval)."""
|
| 120 |
+
# One einsum instead of K torch.kron calls: torch.kron allocates a
|
| 121 |
+
# V x r tensor per summand, and on WDDM those launches dominate when
|
| 122 |
+
# the table is rebuilt inside every checkpointed loss chunk.
|
| 123 |
+
A = torch.stack(list(self.A)) # (K, v1, r1)
|
| 124 |
+
B = torch.stack(list(self.B)) # (K, v2, r2)
|
| 125 |
+
t = torch.einsum("kvp,kwq->vwpq", A, B)
|
| 126 |
+
return t.reshape(self.v1 * self.v2, self.r1 * self.r2)
|
| 127 |
+
|
| 128 |
+
|
| 129 |
+
class RoutedEmbedding(nn.Module):
|
| 130 |
+
"""Dense token embeddings selected by a sparse top-k router.
|
| 131 |
+
|
| 132 |
+
The token id is first mapped to a small routing code
|
| 133 |
+
``route[id] in R^{route_dim}``. A linear router turns that code into
|
| 134 |
+
``v_eff`` logits; the top-k logits are softmaxed and used to mix the
|
| 135 |
+
corresponding rows of a dense bank ``bank[v_eff, d_model]``. The effective
|
| 136 |
+
embedding table is therefore dense in width but sparse in computation:
|
| 137 |
+
|
| 138 |
+
``E_eff[v] = sum_{i in topk(v)} softmax(logits[v])_i * bank[i]``
|
| 139 |
+
|
| 140 |
+
This avoids the low-rank bottleneck of the factorised embedding (the output
|
| 141 |
+
is a full ``d_model`` vector, not an ``r``-dimensional projection) while
|
| 142 |
+
keeping the stored table at ``v_eff * d_model`` instead of ``V * d_model``.
|
| 143 |
+
The same ``E_eff`` is reused by the tied output head, so the head adds no
|
| 144 |
+
parameters.
|
| 145 |
+
|
| 146 |
+
Args:
|
| 147 |
+
vocab_size: Number of token ids.
|
| 148 |
+
d_model: Embedding width.
|
| 149 |
+
v_eff: Number of virtual embedding rows (default 5,000).
|
| 150 |
+
route_dim: Width of the routing code (default 6).
|
| 151 |
+
top_k: Number of bank rows mixed per token (default 10).
|
| 152 |
+
temperature: Softmax temperature on the selected logits.
|
| 153 |
+
|
| 154 |
+
Raises:
|
| 155 |
+
ValueError: If ``top_k`` is outside ``[1, v_eff]`` or ``route_dim`` is
|
| 156 |
+
not positive.
|
| 157 |
+
"""
|
| 158 |
+
|
| 159 |
+
def __init__(self, vocab_size: int, d_model: int, v_eff: int = 5000,
|
| 160 |
+
route_dim: int = 6, top_k: int = 10,
|
| 161 |
+
temperature: float = 1.0) -> None:
|
| 162 |
+
super().__init__()
|
| 163 |
+
if route_dim <= 0:
|
| 164 |
+
raise ValueError(f"route_dim must be positive, got {route_dim}")
|
| 165 |
+
if not 1 <= top_k <= v_eff:
|
| 166 |
+
raise ValueError(f"top_k must be in [1, {v_eff}], got {top_k}")
|
| 167 |
+
self.vocab_size = int(vocab_size)
|
| 168 |
+
self.d = int(d_model)
|
| 169 |
+
self.v_eff = int(v_eff)
|
| 170 |
+
self.route_dim = int(route_dim)
|
| 171 |
+
self.top_k = int(top_k)
|
| 172 |
+
self.temperature = float(temperature)
|
| 173 |
+
|
| 174 |
+
self.route = nn.Embedding(self.vocab_size, self.route_dim)
|
| 175 |
+
self.route_proj = nn.Linear(self.route_dim, self.v_eff, bias=False)
|
| 176 |
+
self.bank = nn.Embedding(self.v_eff, self.d)
|
| 177 |
+
self.reset_parameters()
|
| 178 |
+
|
| 179 |
+
def reset_parameters(self) -> None:
|
| 180 |
+
"""Initialise the router and the bank with matching output scale.
|
| 181 |
+
|
| 182 |
+
The bank rows are scaled by ``sqrt(top_k)`` so that the weighted
|
| 183 |
+
average of ``top_k`` rows has the same standard deviation as a plain
|
| 184 |
+
embedding row (the softmax weights sum to one and are close to uniform
|
| 185 |
+
at initialisation).
|
| 186 |
+
"""
|
| 187 |
+
nn.init.normal_(self.route.weight, mean=0.0, std=0.1)
|
| 188 |
+
nn.init.normal_(self.route_proj.weight, mean=0.0,
|
| 189 |
+
std=1.0 / math.sqrt(self.route_dim))
|
| 190 |
+
nn.init.normal_(self.bank.weight, mean=0.0,
|
| 191 |
+
std=0.02 * math.sqrt(self.top_k))
|
| 192 |
+
|
| 193 |
+
def set_top_k(self, top_k: int) -> None:
|
| 194 |
+
"""Change the number of bank rows mixed per token.
|
| 195 |
+
|
| 196 |
+
Args:
|
| 197 |
+
top_k: New value, must be in ``[1, v_eff]``.
|
| 198 |
+
|
| 199 |
+
Raises:
|
| 200 |
+
ValueError: If ``top_k`` is outside the valid range.
|
| 201 |
+
"""
|
| 202 |
+
if not 1 <= int(top_k) <= self.v_eff:
|
| 203 |
+
raise ValueError(f"top_k must be in [1, {self.v_eff}], got {top_k}")
|
| 204 |
+
self.top_k = int(top_k)
|
| 205 |
+
|
| 206 |
+
def route_logits(self) -> torch.Tensor:
|
| 207 |
+
"""Return the router logits for every vocabulary entry.
|
| 208 |
+
|
| 209 |
+
Returns:
|
| 210 |
+
A ``(vocab_size, v_eff)`` tensor.
|
| 211 |
+
"""
|
| 212 |
+
codes = self.route.weight
|
| 213 |
+
return self.route_proj(codes) / max(self.temperature, 1e-5)
|
| 214 |
+
|
| 215 |
+
def effective_embeddings(self) -> torch.Tensor:
|
| 216 |
+
"""Build the dense ``(vocab_size, d_model)`` effective embedding table.
|
| 217 |
+
|
| 218 |
+
Returns:
|
| 219 |
+
The top-k mixture of bank rows for every token id.
|
| 220 |
+
"""
|
| 221 |
+
logits = self.route_logits()
|
| 222 |
+
values, indices = logits.topk(self.top_k, dim=-1)
|
| 223 |
+
weights = F.softmax(values, dim=-1)
|
| 224 |
+
rows = self.bank(indices) # (V, k, d)
|
| 225 |
+
return (rows * weights.unsqueeze(-1)).sum(dim=-2)
|
| 226 |
+
|
| 227 |
+
def forward(self, idx: torch.Tensor) -> torch.Tensor:
|
| 228 |
+
"""Look up the routed embedding of ``idx``.
|
| 229 |
+
|
| 230 |
+
Args:
|
| 231 |
+
idx: Integer token ids of any shape.
|
| 232 |
+
|
| 233 |
+
Returns:
|
| 234 |
+
A float tensor with shape ``idx.shape + (d_model,)``.
|
| 235 |
+
"""
|
| 236 |
+
return F.embedding(idx, self.effective_embeddings())
|
| 237 |
+
|
| 238 |
+
|
| 239 |
+
def _kron_out_dims(d_out: int, n2: int) -> Tuple[int, int]:
|
| 240 |
+
"""Split ``d_out`` into ``(n1, n2)`` reusing ``n2`` when possible."""
|
| 241 |
+
if d_out % n2 == 0:
|
| 242 |
+
return d_out // n2, n2
|
| 243 |
+
for n2c in range(int(math.sqrt(d_out)), 0, -1):
|
| 244 |
+
if d_out % n2c == 0:
|
| 245 |
+
return d_out // n2c, n2c
|
| 246 |
+
return d_out, 1
|
| 247 |
+
|
| 248 |
+
|
| 249 |
+
class KronBlock(nn.Module):
|
| 250 |
+
def __init__(self, cfg: Mapping[str, Any]) -> None:
|
| 251 |
+
super().__init__()
|
| 252 |
+
mc = cfg["model"]
|
| 253 |
+
kc = mc["kron"]
|
| 254 |
+
d = int(mc["d_model"])
|
| 255 |
+
nh = int(mc["n_heads"])
|
| 256 |
+
nkv = mc.get("n_kv_heads") or nh
|
| 257 |
+
self.d = d
|
| 258 |
+
self.n_heads = nh
|
| 259 |
+
self.n_kv_heads = int(nkv)
|
| 260 |
+
self.d_head = d // nh
|
| 261 |
+
assert self.d_head * nh == d, "d_model must be divisible by n_heads"
|
| 262 |
+
d_kv = self.d_head * self.n_kv_heads
|
| 263 |
+
|
| 264 |
+
self.use_kron_qkv = bool(mc["use_kron_qkv"])
|
| 265 |
+
self.use_kron_ffn = bool(mc["use_kron_ffn"])
|
| 266 |
+
self.ffn_silu = bool(mc.get("ffn_silu", True))
|
| 267 |
+
self.ffn_gated = bool(mc.get("ffn_gated", False)) and self.use_kron_ffn
|
| 268 |
+
# Feature interaction: a bilinear (GLU) FFN mixes features
|
| 269 |
+
# multiplicatively, ``W_out(act(W_a x) * (W_b x))``, which the linear
|
| 270 |
+
# Kronecker composition cannot express. ``ffn_mix_kron`` adds the
|
| 271 |
+
# standard Kronecker map back as a non-interacting component.
|
| 272 |
+
self.ffn_mix = str(mc.get("ffn_mix", "kron")).lower()
|
| 273 |
+
if self.ffn_mix not in {"kron", "glu"}:
|
| 274 |
+
raise ValueError("ffn_mix must be 'kron' or 'glu'")
|
| 275 |
+
self.ffn_mix_rank = int(mc.get("ffn_mix_rank", 16))
|
| 276 |
+
self.ffn_mix_kron = bool(mc.get("ffn_mix_kron", False))
|
| 277 |
+
|
| 278 |
+
self.norm1 = RMSNorm(d, float(mc["norm_eps"]))
|
| 279 |
+
self.norm2 = RMSNorm(d, float(mc["norm_eps"]))
|
| 280 |
+
# Attention kernel: "auto" lets PyTorch pick the best fused SDPA
|
| 281 |
+
# backend; "flash"/"mem_efficient"/"math" pin one for benchmarking.
|
| 282 |
+
# This only swaps the QK^T-softmax-AV kernel; q/k/v still come from the
|
| 283 |
+
# Kronecker maps and the output still goes through the Kron o-map.
|
| 284 |
+
self.attn_impl = str(mc.get("attn_impl", "auto")).lower()
|
| 285 |
+
if self.attn_impl not in {"auto", *_ATTN_BACKENDS}:
|
| 286 |
+
raise ValueError(
|
| 287 |
+
f"attn_impl must be auto/{'/'.join(_ATTN_BACKENDS)}, "
|
| 288 |
+
f"got {self.attn_impl!r}")
|
| 289 |
+
# intra-layer normalisation of the attention operands: keeps q/k at a
|
| 290 |
+
# fixed scale regardless of how the Kron branch grows, which removes
|
| 291 |
+
# the logit drift that limits the learning rate in small models.
|
| 292 |
+
self.qk_norm = bool(mc.get("qk_norm", False))
|
| 293 |
+
if self.qk_norm:
|
| 294 |
+
self.q_norm = RMSNorm(self.d_head, float(mc["norm_eps"]))
|
| 295 |
+
self.k_norm = RMSNorm(self.d_head, float(mc["norm_eps"]))
|
| 296 |
+
|
| 297 |
+
self.kc = kc
|
| 298 |
+
self.L = int(kc["L"])
|
| 299 |
+
self.anchor = bool(kc["anchor"])
|
| 300 |
+
self.s_learnable = bool(kc["s_learnable"])
|
| 301 |
+
|
| 302 |
+
n1, n2 = int(kc["n1"]), int(kc["n2"])
|
| 303 |
+
assert n1 * n2 == d, f"n1*n2 must equal d_model ({n1}*{n2} != {d})"
|
| 304 |
+
|
| 305 |
+
def s_of(role: str) -> float:
|
| 306 |
+
s = kc["s_init"]
|
| 307 |
+
if isinstance(s, (int, float)):
|
| 308 |
+
return float(s)
|
| 309 |
+
if role in s:
|
| 310 |
+
return float(s[role])
|
| 311 |
+
if role in ("q", "k", "v", "o") and "qkv" in s:
|
| 312 |
+
return float(s["qkv"])
|
| 313 |
+
return float(s.get("ffn", 1.0))
|
| 314 |
+
|
| 315 |
+
self.s_values = {r: s_of(r) for r in ("q", "k", "v", "o")}
|
| 316 |
+
self.s_values["f"] = self.s_values["fg"] = s_of("f")
|
| 317 |
+
|
| 318 |
+
if self.use_kron_qkv:
|
| 319 |
+
kq = kc.get("K_qkv", None) # E4: extra Kronecker terms on Q only
|
| 320 |
+
self.q = self._make(d, n1, n2, self.L, s_of("q"), self.anchor, kq)
|
| 321 |
+
self.k = self._make(d_kv, n1, n2, self.L, s_of("k"), self.anchor)
|
| 322 |
+
self.v = self._make(d_kv, n1, n2, self.L, s_of("v"), self.anchor)
|
| 323 |
+
self.o = self._make(d, n1, n2, self.L, s_of("o"), self.anchor)
|
| 324 |
+
else:
|
| 325 |
+
self.q = nn.Linear(d, d, bias=False)
|
| 326 |
+
self.k = nn.Linear(d, d_kv, bias=False)
|
| 327 |
+
self.v = nn.Linear(d, d_kv, bias=False)
|
| 328 |
+
self.o = nn.Linear(d, d, bias=False)
|
| 329 |
+
|
| 330 |
+
if self.use_kron_ffn and self.ffn_mix == "kron":
|
| 331 |
+
self.ffn = self._make(d, n1, n2, self.L, s_of("f"), self.anchor)
|
| 332 |
+
if self.ffn_gated:
|
| 333 |
+
self.ffn_g = self._make(d, n1, n2, self.L, s_of("fg"), self.anchor)
|
| 334 |
+
else:
|
| 335 |
+
self.ffn_g = None
|
| 336 |
+
elif self.use_kron_ffn and self.ffn_mix == "glu":
|
| 337 |
+
r = self.ffn_mix_rank
|
| 338 |
+
self.glu_a = nn.Linear(d, r, bias=False)
|
| 339 |
+
self.glu_b = nn.Linear(d, r, bias=False)
|
| 340 |
+
self.glu_o = nn.Linear(r, d, bias=False)
|
| 341 |
+
# The generic 0.02 init leaves the bilinear branch ~300x smaller
|
| 342 |
+
# than a Kron block, i.e. effectively switched off; mark the
|
| 343 |
+
# modules so `_init_weights` scales them to unit activation.
|
| 344 |
+
self.glu_a._glu_role = "in"
|
| 345 |
+
self.glu_b._glu_role = "in"
|
| 346 |
+
self.glu_o._glu_role = "out"
|
| 347 |
+
# Optional non-interacting Kronecker component alongside the
|
| 348 |
+
# bilinear term (Q9k): "plus the standard Kronecker product".
|
| 349 |
+
self.ffn = (self._make(d, n1, n2, self.L, s_of("f"), self.anchor)
|
| 350 |
+
if self.ffn_mix_kron else None)
|
| 351 |
+
self.ffn_g = None
|
| 352 |
+
else:
|
| 353 |
+
self.ffn = DenseFFN(d, float(mc.get("dense_ffn_mult", 1.0)))
|
| 354 |
+
self.ffn_g = None
|
| 355 |
+
|
| 356 |
+
# ------------------------------------------------------------------
|
| 357 |
+
def _make(self, d_out: int, n1: int, n2: int, L: int, s_init: float,
|
| 358 |
+
anchor: bool, k_override: Optional[int] = None) -> KronCompose:
|
| 359 |
+
kc = self.kc
|
| 360 |
+
learn = self.s_learnable
|
| 361 |
+
if d_out == self.d:
|
| 362 |
+
n1o, n2o, L_eff = n1, n2, L
|
| 363 |
+
else: # GQA projection: non-square, single factor
|
| 364 |
+
n1o, n2o = _kron_out_dims(d_out, n2)
|
| 365 |
+
L_eff, anchor, learn = 1, False, False
|
| 366 |
+
return KronCompose(
|
| 367 |
+
self.d, d_out, n1, n2, n1o, n2o,
|
| 368 |
+
L=L_eff, K=int(k_override if k_override is not None else kc["K"]),
|
| 369 |
+
K_win=int(kc["K_win"]),
|
| 370 |
+
s_init=s_init, anchor=anchor, s_learnable=learn,
|
| 371 |
+
stop_gradient=bool(kc["stop_gradient"]),
|
| 372 |
+
theta_std=float(kc["theta_std"]),
|
| 373 |
+
rezero_eps=float(kc["rezero_eps"]),
|
| 374 |
+
s_init_effective=bool(kc.get("s_init_effective", True)),
|
| 375 |
+
init_gain=kc.get("init_gain", "unit"),
|
| 376 |
+
innovation_scale=float(kc.get("innovation_scale", 0.1)),
|
| 377 |
+
normalize_D=bool(kc.get("normalize_D", True)),
|
| 378 |
+
gate_init=float(kc.get("gate_init", 0.1)),
|
| 379 |
+
use_gate=bool(kc.get("gate", True)),
|
| 380 |
+
composition=str(kc.get("composition", "product")),
|
| 381 |
+
sum_alpha_init=float(kc.get("sum_alpha_init", 1.0)),
|
| 382 |
+
dynamic_gate=(kc.get("dynamic_gate")
|
| 383 |
+
if L_eff == L and str(kc.get("composition",
|
| 384 |
+
"product")) == "sum"
|
| 385 |
+
else None),
|
| 386 |
+
groups=(int(kc.get("groups", 1))
|
| 387 |
+
if L_eff == L and str(kc.get("composition",
|
| 388 |
+
"product")) == "grouped"
|
| 389 |
+
else 1),
|
| 390 |
+
alternate_start=(str(kc.get("alternate_start", "sum"))
|
| 391 |
+
if L_eff == L and str(kc.get("composition",
|
| 392 |
+
"product")) == "alternating"
|
| 393 |
+
else "sum"),
|
| 394 |
+
nonlinearity=str(kc.get("nonlinearity", "silu")),
|
| 395 |
+
norm_between=bool(kc.get("norm_between", False)),
|
| 396 |
+
nonlinear_every=int(kc.get("nonlinear_every", 1)),
|
| 397 |
+
nonlinear_mode=str(kc.get("nonlinear_mode", "replace")),
|
| 398 |
+
sigma_mode=str(kc.get("sigma_mode", "learned")),
|
| 399 |
+
sigma_value=float(kc.get("sigma_value", 0.5)),
|
| 400 |
+
)
|
| 401 |
+
|
| 402 |
+
def ffn_module(self) -> nn.Module:
|
| 403 |
+
return self.ffn
|
| 404 |
+
|
| 405 |
+
# ------------------------------------------------------------------
|
| 406 |
+
def forward(self, x: torch.Tensor, state: Dict[str, Optional[torch.Tensor]],
|
| 407 |
+
layer_idx: int, cos: torch.Tensor, sin: torch.Tensor,
|
| 408 |
+
collect: bool = False, mode: str = "auto",
|
| 409 |
+
P: Optional[Dict[str, torch.Tensor]] = None,
|
| 410 |
+
past_kv: Optional[Tuple[torch.Tensor, torch.Tensor]] = None,
|
| 411 |
+
use_cache: bool = False,
|
| 412 |
+
) -> Tuple[torch.Tensor, Dict[str, Optional[torch.Tensor]],
|
| 413 |
+
Dict[str, Any], Optional[Tuple[torch.Tensor,
|
| 414 |
+
torch.Tensor]]]:
|
| 415 |
+
"""``P`` maps role -> precomputed ``(d_out, d_in)`` composed matrix.
|
| 416 |
+
|
| 417 |
+
The matrix already contains the ReZero gate, so it is applied as
|
| 418 |
+
``y = h @ P.T`` -- one GEMM instead of ``L`` matvecs + a gate.
|
| 419 |
+
|
| 420 |
+
``past_kv`` is the cached ``(k, v)`` of the previous tokens (already
|
| 421 |
+
RoPE-rotated); the returned 4th value is the updated cache when
|
| 422 |
+
``use_cache`` is set, else ``None``.
|
| 423 |
+
"""
|
| 424 |
+
B, T, _ = x.shape
|
| 425 |
+
dh, nh = self.d_head, self.n_heads
|
| 426 |
+
cache: Dict[str, Any] = {}
|
| 427 |
+
|
| 428 |
+
h = self.norm1(x)
|
| 429 |
+
|
| 430 |
+
def apply(mod: nn.Module, role: str, inp: torch.Tensor):
|
| 431 |
+
"""Return ``(y, D_or_None)``."""
|
| 432 |
+
if not isinstance(mod, KronCompose):
|
| 433 |
+
return mod(inp), None
|
| 434 |
+
if P is not None and P.get(role) is not None:
|
| 435 |
+
return torch.matmul(inp, P[role].transpose(0, 1)), None
|
| 436 |
+
return mod(inp, state.get(role), layer_idx, mode=mode)
|
| 437 |
+
|
| 438 |
+
q, Dq = apply(self.q, "q", h)
|
| 439 |
+
k, Dk = apply(self.k, "k", h)
|
| 440 |
+
v, Dv = apply(self.v, "v", h)
|
| 441 |
+
for name, D in (("q", Dq), ("k", Dk), ("v", Dv)):
|
| 442 |
+
if D is not None:
|
| 443 |
+
state[name] = D
|
| 444 |
+
|
| 445 |
+
q = q.view(B, T, nh, dh).transpose(1, 2)
|
| 446 |
+
k = k.view(B, T, self.n_kv_heads, dh).transpose(1, 2)
|
| 447 |
+
v = v.view(B, T, self.n_kv_heads, dh).transpose(1, 2)
|
| 448 |
+
if self.qk_norm:
|
| 449 |
+
q = self.q_norm(q)
|
| 450 |
+
k = self.k_norm(k)
|
| 451 |
+
q = apply_rope(q, cos, sin)
|
| 452 |
+
k = apply_rope(k, cos, sin)
|
| 453 |
+
|
| 454 |
+
if past_kv is not None:
|
| 455 |
+
k = torch.cat([past_kv[0], k], dim=2)
|
| 456 |
+
v = torch.cat([past_kv[1], v], dim=2)
|
| 457 |
+
k_cache, v_cache = k, v
|
| 458 |
+
n_past = k.shape[2] - q.shape[2]
|
| 459 |
+
|
| 460 |
+
if self.n_kv_heads != nh:
|
| 461 |
+
rep = nh // self.n_kv_heads
|
| 462 |
+
k = k.repeat_interleave(rep, dim=1)
|
| 463 |
+
v = v.repeat_interleave(rep, dim=1)
|
| 464 |
+
|
| 465 |
+
if collect and n_past == 0:
|
| 466 |
+
scale = 1.0 / math.sqrt(dh)
|
| 467 |
+
scores = torch.matmul(q, k.transpose(-1, -2)) * scale
|
| 468 |
+
mask = torch.ones(T, T, device=x.device, dtype=torch.bool).triu(1)
|
| 469 |
+
scores = scores.masked_fill(mask, float("-inf"))
|
| 470 |
+
p = torch.softmax(scores.float(), dim=-1)
|
| 471 |
+
ent = -(p.clamp_min(1e-9).log() * p).sum(-1)
|
| 472 |
+
cache["attn_entropy"] = ent.mean().item()
|
| 473 |
+
cache["q_drift"] = ((q - h.view(B, T, nh, dh).transpose(1, 2)).norm()
|
| 474 |
+
/ h.norm().clamp_min(1e-9)).item()
|
| 475 |
+
del scores, p, mask
|
| 476 |
+
cache["q_sigma_min"] = torch.linalg.svdvals(
|
| 477 |
+
q.reshape(-1, dh).float()).min().item()
|
| 478 |
+
cache["k_sigma_min"] = torch.linalg.svdvals(
|
| 479 |
+
k.reshape(-1, dh).float()).min().item()
|
| 480 |
+
|
| 481 |
+
out = self._attend(q, k, v, n_past)
|
| 482 |
+
out = out.transpose(1, 2).reshape(B, T, self.d)
|
| 483 |
+
|
| 484 |
+
o, Do = apply(self.o, "o", out)
|
| 485 |
+
if Do is not None:
|
| 486 |
+
state["o"] = Do
|
| 487 |
+
x = x + o
|
| 488 |
+
|
| 489 |
+
h2 = self.norm2(x)
|
| 490 |
+
if self.ffn_mix == "glu":
|
| 491 |
+
# Bilinear feature interaction: act(W_a x) * (W_b x).
|
| 492 |
+
f = self.glu_o(F.silu(self.glu_a(h2)) * self.glu_b(h2))
|
| 493 |
+
if isinstance(self.ffn, KronCompose):
|
| 494 |
+
fk, Df = apply(self.ffn, "f", h2)
|
| 495 |
+
if Df is not None:
|
| 496 |
+
state["f"] = Df
|
| 497 |
+
f = f + fk
|
| 498 |
+
elif not isinstance(self.ffn, KronCompose):
|
| 499 |
+
f = self.ffn(h2)
|
| 500 |
+
elif self.ffn_gated:
|
| 501 |
+
fa, Da = apply(self.ffn, "f", h2)
|
| 502 |
+
fb, Db = apply(self.ffn_g, "fg", h2)
|
| 503 |
+
if Da is not None:
|
| 504 |
+
state["f"] = Da
|
| 505 |
+
if Db is not None:
|
| 506 |
+
state["fg"] = Db
|
| 507 |
+
f = fa * F.silu(fb)
|
| 508 |
+
else:
|
| 509 |
+
f, Df = apply(self.ffn, "f", h2)
|
| 510 |
+
if Df is not None:
|
| 511 |
+
state["f"] = Df
|
| 512 |
+
if self.ffn_silu:
|
| 513 |
+
f = F.silu(f)
|
| 514 |
+
x = x + f
|
| 515 |
+
new_kv = (k_cache, v_cache) if use_cache else None
|
| 516 |
+
return x, state, cache, new_kv
|
| 517 |
+
|
| 518 |
+
def _attend(self, q: torch.Tensor, k: torch.Tensor, v: torch.Tensor,
|
| 519 |
+
n_past: int = 0) -> torch.Tensor:
|
| 520 |
+
"""SDPA with an optional KV cache and the configured backend."""
|
| 521 |
+
def call(**kw):
|
| 522 |
+
if self.attn_impl == "auto":
|
| 523 |
+
return F.scaled_dot_product_attention(q, k, v, **kw)
|
| 524 |
+
with sdpa_kernel(_ATTN_BACKENDS[self.attn_impl]):
|
| 525 |
+
return F.scaled_dot_product_attention(q, k, v, **kw)
|
| 526 |
+
|
| 527 |
+
if n_past == 0:
|
| 528 |
+
return call(is_causal=True)
|
| 529 |
+
if q.shape[2] == 1:
|
| 530 |
+
# One new query token sees every cached key.
|
| 531 |
+
return call(is_causal=False)
|
| 532 |
+
# Chunked prefill with a cache: query i (absolute position n_past+i)
|
| 533 |
+
# must not see keys beyond n_past+i.
|
| 534 |
+
qpos = torch.arange(q.shape[2], device=q.device)[:, None] + n_past
|
| 535 |
+
kpos = torch.arange(k.shape[2], device=q.device)[None, :]
|
| 536 |
+
mask = kpos <= qpos
|
| 537 |
+
return call(attn_mask=mask)
|
| 538 |
+
|
| 539 |
+
|
| 540 |
+
class KronTransformer(nn.Module):
|
| 541 |
+
"""Byte-level decoder-only transformer with Kronecker matrix maps."""
|
| 542 |
+
|
| 543 |
+
def __init__(self, cfg: Mapping[str, Any]) -> None:
|
| 544 |
+
super().__init__()
|
| 545 |
+
mc = cfg["model"]
|
| 546 |
+
dc = cfg["data"]
|
| 547 |
+
self.cfg = cfg
|
| 548 |
+
self.vocab_size = int(dc["vocab_size"])
|
| 549 |
+
self.d = int(mc["d_model"])
|
| 550 |
+
self.n_layers = int(mc["n_layers"])
|
| 551 |
+
# Recursive weight sharing: n_unique blocks applied `recurse` times.
|
| 552 |
+
# The effective depth is n_unique * recurse; parameters are counted once.
|
| 553 |
+
self.n_unique = int(mc.get("n_unique_blocks") or self.n_layers)
|
| 554 |
+
self.recurse = int(mc.get("recurse") or 1)
|
| 555 |
+
if self.n_unique < 1 or self.recurse < 1:
|
| 556 |
+
raise ValueError("n_unique_blocks and recurse must be >= 1")
|
| 557 |
+
self.n_layers = self.n_unique * self.recurse
|
| 558 |
+
self.n_heads = int(mc["n_heads"])
|
| 559 |
+
self.d_head = self.d // self.n_heads
|
| 560 |
+
self.rope_theta = float(mc["rope_theta"])
|
| 561 |
+
self.seq_len = int(dc["seq_len"])
|
| 562 |
+
self.eot_id = int(dc.get("eot_id") or (self.vocab_size - 1))
|
| 563 |
+
|
| 564 |
+
self.embed_rank = int(mc.get("embed_rank") or 0)
|
| 565 |
+
self._glu_rank = int(mc.get("ffn_mix_rank", 16))
|
| 566 |
+
mode = str(mc.get("embed_mode", "auto")).lower()
|
| 567 |
+
if mode == "auto":
|
| 568 |
+
mode = "factorised" if self.embed_rank > 0 else "dense"
|
| 569 |
+
if mode not in {"factorised", "dense", "routed", "kron"}:
|
| 570 |
+
raise ValueError(
|
| 571 |
+
f"embed_mode must be auto/factorised/dense/routed/kron, "
|
| 572 |
+
f"got {mode!r}")
|
| 573 |
+
self.embed_mode = mode
|
| 574 |
+
self.routed: Optional[RoutedEmbedding] = None
|
| 575 |
+
self.kron_embed: Optional[KronEmbedding] = None
|
| 576 |
+
if mode == "routed":
|
| 577 |
+
rc = mc.get("routed", {})
|
| 578 |
+
self.routed = RoutedEmbedding(
|
| 579 |
+
self.vocab_size, self.d,
|
| 580 |
+
v_eff=int(rc.get("v_eff", 5000)),
|
| 581 |
+
route_dim=int(rc.get("route_dim", 6)),
|
| 582 |
+
top_k=int(rc.get("top_k", 10)),
|
| 583 |
+
temperature=float(rc.get("temperature", 1.0)),
|
| 584 |
+
)
|
| 585 |
+
self.embed_rank = 0
|
| 586 |
+
self.embed = None
|
| 587 |
+
self.embed_up = None
|
| 588 |
+
self.lm_head = None
|
| 589 |
+
elif mode == "kron":
|
| 590 |
+
if self.embed_rank <= 0:
|
| 591 |
+
raise ValueError("kron embedding requires embed_rank > 0")
|
| 592 |
+
# V x r table as a sum of Kronecker products; the up-projection
|
| 593 |
+
# U (d x r) is kept, so lookup and the tied head stay identical to
|
| 594 |
+
# the factorised mode -- only the V x r storage is compressed.
|
| 595 |
+
ek = mc.get("embed_kron", {})
|
| 596 |
+
self.kron_embed = KronEmbedding(
|
| 597 |
+
self.vocab_size, self.embed_rank,
|
| 598 |
+
v1=int(ek.get("v1", 64)), v2=int(ek.get("v2", 64)),
|
| 599 |
+
r1=int(ek.get("r1", 16)), r2=int(ek.get("r2", 8)),
|
| 600 |
+
K=int(ek.get("K", 32)))
|
| 601 |
+
self.embed = None
|
| 602 |
+
self.embed_up = nn.Linear(self.embed_rank, self.d, bias=False)
|
| 603 |
+
self.lm_head = None
|
| 604 |
+
elif mode == "factorised":
|
| 605 |
+
if self.embed_rank <= 0:
|
| 606 |
+
raise ValueError("factorised embedding requires embed_rank > 0")
|
| 607 |
+
# ALBERT-style factorised (tied) embedding: V x r then r x d.
|
| 608 |
+
# Keeps the embedding from eating the parameter budget:
|
| 609 |
+
# V*r + r*d instead of V*d
|
| 610 |
+
self.embed = nn.Embedding(self.vocab_size, self.embed_rank)
|
| 611 |
+
self.embed_up = nn.Linear(self.embed_rank, self.d, bias=False)
|
| 612 |
+
self.lm_head = None
|
| 613 |
+
else:
|
| 614 |
+
self.embed_rank = 0
|
| 615 |
+
self.embed = nn.Embedding(self.vocab_size, self.d)
|
| 616 |
+
self.embed_up = None
|
| 617 |
+
self.lm_head = nn.Linear(self.d, self.vocab_size, bias=False)
|
| 618 |
+
if mc["tie_embeddings"]:
|
| 619 |
+
self.lm_head.weight = self.embed.weight
|
| 620 |
+
|
| 621 |
+
self.blocks = nn.ModuleList([KronBlock(cfg)
|
| 622 |
+
for _ in range(self.n_unique)])
|
| 623 |
+
self.norm_f = RMSNorm(self.d, float(mc["norm_eps"]))
|
| 624 |
+
|
| 625 |
+
self._rope_cache: Dict[Tuple[int, torch.device], Tuple[torch.Tensor, torch.Tensor]] = {}
|
| 626 |
+
self.last_trace: Optional[List[Dict[str, torch.Tensor]]] = None
|
| 627 |
+
self.last_stats: Dict[str, float] = {}
|
| 628 |
+
# "auto" | "matrix" | "sequential" -- see KronCompose.forward
|
| 629 |
+
self.force_mode: Optional[str] = None
|
| 630 |
+
self.use_bank = bool(mc.get("batched_prelude", True))
|
| 631 |
+
self._prelude_cache = None
|
| 632 |
+
self._eff_cache: Optional[torch.Tensor] = None
|
| 633 |
+
self.apply(self._init_weights)
|
| 634 |
+
if self.routed is not None:
|
| 635 |
+
self.routed.reset_parameters()
|
| 636 |
+
if self.recurse > 1:
|
| 637 |
+
# Reference the same blocks again instead of duplicating weights;
|
| 638 |
+
# `parameters()` deduplicates, so the parameter count is unchanged.
|
| 639 |
+
unique = list(self.blocks)
|
| 640 |
+
self.blocks = nn.ModuleList(
|
| 641 |
+
[unique[i % self.n_unique] for i in range(self.n_layers)])
|
| 642 |
+
self.max_recurse = self.recurse
|
| 643 |
+
# Optional per-pass learned bias, so a shared block can tell which
|
| 644 |
+
# pass it is in (Universal-Transformer-style step signal).
|
| 645 |
+
self.pass_bias = (nn.Parameter(torch.zeros(self.max_recurse, self.d))
|
| 646 |
+
if bool(mc.get("pass_embedding"))
|
| 647 |
+
and self.max_recurse > 1 else None)
|
| 648 |
+
self._banks: List[KronBank] = self._build_banks() if self.use_bank else []
|
| 649 |
+
# >0: compute the training loss in row chunks of this size (through
|
| 650 |
+
# torch.utils.checkpoint) so that a wide vocabulary never materialises
|
| 651 |
+
# the full B*T*V logits tensor. 0 disables chunking.
|
| 652 |
+
self.loss_chunk = int(mc.get("loss_chunk") or 0)
|
| 653 |
+
|
| 654 |
+
# -- init -------------------------------------------------------------
|
| 655 |
+
def _init_weights(self, module: nn.Module) -> None:
|
| 656 |
+
if isinstance(module, KronCompose):
|
| 657 |
+
return # orthogonal init already applied
|
| 658 |
+
if isinstance(module, nn.Linear):
|
| 659 |
+
# the up-projection of a factorised embedding needs std 1/sqrt(r)
|
| 660 |
+
# so that E@U has the same scale as a direct V x d table
|
| 661 |
+
glu_role = getattr(module, "_glu_role", None)
|
| 662 |
+
if glu_role == "in":
|
| 663 |
+
std = 1.0 / max(1.0, float(self.d) ** 0.5)
|
| 664 |
+
elif glu_role == "out":
|
| 665 |
+
std = 1.0 / max(1.0, float(self._glu_rank) ** 0.5)
|
| 666 |
+
else:
|
| 667 |
+
std = 1.0 / max(1.0, float(self.embed_rank) ** 0.5) \
|
| 668 |
+
if module is getattr(self, "embed_up", None) else 0.02
|
| 669 |
+
nn.init.normal_(module.weight, mean=0.0, std=std)
|
| 670 |
+
if module.bias is not None:
|
| 671 |
+
nn.init.zeros_(module.bias)
|
| 672 |
+
elif isinstance(module, nn.Embedding):
|
| 673 |
+
nn.init.normal_(module.weight, mean=0.0, std=0.02)
|
| 674 |
+
elif isinstance(module, RMSNorm):
|
| 675 |
+
nn.init.ones_(module.weight)
|
| 676 |
+
|
| 677 |
+
# -- banks ------------------------------------------------------------
|
| 678 |
+
def _build_banks(self) -> List[KronBank]:
|
| 679 |
+
groups: Dict[tuple, Tuple[list, list, list]] = {}
|
| 680 |
+
for i, block in enumerate(self.blocks):
|
| 681 |
+
for role in ROLES:
|
| 682 |
+
if role == "f":
|
| 683 |
+
mod = block.ffn
|
| 684 |
+
elif role == "fg":
|
| 685 |
+
mod = block.ffn_g
|
| 686 |
+
else:
|
| 687 |
+
mod = getattr(block, role)
|
| 688 |
+
if not isinstance(mod, KronCompose):
|
| 689 |
+
continue
|
| 690 |
+
if getattr(mod, "dynamic_gate", None):
|
| 691 |
+
# The composed map depends on the input token, so it cannot
|
| 692 |
+
# be fused into one weight-only prelude per layer.
|
| 693 |
+
continue
|
| 694 |
+
if mod.composition in {"grouped", "alternating", "nonlinear"}:
|
| 695 |
+
# These have their own composition path; keep them out of
|
| 696 |
+
# the batched prelude until they are benchmarked.
|
| 697 |
+
continue
|
| 698 |
+
sig = (mod.L, mod.K, mod.d_in, mod.d_out, mod.n1_in, mod.n2_in,
|
| 699 |
+
mod.n1_out, mod.n2_out, mod.anchor, mod.normalize_D,
|
| 700 |
+
mod.stop_gradient, mod.K_win, mod.rezero_eps,
|
| 701 |
+
mod.gate is None, mod.composition, mod.alpha is None)
|
| 702 |
+
g = groups.setdefault(sig, ([], [], []))
|
| 703 |
+
g[0].append(mod)
|
| 704 |
+
g[1].append(i)
|
| 705 |
+
g[2].append((i, role))
|
| 706 |
+
banks: List[KronBank] = []
|
| 707 |
+
for mods, layers, keys in groups.values():
|
| 708 |
+
if len(mods) < 2:
|
| 709 |
+
continue # nothing to batch
|
| 710 |
+
banks.append(KronBank(mods, layers, keys))
|
| 711 |
+
return banks
|
| 712 |
+
|
| 713 |
+
def _prelude(self) -> Tuple[Dict[Tuple[int, str], torch.Tensor],
|
| 714 |
+
Dict[Tuple[int, str], torch.Tensor]]:
|
| 715 |
+
# In eval mode theta cannot change between calls, so the Kronecker
|
| 716 |
+
# prelude is computed once and reused. Generation calls forward() once
|
| 717 |
+
# per token; without this cache every sampled token would redo the
|
| 718 |
+
# L-fold composition (40 MFLOP vs 11 MFLOP per token at d=192).
|
| 719 |
+
if not self.training and self._prelude_cache is not None:
|
| 720 |
+
return self._prelude_cache
|
| 721 |
+
P_map: Dict[Tuple[int, str], torch.Tensor] = {}
|
| 722 |
+
D_map: Dict[Tuple[int, str], torch.Tensor] = {}
|
| 723 |
+
for bank in self._banks:
|
| 724 |
+
P, D = bank.prelude()
|
| 725 |
+
for j, key in enumerate(bank.keys):
|
| 726 |
+
P_map[key] = P[j]
|
| 727 |
+
D_map[key] = D[j]
|
| 728 |
+
if not self.training:
|
| 729 |
+
self._prelude_cache = (P_map, D_map)
|
| 730 |
+
return P_map, D_map
|
| 731 |
+
|
| 732 |
+
def train(self, mode: bool = True):
|
| 733 |
+
self._prelude_cache = None # weights may change -> recompute
|
| 734 |
+
self._eff_cache = None
|
| 735 |
+
return super().train(mode)
|
| 736 |
+
|
| 737 |
+
# -- rope -------------------------------------------------------------
|
| 738 |
+
def _rope(self, T: int, device: torch.device) -> Tuple[torch.Tensor, torch.Tensor]:
|
| 739 |
+
key = (T, device)
|
| 740 |
+
if key not in self._rope_cache:
|
| 741 |
+
cos, sin = rope_frequencies(T, self.d_head, self.rope_theta, device)
|
| 742 |
+
self._rope_cache[key] = (cos.view(1, 1, T, -1), sin.view(1, 1, T, -1))
|
| 743 |
+
return self._rope_cache[key]
|
| 744 |
+
|
| 745 |
+
# -- embedding / head --------------------------------------------------
|
| 746 |
+
def _effective_embeddings(self) -> torch.Tensor:
|
| 747 |
+
"""Return the routed ``(V, d)`` table, cached in evaluation mode."""
|
| 748 |
+
if not self.training and self._eff_cache is not None:
|
| 749 |
+
return self._eff_cache
|
| 750 |
+
assert self.routed is not None
|
| 751 |
+
eff = self.routed.effective_embeddings()
|
| 752 |
+
if not self.training:
|
| 753 |
+
self._eff_cache = eff
|
| 754 |
+
return eff
|
| 755 |
+
|
| 756 |
+
def set_route_top_k(self, top_k: int) -> None:
|
| 757 |
+
"""Change the number of embedding rows mixed per token.
|
| 758 |
+
|
| 759 |
+
Args:
|
| 760 |
+
top_k: New value, must be in ``[1, v_eff]``.
|
| 761 |
+
|
| 762 |
+
Raises:
|
| 763 |
+
RuntimeError: If the model does not use routed embeddings.
|
| 764 |
+
"""
|
| 765 |
+
if self.routed is None:
|
| 766 |
+
raise RuntimeError("set_route_top_k requires embed_mode='routed'")
|
| 767 |
+
self.routed.set_top_k(top_k)
|
| 768 |
+
self._eff_cache = None
|
| 769 |
+
|
| 770 |
+
def _embed_tokens(self, idx: torch.Tensor) -> torch.Tensor:
|
| 771 |
+
if self.routed is not None:
|
| 772 |
+
return F.embedding(idx, self._effective_embeddings())
|
| 773 |
+
x = F.embedding(idx, self.embed_table())
|
| 774 |
+
if self.embed_up is not None:
|
| 775 |
+
x = self.embed_up(x)
|
| 776 |
+
return x
|
| 777 |
+
|
| 778 |
+
def embed_table(self) -> torch.Tensor:
|
| 779 |
+
"""The ``V x r`` lookup table: the kron composition or the dense factor.
|
| 780 |
+
|
| 781 |
+
The kron table is materialised fresh on every call; it costs
|
| 782 |
+
``K * V * r`` MACs, which is amortised over the whole forward pass and
|
| 783 |
+
never cached (caching a tensor that carries a training-time graph would
|
| 784 |
+
leak gradients between forwards).
|
| 785 |
+
"""
|
| 786 |
+
if self.kron_embed is not None:
|
| 787 |
+
return self.kron_embed.table()
|
| 788 |
+
assert self.embed is not None
|
| 789 |
+
return self.embed.weight
|
| 790 |
+
|
| 791 |
+
def _logits(self, x: torch.Tensor,
|
| 792 |
+
eff: Optional[torch.Tensor] = None,
|
| 793 |
+
emb: Optional[torch.Tensor] = None) -> torch.Tensor:
|
| 794 |
+
if eff is not None:
|
| 795 |
+
# The routed table is the input embedding and the tied head at the
|
| 796 |
+
# same time: logits[v] = h . E_eff[v].
|
| 797 |
+
return torch.matmul(x, eff.t())
|
| 798 |
+
if self.embed_up is not None:
|
| 799 |
+
# tied: h (d) -> r via U (shape d x r) -> V via E^T (r x V).
|
| 800 |
+
# Costs d*r + r*V instead of d*V.
|
| 801 |
+
w = emb if emb is not None else self.embed_table()
|
| 802 |
+
return torch.matmul(torch.matmul(x, self.embed_up.weight), w.t())
|
| 803 |
+
return self.lm_head(x)
|
| 804 |
+
|
| 805 |
+
def _ce_chunk(self, xc: torch.Tensor, yc: torch.Tensor) -> torch.Tensor:
|
| 806 |
+
"""Cross-entropy over one row chunk; the checkpointed unit of
|
| 807 |
+
:meth:`_chunked_loss`. Runs under ``no_grad`` in the forward (freeing
|
| 808 |
+
the chunk's logits) and is recomputed in the backward, so at most one
|
| 809 |
+
chunk of ``chunk x V`` logits exists at a time. All head weights it
|
| 810 |
+
reads are parameters (leaves), so gradients flow; the kron-embed table
|
| 811 |
+
is rebuilt from its factor parameters inside the recompute."""
|
| 812 |
+
logits = self._logits(xc)
|
| 813 |
+
return F.cross_entropy(logits, yc, reduction="sum")
|
| 814 |
+
|
| 815 |
+
def _chunked_loss(self, x: torch.Tensor, targets: torch.Tensor) -> torch.Tensor:
|
| 816 |
+
"""Mean cross-entropy in row chunks of ``loss_chunk`` rows.
|
| 817 |
+
|
| 818 |
+
A wide vocabulary makes the full ``B*T*V`` logits tensor the largest
|
| 819 |
+
single allocation of the training step (100 MB at 32x192x4096 fp32).
|
| 820 |
+
Each chunk runs under :func:`torch.utils.checkpoint.checkpoint`, so
|
| 821 |
+
only one chunk of logits is alive at a time and the backward recomputes
|
| 822 |
+
them on demand. Mathematically identical to the unchunked loss. Not
|
| 823 |
+
used for routed embeddings: their effective table is a non-leaf tensor
|
| 824 |
+
whose gradients checkpointing would detach.
|
| 825 |
+
"""
|
| 826 |
+
y = targets.reshape(-1)
|
| 827 |
+
xr = x.reshape(-1, x.size(-1))
|
| 828 |
+
n = y.numel()
|
| 829 |
+
total: Optional[torch.Tensor] = None
|
| 830 |
+
for i in range(0, n, self.loss_chunk):
|
| 831 |
+
chunk = checkpoint(self._ce_chunk, xr[i:i + self.loss_chunk],
|
| 832 |
+
y[i:i + self.loss_chunk], use_reentrant=False)
|
| 833 |
+
total = chunk if total is None else total + chunk
|
| 834 |
+
assert total is not None, "loss_chunk > 0 guarantees one chunk"
|
| 835 |
+
return total / n
|
| 836 |
+
|
| 837 |
+
# -- forward ----------------------------------------------------------
|
| 838 |
+
def forward(self, idx: torch.Tensor, targets: Optional[torch.Tensor] = None,
|
| 839 |
+
collect: bool = False,
|
| 840 |
+
past_kv: Optional[List[Tuple[torch.Tensor, torch.Tensor]]] = None,
|
| 841 |
+
use_cache: bool = False) -> Dict[str, Any]:
|
| 842 |
+
B, T = idx.shape
|
| 843 |
+
past_len = int(past_kv[0][0].shape[2]) if past_kv else 0
|
| 844 |
+
cos, sin = self._rope(past_len + T, idx.device)
|
| 845 |
+
cos, sin = cos[:, :, past_len:], sin[:, :, past_len:]
|
| 846 |
+
eff = self._effective_embeddings() if self.routed is not None else None
|
| 847 |
+
x = F.embedding(idx, eff) if eff is not None else self._embed_tokens(idx)
|
| 848 |
+
P_map: Dict[Tuple[int, str], torch.Tensor] = {}
|
| 849 |
+
D_map: Dict[Tuple[int, str], torch.Tensor] = {}
|
| 850 |
+
if self._banks:
|
| 851 |
+
P_map, D_map = self._prelude()
|
| 852 |
+
|
| 853 |
+
state: Dict[str, Optional[torch.Tensor]] = {r: None for r in ROLES}
|
| 854 |
+
trace: List[Dict[str, Any]] = []
|
| 855 |
+
stats: Dict[str, float] = {}
|
| 856 |
+
|
| 857 |
+
h_norms = [x.detach().norm().item()]
|
| 858 |
+
new_kv: Optional[List[Tuple[torch.Tensor, torch.Tensor]]] = (
|
| 859 |
+
[] if use_cache else None)
|
| 860 |
+
for i, block in enumerate(self.blocks):
|
| 861 |
+
if self.pass_bias is not None and i % self.n_unique == 0:
|
| 862 |
+
x = x + self.pass_bias[i // self.n_unique]
|
| 863 |
+
if D_map:
|
| 864 |
+
for role in ROLES:
|
| 865 |
+
if (i, role) in D_map:
|
| 866 |
+
state[role] = D_map[(i, role)]
|
| 867 |
+
P = ({role: P_map.get((i, role)) for role in ROLES}
|
| 868 |
+
if P_map else None)
|
| 869 |
+
x, state, cache, kv = block(
|
| 870 |
+
x, state, i, cos, sin, collect=collect,
|
| 871 |
+
mode=self.force_mode or "auto", P=P,
|
| 872 |
+
past_kv=past_kv[i] if past_kv is not None else None,
|
| 873 |
+
use_cache=use_cache)
|
| 874 |
+
if new_kv is not None:
|
| 875 |
+
new_kv.append(kv)
|
| 876 |
+
if collect:
|
| 877 |
+
stats.update({f"L{i}.{k}": v for k, v in cache.items()})
|
| 878 |
+
trace.append({
|
| 879 |
+
"D": {k: v.detach().clone() for k, v in state.items()
|
| 880 |
+
if v is not None},
|
| 881 |
+
"s": dict(block.s_values),
|
| 882 |
+
})
|
| 883 |
+
h_norms.append(x.detach().norm().item())
|
| 884 |
+
|
| 885 |
+
x = self.norm_f(x)
|
| 886 |
+
emb = self.embed_table() if self.kron_embed is not None else None
|
| 887 |
+
|
| 888 |
+
if (targets is not None and self.loss_chunk > 0
|
| 889 |
+
and self.routed is None):
|
| 890 |
+
# Chunked loss: never materialise the full B*T*V logits tensor.
|
| 891 |
+
# Used for training, validation and diagnostics alike; `collect`
|
| 892 |
+
# still records the trace and stats from the block outputs.
|
| 893 |
+
out: Dict[str, Any] = {"loss": self._chunked_loss(x, targets)}
|
| 894 |
+
else:
|
| 895 |
+
logits = self._logits(x, eff, emb)
|
| 896 |
+
out = {"logits": logits}
|
| 897 |
+
if targets is not None:
|
| 898 |
+
out["loss"] = F.cross_entropy(
|
| 899 |
+
logits.view(-1, self.vocab_size), targets.reshape(-1))
|
| 900 |
+
if collect:
|
| 901 |
+
ratios = [(h_norms[i + 1] + 1e-9) / (h_norms[i] + 1e-9)
|
| 902 |
+
for i in range(len(h_norms) - 1)]
|
| 903 |
+
stats["h_norm_ratio_min"] = min(ratios)
|
| 904 |
+
stats["h_norm_ratio_max"] = max(ratios)
|
| 905 |
+
stats["h_norm_ratio_mean"] = sum(ratios) / len(ratios)
|
| 906 |
+
self.last_trace = trace
|
| 907 |
+
self.last_stats = stats
|
| 908 |
+
out["trace"] = trace
|
| 909 |
+
out["stats"] = stats
|
| 910 |
+
if use_cache:
|
| 911 |
+
out["past_kv"] = new_kv
|
| 912 |
+
return out
|
| 913 |
+
|
| 914 |
+
def set_recurse(self, recurse: int) -> None:
|
| 915 |
+
"""Change the number of block passes at inference time.
|
| 916 |
+
|
| 917 |
+
The blocks are shared, so this only rebuilds the execution schedule and
|
| 918 |
+
the batched prelude. A model trained with ``recurse=1`` is *not*
|
| 919 |
+
guaranteed to work at ``recurse>1``; this method exists to measure that
|
| 920 |
+
extrapolation explicitly.
|
| 921 |
+
"""
|
| 922 |
+
recurse = int(recurse)
|
| 923 |
+
if recurse < 1:
|
| 924 |
+
raise ValueError("recurse must be >= 1")
|
| 925 |
+
unique = list(self.blocks[:self.n_unique])
|
| 926 |
+
self.recurse = recurse
|
| 927 |
+
self.n_layers = self.n_unique * recurse
|
| 928 |
+
self.blocks = nn.ModuleList(
|
| 929 |
+
[unique[i % self.n_unique] for i in range(self.n_layers)])
|
| 930 |
+
self._prelude_cache = None
|
| 931 |
+
self._banks = self._build_banks() if self.use_bank else []
|
| 932 |
+
|
| 933 |
+
# -- introspection ----------------------------------------------------
|
| 934 |
+
@property
|
| 935 |
+
def n_params(self) -> int:
|
| 936 |
+
return sum(p.numel() for p in self.parameters())
|
| 937 |
+
|
| 938 |
+
@property
|
| 939 |
+
def n_params_matrix(self) -> int:
|
| 940 |
+
"""Parameters in matrix transforms (Q,K,V,O,FFN), excluding norms/embed."""
|
| 941 |
+
total = 0
|
| 942 |
+
for name, p in self.named_parameters():
|
| 943 |
+
if ("norm" in name or "embed" in name or "lm_head" in name
|
| 944 |
+
or "route" in name or "bank" in name or "pass_bias" in name):
|
| 945 |
+
continue
|
| 946 |
+
total += p.numel()
|
| 947 |
+
return total
|
| 948 |
+
|
| 949 |
+
def kron_modules(self) -> List[Tuple[int, str, KronCompose]]:
|
| 950 |
+
out: List[Tuple[int, str, KronCompose]] = []
|
| 951 |
+
for i, block in enumerate(self.blocks):
|
| 952 |
+
for role in ROLES:
|
| 953 |
+
if role == "f":
|
| 954 |
+
mod = block.ffn
|
| 955 |
+
elif role == "fg":
|
| 956 |
+
mod = block.ffn_g
|
| 957 |
+
else:
|
| 958 |
+
mod = getattr(block, role)
|
| 959 |
+
if isinstance(mod, KronCompose):
|
| 960 |
+
out.append((i, role, mod))
|
| 961 |
+
return out
|
| 962 |
+
|
| 963 |
+
def flops_per_token(self, seq_len: Optional[int] = None, training: bool = False,
|
| 964 |
+
batch_size: int = 1) -> float:
|
| 965 |
+
"""Analytic forward FLOPs/token (MACs * 2).
|
| 966 |
+
|
| 967 |
+
With ``batched_prelude`` the L-fold composition is always fused into a
|
| 968 |
+
single d x d product, so the per-token cost is ``d^2`` plus the
|
| 969 |
+
amortised ``L*d^3/tokens`` product.
|
| 970 |
+
"""
|
| 971 |
+
T = self.seq_len if seq_len is None else seq_len
|
| 972 |
+
tokens = max(1, int(T) * int(batch_size))
|
| 973 |
+
mc = self.cfg["model"]
|
| 974 |
+
d = self.d
|
| 975 |
+
fused = self.use_bank or tokens >= max(256, d)
|
| 976 |
+
composition = str(mc["kron"].get("composition", "product"))
|
| 977 |
+
|
| 978 |
+
def compose_cost(out_d: int, L: int) -> float:
|
| 979 |
+
if composition == "nonlinear":
|
| 980 |
+
# ceil(L/k) fused GEMMs per token; k=1 is L matvecs, k=L is
|
| 981 |
+
# one composed GEMM plus one activation.
|
| 982 |
+
k = max(1, int(mc["kron"].get("nonlinear_every", 1)))
|
| 983 |
+
blocks = math.ceil(L / k)
|
| 984 |
+
extra = L * d if mc["kron"].get("norm_between") else 0
|
| 985 |
+
return blocks * d * out_d + L * d + extra
|
| 986 |
+
if fused and out_d == d and L > 1:
|
| 987 |
+
if composition == "sum":
|
| 988 |
+
return d * out_d + L * d * d + L * d
|
| 989 |
+
if composition == "grouped":
|
| 990 |
+
groups = max(1, int(mc["kron"].get("groups", 1)))
|
| 991 |
+
return (d * out_d + L * d * d
|
| 992 |
+
+ (groups - 1) * d * d * out_d / tokens + L * d)
|
| 993 |
+
if composition == "alternating":
|
| 994 |
+
start = str(mc["kron"].get("alternate_start", "sum"))
|
| 995 |
+
products = ((L + 1) // 2 if start == "product" else L // 2)
|
| 996 |
+
return (d * out_d + L * d * d
|
| 997 |
+
+ products * d * d * out_d / tokens + L * d)
|
| 998 |
+
return d * out_d + L * d * d * out_d / tokens + L * d
|
| 999 |
+
return L * d * out_d + L * d
|
| 1000 |
+
|
| 1001 |
+
macs = 0.0
|
| 1002 |
+
for block in self.blocks:
|
| 1003 |
+
d_kv = block.d_head * block.n_kv_heads
|
| 1004 |
+
if block.use_kron_qkv:
|
| 1005 |
+
macs += compose_cost(d, block.L)
|
| 1006 |
+
macs += compose_cost(d_kv, block.k.L)
|
| 1007 |
+
macs += compose_cost(d_kv, block.v.L)
|
| 1008 |
+
macs += compose_cost(d, block.L)
|
| 1009 |
+
else:
|
| 1010 |
+
macs += 2 * d * d + 2 * d * d_kv
|
| 1011 |
+
macs += 2 * T * d # QK^T + AV
|
| 1012 |
+
if block.use_kron_ffn and block.ffn_mix == "glu":
|
| 1013 |
+
r = block.ffn_mix_rank
|
| 1014 |
+
macs += 3 * d * r # W_a, W_b, W_out
|
| 1015 |
+
if block.ffn_mix_kron:
|
| 1016 |
+
macs += compose_cost(d, block.L)
|
| 1017 |
+
elif block.use_kron_ffn:
|
| 1018 |
+
macs += compose_cost(d, block.L)
|
| 1019 |
+
if block.ffn_gated:
|
| 1020 |
+
macs += compose_cost(d, block.L)
|
| 1021 |
+
else:
|
| 1022 |
+
hidden = int(round(d * float(mc.get("dense_ffn_mult", 1.0))))
|
| 1023 |
+
macs += 2 * d * hidden
|
| 1024 |
+
if self.routed is not None:
|
| 1025 |
+
# Router logits + top-k bank mixture + tied effective head, all
|
| 1026 |
+
# amortised over the tokens in the forward pass.
|
| 1027 |
+
macs += (self.vocab_size * self.routed.route_dim * self.routed.v_eff
|
| 1028 |
+
+ self.vocab_size * self.routed.top_k * d
|
| 1029 |
+
+ d * self.vocab_size) / tokens
|
| 1030 |
+
elif self.embed_up is not None:
|
| 1031 |
+
r = float(self.embed_rank)
|
| 1032 |
+
macs += d * r + r * self.vocab_size # factorised tied head
|
| 1033 |
+
if self.kron_embed is not None:
|
| 1034 |
+
# Materialising the V x r kron table, amortised over the
|
| 1035 |
+
# forward pass' tokens.
|
| 1036 |
+
macs += self.kron_embed.K * self.vocab_size * r / tokens
|
| 1037 |
+
else:
|
| 1038 |
+
macs += d * self.vocab_size # lm head
|
| 1039 |
+
macs += 2 * d
|
| 1040 |
+
return 2.0 * macs * (3.0 if training else 1.0)
|
| 1041 |
+
|
| 1042 |
+
@torch.no_grad()
|
| 1043 |
+
def generate(self, idx: torch.Tensor, max_new_tokens: int = 200,
|
| 1044 |
+
temperature: float = 1.0, top_k: Optional[int] = 8,
|
| 1045 |
+
eot_id: Optional[int] = None,
|
| 1046 |
+
stop_at_eot: bool = False,
|
| 1047 |
+
use_cache: bool = True) -> torch.Tensor:
|
| 1048 |
+
"""Sample tokens; with ``use_cache`` each step attends to cached K/V.
|
| 1049 |
+
|
| 1050 |
+
The cache holds at most ``seq_len`` tokens; once the context is full,
|
| 1051 |
+
generation falls back to a full forward on the last ``seq_len`` tokens
|
| 1052 |
+
(the behaviour of the uncached path).
|
| 1053 |
+
"""
|
| 1054 |
+
was_training = self.training
|
| 1055 |
+
eot_id = int(self.eot_id if eot_id is None else eot_id)
|
| 1056 |
+
self.eval()
|
| 1057 |
+
|
| 1058 |
+
def sample(logits: torch.Tensor) -> torch.Tensor:
|
| 1059 |
+
logits = logits / max(temperature, 1e-5)
|
| 1060 |
+
if top_k is not None and top_k > 0:
|
| 1061 |
+
k = min(top_k, logits.size(-1))
|
| 1062 |
+
thresh = torch.topk(logits, k, dim=-1).values[:, -1:]
|
| 1063 |
+
logits = logits.masked_fill(logits < thresh, float("-inf"))
|
| 1064 |
+
probs = torch.softmax(logits, dim=-1)
|
| 1065 |
+
return torch.multinomial(probs, num_samples=1)
|
| 1066 |
+
|
| 1067 |
+
past = None
|
| 1068 |
+
for _ in range(max_new_tokens):
|
| 1069 |
+
if use_cache and past is not None and past[0][0].shape[2] < self.seq_len:
|
| 1070 |
+
out = self(idx[:, -1:], past_kv=past, use_cache=True)
|
| 1071 |
+
past = out["past_kv"]
|
| 1072 |
+
else:
|
| 1073 |
+
ctx = idx[:, -self.seq_len:]
|
| 1074 |
+
out = self(ctx, use_cache=use_cache)
|
| 1075 |
+
past = out.get("past_kv")
|
| 1076 |
+
nxt = sample(out["logits"][:, -1, :].float())
|
| 1077 |
+
idx = torch.cat([idx, nxt], dim=1)
|
| 1078 |
+
if stop_at_eot and int(nxt[0, 0]) == eot_id:
|
| 1079 |
+
break
|
| 1080 |
+
if was_training:
|
| 1081 |
+
self.train()
|
| 1082 |
+
return idx
|
| 1083 |
+
|
| 1084 |
+
@property
|
| 1085 |
+
def device(self) -> torch.device:
|
| 1086 |
+
return next(self.parameters()).device
|
kronllm/plots.py
ADDED
|
@@ -0,0 +1,134 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""Plotting helpers shared by the dashboard and notebooks."""
|
| 2 |
+
from __future__ import annotations
|
| 3 |
+
|
| 4 |
+
from pathlib import Path
|
| 5 |
+
from typing import Any, Dict, Optional
|
| 6 |
+
|
| 7 |
+
import matplotlib
|
| 8 |
+
matplotlib.use("Agg")
|
| 9 |
+
import matplotlib.pyplot as plt # noqa: E402
|
| 10 |
+
import pandas as pd # noqa: E402
|
| 11 |
+
|
| 12 |
+
COLORS = {"train": "#3b82f6", "val": "#ef4444", "diag": "#10b981"}
|
| 13 |
+
|
| 14 |
+
|
| 15 |
+
def read_log(path: str | Path) -> pd.DataFrame:
|
| 16 |
+
path = Path(path)
|
| 17 |
+
if not path.exists():
|
| 18 |
+
return pd.DataFrame()
|
| 19 |
+
try:
|
| 20 |
+
df = pd.read_csv(path)
|
| 21 |
+
except Exception:
|
| 22 |
+
return pd.DataFrame()
|
| 23 |
+
return df
|
| 24 |
+
|
| 25 |
+
|
| 26 |
+
def _ensure_axes(fig, n: int = 1):
|
| 27 |
+
"""Reuse the axes of a long-lived figure (avoids leaking figures in the UI)."""
|
| 28 |
+
while len(fig.axes) < n:
|
| 29 |
+
fig.add_subplot(1, n, len(fig.axes) + 1)
|
| 30 |
+
return fig.axes
|
| 31 |
+
|
| 32 |
+
|
| 33 |
+
def loss_curves(log_path: str | Path, smooth: int = 10, fig=None):
|
| 34 |
+
"""Train (smoothed) + validation loss from a training ``log.csv``."""
|
| 35 |
+
df = read_log(log_path)
|
| 36 |
+
if df.empty or "split" not in df:
|
| 37 |
+
return None
|
| 38 |
+
if fig is None:
|
| 39 |
+
fig = plt.figure(figsize=(7, 3.6))
|
| 40 |
+
ax = _ensure_axes(fig, 1)[0]
|
| 41 |
+
ax.clear()
|
| 42 |
+
tr = df[df["split"] == "train"].dropna(subset=["loss"])
|
| 43 |
+
va = df[df["split"] == "val"].dropna(subset=["loss"])
|
| 44 |
+
if len(tr):
|
| 45 |
+
y = tr["loss"].rolling(min(smooth, max(1, len(tr))), min_periods=1).mean()
|
| 46 |
+
ax.plot(tr["step"], tr["loss"], color=COLORS["train"], alpha=0.25, lw=1)
|
| 47 |
+
ax.plot(tr["step"], y, color=COLORS["train"], label="train (smooth)")
|
| 48 |
+
if len(va):
|
| 49 |
+
ax.plot(va["step"], va["loss"], color=COLORS["val"], marker="o",
|
| 50 |
+
ms=3, lw=1.5, label="val")
|
| 51 |
+
ax.set_xlabel("optimizer step")
|
| 52 |
+
ax.set_ylabel("cross-entropy")
|
| 53 |
+
ax.set_title("loss curves")
|
| 54 |
+
ax.grid(alpha=0.25)
|
| 55 |
+
ax.legend(loc="best", fontsize=8)
|
| 56 |
+
fig.tight_layout()
|
| 57 |
+
return fig
|
| 58 |
+
|
| 59 |
+
|
| 60 |
+
def diag_curves(log_path: str | Path, fig=None):
|
| 61 |
+
df = read_log(log_path)
|
| 62 |
+
if df.empty or "split" not in df:
|
| 63 |
+
return None
|
| 64 |
+
dg = df[df["split"] == "diag"]
|
| 65 |
+
if dg.empty:
|
| 66 |
+
return None
|
| 67 |
+
cols = [c for c in ("sigma_theta_mean", "sigma_theta_var", "attn_entropy_mean",
|
| 68 |
+
"h_norm_ratio_mean", "eff_rank_mean", "eff_rank_mean",
|
| 69 |
+
"q_drift_mean") if c in dg]
|
| 70 |
+
if not cols:
|
| 71 |
+
return None
|
| 72 |
+
if fig is None:
|
| 73 |
+
fig = plt.figure(figsize=(7, 3.2))
|
| 74 |
+
ax = _ensure_axes(fig, 1)[0]
|
| 75 |
+
ax.clear()
|
| 76 |
+
for c in cols:
|
| 77 |
+
ax.plot(dg["step"], dg[c], marker=".", label=c)
|
| 78 |
+
ax.set_xlabel("step")
|
| 79 |
+
ax.set_title("diagnostics")
|
| 80 |
+
ax.grid(alpha=0.25)
|
| 81 |
+
ax.legend(fontsize=7, ncol=2)
|
| 82 |
+
fig.tight_layout()
|
| 83 |
+
return fig
|
| 84 |
+
|
| 85 |
+
|
| 86 |
+
def read_summary(results_dir: str | Path) -> pd.DataFrame:
|
| 87 |
+
path = Path(results_dir) / "summary.csv"
|
| 88 |
+
if not path.exists():
|
| 89 |
+
return pd.DataFrame()
|
| 90 |
+
try:
|
| 91 |
+
return pd.read_csv(path)
|
| 92 |
+
except Exception:
|
| 93 |
+
return pd.DataFrame()
|
| 94 |
+
|
| 95 |
+
|
| 96 |
+
def compare_plot(results_dir: str | Path, fig=None):
|
| 97 |
+
df = read_summary(results_dir)
|
| 98 |
+
if df.empty:
|
| 99 |
+
return None
|
| 100 |
+
df = df.dropna(subset=["val_ppl"])
|
| 101 |
+
if fig is None:
|
| 102 |
+
fig = plt.figure(figsize=(11, 3.8))
|
| 103 |
+
axes = _ensure_axes(fig, 2)
|
| 104 |
+
for a in axes[:2]:
|
| 105 |
+
a.clear()
|
| 106 |
+
exp = df["experiment"].astype(str)
|
| 107 |
+
axes[0].bar(exp, df["val_ppl"], color="#6366f1")
|
| 108 |
+
axes[0].set_ylabel("val perplexity")
|
| 109 |
+
axes[0].set_title("E-series: validation PPL")
|
| 110 |
+
axes[0].grid(alpha=0.25, axis="y")
|
| 111 |
+
axes[1].scatter(df["n_params"], df["val_ppl"], c="#ef4444", s=40)
|
| 112 |
+
for _, row in df.iterrows():
|
| 113 |
+
axes[1].annotate(str(row["experiment"]),
|
| 114 |
+
(row["n_params"], row["val_ppl"]),
|
| 115 |
+
fontsize=7, xytext=(3, 3), textcoords="offset points")
|
| 116 |
+
axes[1].set_xscale("log")
|
| 117 |
+
axes[1].set_xlabel("parameters")
|
| 118 |
+
axes[1].set_ylabel("val perplexity")
|
| 119 |
+
axes[1].set_title("quality vs. budget")
|
| 120 |
+
axes[1].grid(alpha=0.25)
|
| 121 |
+
fig.tight_layout()
|
| 122 |
+
return fig
|
| 123 |
+
|
| 124 |
+
|
| 125 |
+
def results_table(results_dir: str | Path) -> pd.DataFrame:
|
| 126 |
+
df = read_summary(results_dir)
|
| 127 |
+
if df.empty:
|
| 128 |
+
return pd.DataFrame({"info": ["no experiments finished yet"]})
|
| 129 |
+
cols = [c for c in ("experiment", "n_params", "n_params_matrix", "val_ppl",
|
| 130 |
+
"ppl_ratio_vs_E0", "sec_per_step", "eff_rank_D",
|
| 131 |
+
"eff_rank_W", "eff_rank_P", "cos_prev_max",
|
| 132 |
+
"tau_mean", "gate_mean", "eff_rank_spearman",
|
| 133 |
+
"sigma_theta_var", "peak_vram_gb") if c in df]
|
| 134 |
+
return df[cols].sort_values("val_ppl")
|
kronllm/quantization.py
ADDED
|
@@ -0,0 +1,233 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""Post-training quantization for KronLLM checkpoints.
|
| 2 |
+
|
| 3 |
+
KronLLM does not contain any ``nn.Linear`` layer: every weight matrix is a
|
| 4 |
+
Kronecker composition of small factors, so ``torch.ao.quantization`` cannot see
|
| 5 |
+
it. Instead we quantize the *stored* tensors -- factors, embeddings and the
|
| 6 |
+
tied head -- with symmetric per-channel scales and dequantize them when the
|
| 7 |
+
checkpoint is loaded. The forward pass is therefore bit-identical in structure
|
| 8 |
+
and only sees float weights; the win is a 4x (int8) or 8x (int4) smaller file,
|
| 9 |
+
which is what makes the model practical on a phone or a microcontroller.
|
| 10 |
+
|
| 11 |
+
Typical use::
|
| 12 |
+
|
| 13 |
+
from kronllm.quantization import quantize_checkpoint, load_any_checkpoint
|
| 14 |
+
|
| 15 |
+
report = quantize_checkpoint("runs/Q2/best.pt", "runs/Q2/model_int8.pt", bits=8)
|
| 16 |
+
cfg, state = load_any_checkpoint("runs/Q2/model_int8.pt")
|
| 17 |
+
"""
|
| 18 |
+
from __future__ import annotations
|
| 19 |
+
|
| 20 |
+
import time
|
| 21 |
+
from pathlib import Path
|
| 22 |
+
from typing import Any, Dict, Mapping, Tuple
|
| 23 |
+
|
| 24 |
+
import torch
|
| 25 |
+
|
| 26 |
+
# Tensors smaller than this are stored in float32: scales would cost more than
|
| 27 |
+
# the payload, and 1-D parameters (RMSNorm, tau, gate) must stay exact.
|
| 28 |
+
_MIN_NUMEL = 256
|
| 29 |
+
|
| 30 |
+
# Name fragments that must never be quantized (normalisation and scalars).
|
| 31 |
+
_SKIP_FRAGMENTS = ("norm", "tau", "gate", "theta", "anchor")
|
| 32 |
+
|
| 33 |
+
|
| 34 |
+
def _is_quantizable(name: str, tensor: torch.Tensor) -> bool:
|
| 35 |
+
"""Return True if ``tensor`` should be stored in low precision.
|
| 36 |
+
|
| 37 |
+
Args:
|
| 38 |
+
name: Parameter name inside the state dict.
|
| 39 |
+
tensor: Parameter tensor.
|
| 40 |
+
|
| 41 |
+
Returns:
|
| 42 |
+
True for 2-D (or higher) float tensors of at least ``_MIN_NUMEL``
|
| 43 |
+
elements whose name does not match a normalisation/scalar role.
|
| 44 |
+
"""
|
| 45 |
+
if not tensor.is_floating_point() or tensor.dim() < 2:
|
| 46 |
+
return False
|
| 47 |
+
if tensor.numel() < _MIN_NUMEL:
|
| 48 |
+
return False
|
| 49 |
+
return not any(fragment in name for fragment in _SKIP_FRAGMENTS)
|
| 50 |
+
|
| 51 |
+
|
| 52 |
+
def _pack_int4(codes: torch.Tensor) -> torch.Tensor:
|
| 53 |
+
"""Pack signed int4 codes (range ``[-8, 7]``) two per byte.
|
| 54 |
+
|
| 55 |
+
Args:
|
| 56 |
+
codes: Int8 tensor whose values are already clamped to ``[-8, 7]``.
|
| 57 |
+
|
| 58 |
+
Returns:
|
| 59 |
+
Flat uint8 tensor with the low nibble holding the even elements.
|
| 60 |
+
"""
|
| 61 |
+
flat = codes.reshape(-1).to(torch.int16)
|
| 62 |
+
if flat.numel() % 2:
|
| 63 |
+
flat = torch.cat([flat, torch.zeros(1, dtype=flat.dtype)])
|
| 64 |
+
shifted = flat + 8 # 0..15, safe in int16
|
| 65 |
+
packed = shifted[0::2] | (shifted[1::2] << 4)
|
| 66 |
+
return packed.to(torch.uint8)
|
| 67 |
+
|
| 68 |
+
|
| 69 |
+
def _unpack_int4(packed: torch.Tensor, numel: int) -> torch.Tensor:
|
| 70 |
+
"""Inverse of :func:`_pack_int4`.
|
| 71 |
+
|
| 72 |
+
Args:
|
| 73 |
+
packed: Flat uint8 tensor produced by :func:`_pack_int4`.
|
| 74 |
+
numel: Number of original elements.
|
| 75 |
+
|
| 76 |
+
Returns:
|
| 77 |
+
Int8 tensor of length ``numel`` in ``[-8, 7]``.
|
| 78 |
+
"""
|
| 79 |
+
low = (packed & 0x0F).to(torch.int16) - 8
|
| 80 |
+
high = ((packed >> 4) & 0x0F).to(torch.int16) - 8
|
| 81 |
+
out = torch.empty(packed.numel() * 2, dtype=torch.int16)
|
| 82 |
+
out[0::2] = low
|
| 83 |
+
out[1::2] = high
|
| 84 |
+
return out[:numel].to(torch.int8)
|
| 85 |
+
|
| 86 |
+
|
| 87 |
+
def _quantize_tensor(tensor: torch.Tensor, bits: int
|
| 88 |
+
) -> Tuple[torch.Tensor, torch.Tensor]:
|
| 89 |
+
"""Symmetric per-channel quantization of one tensor.
|
| 90 |
+
|
| 91 |
+
Args:
|
| 92 |
+
tensor: Float tensor to quantize.
|
| 93 |
+
bits: Target bit width, either 8 or 4.
|
| 94 |
+
|
| 95 |
+
Returns:
|
| 96 |
+
A pair ``(codes, scale)``. ``codes`` is int8 (values clamped to the
|
| 97 |
+
signed range) and ``scale`` broadcasts against ``tensor``.
|
| 98 |
+
"""
|
| 99 |
+
qmax = float(2 ** (bits - 1) - 1)
|
| 100 |
+
reduce_dims = tuple(range(1, tensor.dim()))
|
| 101 |
+
amax = tensor.detach().abs().amax(dim=reduce_dims, keepdim=True)
|
| 102 |
+
scale = (amax / qmax).clamp_min(1e-8)
|
| 103 |
+
codes = torch.round(tensor.detach() / scale).clamp(-qmax, qmax).to(torch.int8)
|
| 104 |
+
return codes, scale
|
| 105 |
+
|
| 106 |
+
|
| 107 |
+
def quantize_state_dict(state: Mapping[str, torch.Tensor], bits: int = 8
|
| 108 |
+
) -> Dict[str, Any]:
|
| 109 |
+
"""Quantize every eligible tensor of a state dict.
|
| 110 |
+
|
| 111 |
+
Args:
|
| 112 |
+
state: Float state dict as stored in a training checkpoint.
|
| 113 |
+
bits: 8 or 4.
|
| 114 |
+
|
| 115 |
+
Returns:
|
| 116 |
+
A serialisable dict mapping parameter names to either a float tensor
|
| 117 |
+
(kept exact) or a ``{"q", "scale", "shape"}`` record.
|
| 118 |
+
"""
|
| 119 |
+
if bits not in (8, 4):
|
| 120 |
+
raise ValueError(f"bits must be 8 or 4, got {bits}")
|
| 121 |
+
|
| 122 |
+
out: Dict[str, Any] = {}
|
| 123 |
+
for name, tensor in state.items():
|
| 124 |
+
if not _is_quantizable(name, tensor):
|
| 125 |
+
out[name] = tensor.detach().clone()
|
| 126 |
+
continue
|
| 127 |
+
codes, scale = _quantize_tensor(tensor, bits)
|
| 128 |
+
record: Dict[str, Any] = {"shape": tuple(tensor.shape), "scale": scale}
|
| 129 |
+
if bits == 8:
|
| 130 |
+
record["q"] = codes.reshape(tensor.shape)
|
| 131 |
+
else:
|
| 132 |
+
record["q"] = _pack_int4(codes)
|
| 133 |
+
record["numel"] = tensor.numel()
|
| 134 |
+
out[name] = record
|
| 135 |
+
return out
|
| 136 |
+
|
| 137 |
+
|
| 138 |
+
def dequantize_state_dict(quantized: Mapping[str, Any], bits: int
|
| 139 |
+
) -> Dict[str, torch.Tensor]:
|
| 140 |
+
"""Rebuild a float state dict from :func:`quantize_state_dict` output.
|
| 141 |
+
|
| 142 |
+
Args:
|
| 143 |
+
quantized: Output of :func:`quantize_state_dict`.
|
| 144 |
+
bits: Bit width used during quantization.
|
| 145 |
+
|
| 146 |
+
Returns:
|
| 147 |
+
Float state dict that can be passed to ``model.load_state_dict``.
|
| 148 |
+
"""
|
| 149 |
+
state: Dict[str, torch.Tensor] = {}
|
| 150 |
+
for name, value in quantized.items():
|
| 151 |
+
if not isinstance(value, Mapping):
|
| 152 |
+
state[name] = value.float() if value.is_floating_point() else value
|
| 153 |
+
continue
|
| 154 |
+
scale = value["scale"].float()
|
| 155 |
+
if bits == 8:
|
| 156 |
+
codes = value["q"].float()
|
| 157 |
+
else:
|
| 158 |
+
codes = _unpack_int4(value["q"], int(value["numel"])).float()
|
| 159 |
+
codes = codes.reshape(value["shape"])
|
| 160 |
+
state[name] = codes * scale
|
| 161 |
+
return state
|
| 162 |
+
|
| 163 |
+
|
| 164 |
+
def quantize_checkpoint(src: str | Path, dst: str | Path, bits: int = 8
|
| 165 |
+
) -> Dict[str, Any]:
|
| 166 |
+
"""Quantize a training checkpoint and write a compact checkpoint.
|
| 167 |
+
|
| 168 |
+
Args:
|
| 169 |
+
src: Path to a checkpoint written by ``kronllm.train`` (``best.pt``).
|
| 170 |
+
dst: Destination path for the quantized checkpoint.
|
| 171 |
+
bits: 8 or 4.
|
| 172 |
+
|
| 173 |
+
Returns:
|
| 174 |
+
A report with the original and quantized sizes plus the parameter count.
|
| 175 |
+
|
| 176 |
+
Raises:
|
| 177 |
+
FileNotFoundError: If ``src`` does not exist.
|
| 178 |
+
"""
|
| 179 |
+
src, dst = Path(src), Path(dst)
|
| 180 |
+
if not src.exists():
|
| 181 |
+
raise FileNotFoundError(src)
|
| 182 |
+
|
| 183 |
+
checkpoint = torch.load(src, map_location="cpu", weights_only=False)
|
| 184 |
+
state = checkpoint.get("model", checkpoint)
|
| 185 |
+
quantized = quantize_state_dict(state, bits=bits)
|
| 186 |
+
|
| 187 |
+
payload = {
|
| 188 |
+
"quantized": True,
|
| 189 |
+
"bits": bits,
|
| 190 |
+
"cfg": checkpoint.get("cfg"),
|
| 191 |
+
"step": checkpoint.get("step"),
|
| 192 |
+
"val_ppl": checkpoint.get("val_ppl"),
|
| 193 |
+
"state": quantized,
|
| 194 |
+
}
|
| 195 |
+
dst.parent.mkdir(parents=True, exist_ok=True)
|
| 196 |
+
torch.save(payload, dst)
|
| 197 |
+
|
| 198 |
+
return {
|
| 199 |
+
"src": str(src),
|
| 200 |
+
"dst": str(dst),
|
| 201 |
+
"bits": bits,
|
| 202 |
+
"params": int(sum(v.numel() for v in state.values())),
|
| 203 |
+
"size_before_mb": round(src.stat().st_size / 1e6, 3),
|
| 204 |
+
"size_after_mb": round(dst.stat().st_size / 1e6, 3),
|
| 205 |
+
"compression": round(src.stat().st_size / dst.stat().st_size, 2),
|
| 206 |
+
"timestamp": time.strftime("%Y-%m-%dT%H:%M:%S"),
|
| 207 |
+
}
|
| 208 |
+
|
| 209 |
+
|
| 210 |
+
def load_any_checkpoint(path: str | Path) -> Tuple[Dict[str, Any], Dict[str, torch.Tensor]]:
|
| 211 |
+
"""Load a float or a quantized checkpoint.
|
| 212 |
+
|
| 213 |
+
Args:
|
| 214 |
+
path: Checkpoint path.
|
| 215 |
+
|
| 216 |
+
Returns:
|
| 217 |
+
A pair ``(cfg, state_dict)`` where ``state_dict`` is always float and
|
| 218 |
+
can be loaded into :class:`kronllm.model.KronTransformer`.
|
| 219 |
+
|
| 220 |
+
Raises:
|
| 221 |
+
ValueError: If the checkpoint is quantized but carries no config.
|
| 222 |
+
"""
|
| 223 |
+
path = Path(path)
|
| 224 |
+
checkpoint = torch.load(path, map_location="cpu", weights_only=False)
|
| 225 |
+
if not checkpoint.get("quantized"):
|
| 226 |
+
cfg = checkpoint.get("cfg")
|
| 227 |
+
return cfg, checkpoint.get("model", checkpoint)
|
| 228 |
+
|
| 229 |
+
cfg = checkpoint.get("cfg")
|
| 230 |
+
if cfg is None:
|
| 231 |
+
raise ValueError(f"quantized checkpoint {path} has no config")
|
| 232 |
+
state = dequantize_state_dict(checkpoint["state"], int(checkpoint["bits"]))
|
| 233 |
+
return cfg, state
|
kronllm/testutils.py
ADDED
|
@@ -0,0 +1,29 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""Config helpers shared by the test suite (also usable from scripts)."""
|
| 2 |
+
from __future__ import annotations
|
| 3 |
+
|
| 4 |
+
from typing import Any, Dict
|
| 5 |
+
|
| 6 |
+
from .config import DEFAULTS, deep_merge
|
| 7 |
+
|
| 8 |
+
|
| 9 |
+
def make_cfg(**overrides: Any) -> Dict[str, Any]:
|
| 10 |
+
"""Small, fast config for unit/integration tests.
|
| 11 |
+
|
| 12 |
+
``overrides`` use dotted keys, e.g. ``{"model.d_model": 64}``.
|
| 13 |
+
"""
|
| 14 |
+
cfg = deep_merge(DEFAULTS, {
|
| 15 |
+
"data": {"seq_len": 64, "vocab_size": 256},
|
| 16 |
+
"model": {
|
| 17 |
+
"d_model": 64, "n_layers": 2, "n_heads": 4,
|
| 18 |
+
"kron": {"n1": 8, "n2": 8},
|
| 19 |
+
},
|
| 20 |
+
"training": {"batch_size": 2, "grad_accum": 1, "amp": "none",
|
| 21 |
+
"device": "cpu", "warmup_steps": 5, "eval_every": 1000},
|
| 22 |
+
})
|
| 23 |
+
for key, value in overrides.items():
|
| 24 |
+
node = cfg
|
| 25 |
+
parts = key.split(".")
|
| 26 |
+
for p in parts[:-1]:
|
| 27 |
+
node = node.setdefault(p, {})
|
| 28 |
+
node[parts[-1]] = value
|
| 29 |
+
return cfg
|
kronllm/tokenizer.py
ADDED
|
@@ -0,0 +1,197 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""Tokenizers: the byte baseline and a byte-level BPE.
|
| 2 |
+
|
| 3 |
+
Both expose the same tiny interface::
|
| 4 |
+
|
| 5 |
+
tok.vocab_size # int
|
| 6 |
+
tok.bytes_per_token # float, used for bits-per-byte reporting
|
| 7 |
+
tok.eot_id # id reserved for <|endoftext|>
|
| 8 |
+
tok.encode(text) # str -> list[int]
|
| 9 |
+
tok.decode(ids) # list[int] -> str
|
| 10 |
+
|
| 11 |
+
The byte tokenizer is the TZ baseline (V = 256, one token per byte). The BPE
|
| 12 |
+
tokenizer lifts ~3 bytes into one position, which is the principled version of
|
| 13 |
+
"three bytes at a time": instead of a fixed 3-byte chunk (a 16.7M vocabulary)
|
| 14 |
+
the merges are learned, so frequent trigrams such as ``the``/``ing``/``and``
|
| 15 |
+
become single ids while rare byte triples stay split.
|
| 16 |
+
|
| 17 |
+
``bits_per_byte`` is the metric that makes the two comparable::
|
| 18 |
+
|
| 19 |
+
bpb = loss_nats_per_token / bytes_per_token / ln(2)
|
| 20 |
+
"""
|
| 21 |
+
from __future__ import annotations
|
| 22 |
+
|
| 23 |
+
import json
|
| 24 |
+
from pathlib import Path
|
| 25 |
+
from typing import Any, List, Mapping, Sequence
|
| 26 |
+
|
| 27 |
+
import numpy as np
|
| 28 |
+
|
| 29 |
+
EOT_BYTE = 255
|
| 30 |
+
EOT_TEXT = "<|endoftext|>"
|
| 31 |
+
|
| 32 |
+
|
| 33 |
+
# ---------------------------------------------------------------------------
|
| 34 |
+
# byte baseline
|
| 35 |
+
# ---------------------------------------------------------------------------
|
| 36 |
+
class ByteTokenizer:
|
| 37 |
+
name = "byte"
|
| 38 |
+
vocab_size = 256
|
| 39 |
+
bytes_per_token = 1.0
|
| 40 |
+
eot_id = 255
|
| 41 |
+
|
| 42 |
+
def encode(self, text: str) -> List[int]:
|
| 43 |
+
return list(text.encode("utf-8", errors="ignore"))
|
| 44 |
+
|
| 45 |
+
def decode(self, ids: Sequence[int]) -> str:
|
| 46 |
+
clean = [int(i) & 0xFF for i in ids if int(i) != self.eot_id]
|
| 47 |
+
return bytes(clean).decode("utf-8", errors="replace")
|
| 48 |
+
|
| 49 |
+
def __repr__(self) -> str:
|
| 50 |
+
return "ByteTokenizer(vocab=256)"
|
| 51 |
+
|
| 52 |
+
|
| 53 |
+
# ---------------------------------------------------------------------------
|
| 54 |
+
# byte-level BPE
|
| 55 |
+
# ---------------------------------------------------------------------------
|
| 56 |
+
class BPETokenizer:
|
| 57 |
+
name = "bpe"
|
| 58 |
+
|
| 59 |
+
def __init__(self, path: str | Path, vocab_size: int,
|
| 60 |
+
bytes_per_token: float, eot_id: int) -> None:
|
| 61 |
+
from tokenizers import Tokenizer
|
| 62 |
+
|
| 63 |
+
self.path = Path(path)
|
| 64 |
+
self._tok = Tokenizer.from_file(str(self.path))
|
| 65 |
+
self.vocab_size = int(vocab_size)
|
| 66 |
+
self.bytes_per_token = float(bytes_per_token)
|
| 67 |
+
self.eot_id = int(eot_id)
|
| 68 |
+
if self._tok.get_vocab_size() > self.vocab_size:
|
| 69 |
+
raise ValueError(
|
| 70 |
+
f"tokenizer has {self._tok.get_vocab_size()} ids but "
|
| 71 |
+
f"vocab_size={self.vocab_size}")
|
| 72 |
+
|
| 73 |
+
def encode(self, text: str) -> List[int]:
|
| 74 |
+
return self._tok.encode(text).ids
|
| 75 |
+
|
| 76 |
+
def encode_batch(self, texts: Sequence[str]) -> List[List[int]]:
|
| 77 |
+
return [e.ids for e in self._tok.encode_batch(list(texts))]
|
| 78 |
+
|
| 79 |
+
def decode(self, ids: Sequence[int]) -> str:
|
| 80 |
+
clean = [int(i) for i in ids if int(i) != self.eot_id]
|
| 81 |
+
if not clean:
|
| 82 |
+
return ""
|
| 83 |
+
return self._tok.decode(clean, skip_special_tokens=False)
|
| 84 |
+
|
| 85 |
+
def __repr__(self) -> str:
|
| 86 |
+
return (f"BPETokenizer(vocab={self.vocab_size}, "
|
| 87 |
+
f"{self.bytes_per_token:.2f} bytes/token)")
|
| 88 |
+
|
| 89 |
+
|
| 90 |
+
# ---------------------------------------------------------------------------
|
| 91 |
+
# factory
|
| 92 |
+
# ---------------------------------------------------------------------------
|
| 93 |
+
def meta_path(data_dir: str | Path, name: str | None = None) -> Path:
|
| 94 |
+
return Path(data_dir) / (str(name) if name else "tokenizer_meta.json")
|
| 95 |
+
|
| 96 |
+
|
| 97 |
+
def load_tokenizer(cfg: Mapping[str, Any]):
|
| 98 |
+
"""Build the tokenizer described by ``cfg['data']``."""
|
| 99 |
+
dc = cfg["data"]
|
| 100 |
+
kind = str(dc.get("tokenizer", "byte")).lower()
|
| 101 |
+
if kind in ("byte", "bytes", "byte_level"):
|
| 102 |
+
return ByteTokenizer()
|
| 103 |
+
if kind != "bpe":
|
| 104 |
+
raise ValueError(f"unknown tokenizer {kind!r}")
|
| 105 |
+
path = meta_path(dc.get("data_dir", "data"), dc.get("meta_file"))
|
| 106 |
+
if not path.exists():
|
| 107 |
+
raise FileNotFoundError(
|
| 108 |
+
f"{path} missing -- run `python scripts/prepare_bpe.py` first")
|
| 109 |
+
meta = json.loads(path.read_text(encoding="utf-8"))
|
| 110 |
+
return BPETokenizer(meta["tokenizer"], meta["vocab_size"],
|
| 111 |
+
meta["bytes_per_token"], meta["eot_id"])
|
| 112 |
+
|
| 113 |
+
|
| 114 |
+
def bits_per_byte(loss_nats: float, bytes_per_token: float) -> float:
|
| 115 |
+
"""Convert a per-token cross-entropy into bits per *byte* of text.
|
| 116 |
+
|
| 117 |
+
This is the only fair way to compare the byte model (1 token = 1 byte)
|
| 118 |
+
with a BPE model (1 token = ~3 bytes).
|
| 119 |
+
"""
|
| 120 |
+
import math
|
| 121 |
+
|
| 122 |
+
return float(loss_nats) / max(float(bytes_per_token), 1e-9) / math.log(2.0)
|
| 123 |
+
|
| 124 |
+
|
| 125 |
+
# ---------------------------------------------------------------------------
|
| 126 |
+
# training / corpus encoding
|
| 127 |
+
# ---------------------------------------------------------------------------
|
| 128 |
+
def train_bpe_merges(texts, vocab_size: int, out_path: str | Path,
|
| 129 |
+
min_frequency: int = 2,
|
| 130 |
+
special_tokens: Sequence[str] = ()) -> None:
|
| 131 |
+
"""Train a byte-level BPE whose ids are ``0..vocab_size-1``.
|
| 132 |
+
|
| 133 |
+
``vocab_size`` already includes the 256 byte ids and the special tokens.
|
| 134 |
+
The end-of-story id is *not* part of the tokenizer: callers reserve
|
| 135 |
+
``vocab_size`` itself and insert it manually (see ``encode_corpus``).
|
| 136 |
+
|
| 137 |
+
``special_tokens`` are added after the merges, in the given order, so their
|
| 138 |
+
ids are deterministic: ``vocab_size, vocab_size+1, ...``. The chat
|
| 139 |
+
template (``<|user|>``, ``<|assistant|>``) lives here so fine-tuning and
|
| 140 |
+
inference share one encoding.
|
| 141 |
+
"""
|
| 142 |
+
from tokenizers import Tokenizer, decoders, models, pre_tokenizers, trainers
|
| 143 |
+
|
| 144 |
+
tok = Tokenizer(models.BPE(unk_token=None))
|
| 145 |
+
tok.pre_tokenizer = pre_tokenizers.ByteLevel(add_prefix_space=False,
|
| 146 |
+
use_regex=True)
|
| 147 |
+
tok.decoder = decoders.ByteLevel()
|
| 148 |
+
trainer = trainers.BpeTrainer(
|
| 149 |
+
vocab_size=int(vocab_size),
|
| 150 |
+
min_frequency=int(min_frequency),
|
| 151 |
+
initial_alphabet=pre_tokenizers.ByteLevel.alphabet(),
|
| 152 |
+
special_tokens=[],
|
| 153 |
+
show_progress=False,
|
| 154 |
+
)
|
| 155 |
+
tok.train_from_iterator(list(texts), trainer=trainer)
|
| 156 |
+
if special_tokens:
|
| 157 |
+
tok.add_special_tokens(list(special_tokens))
|
| 158 |
+
out_path = Path(out_path)
|
| 159 |
+
out_path.parent.mkdir(parents=True, exist_ok=True)
|
| 160 |
+
tok.save(str(out_path))
|
| 161 |
+
|
| 162 |
+
|
| 163 |
+
def stories_from_bytes(path: str | Path, limit: int | None = None) -> List[bytes]:
|
| 164 |
+
"""Split a byte-level corpus (``0xFF`` separated) into story blobs."""
|
| 165 |
+
raw = Path(path).read_bytes()
|
| 166 |
+
parts = raw.split(bytes([EOT_BYTE]))
|
| 167 |
+
if limit:
|
| 168 |
+
parts = parts[:limit]
|
| 169 |
+
return [p for p in parts if p.strip()]
|
| 170 |
+
|
| 171 |
+
|
| 172 |
+
def encode_corpus(stories: Sequence[bytes], tok: BPETokenizer,
|
| 173 |
+
out_path: str | Path, eot_id: int,
|
| 174 |
+
chunk: int = 2000) -> dict:
|
| 175 |
+
"""Encode stories into a ``uint16`` memmap; returns byte/token stats.
|
| 176 |
+
|
| 177 |
+
Works in chunks and keeps the token ids as ``uint16`` numpy arrays, so peak
|
| 178 |
+
memory stays around ``2 * n_tokens`` bytes (~120 MB for TinyStories-200K).
|
| 179 |
+
"""
|
| 180 |
+
n_bytes = 0
|
| 181 |
+
pieces: List[np.ndarray] = []
|
| 182 |
+
for start in range(0, len(stories), chunk):
|
| 183 |
+
batch = stories[start:start + chunk]
|
| 184 |
+
n_bytes += sum(len(s) for s in batch)
|
| 185 |
+
texts = [s.decode("utf-8", errors="replace") for s in batch]
|
| 186 |
+
for seq in tok.encode_batch(texts):
|
| 187 |
+
arr = np.empty(len(seq) + 1, dtype=np.uint16)
|
| 188 |
+
arr[:-1] = np.asarray(seq, dtype=np.uint16)
|
| 189 |
+
arr[-1] = eot_id
|
| 190 |
+
pieces.append(arr)
|
| 191 |
+
flat = np.concatenate(pieces) if pieces else np.zeros(0, np.uint16)
|
| 192 |
+
out_path = Path(out_path)
|
| 193 |
+
out_path.parent.mkdir(parents=True, exist_ok=True)
|
| 194 |
+
flat.tofile(out_path)
|
| 195 |
+
n_tokens = len(flat)
|
| 196 |
+
return {"bytes": int(n_bytes), "tokens": int(n_tokens),
|
| 197 |
+
"bytes_per_token": float(n_bytes) / max(1, n_tokens)}
|
kronllm/train.py
ADDED
|
@@ -0,0 +1,571 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""Training / evaluation pipeline for KronLLM."""
|
| 2 |
+
from __future__ import annotations
|
| 3 |
+
|
| 4 |
+
import csv
|
| 5 |
+
import datetime
|
| 6 |
+
import json
|
| 7 |
+
import math
|
| 8 |
+
import os
|
| 9 |
+
import random
|
| 10 |
+
import time
|
| 11 |
+
from dataclasses import dataclass, field
|
| 12 |
+
from pathlib import Path
|
| 13 |
+
from typing import Any, Dict, List, Mapping, Optional, Tuple
|
| 14 |
+
|
| 15 |
+
import numpy as np
|
| 16 |
+
import torch
|
| 17 |
+
import torch.distributed as dist
|
| 18 |
+
import torch.nn as nn
|
| 19 |
+
from torch.nn.parallel import DistributedDataParallel as DDP
|
| 20 |
+
|
| 21 |
+
from .config import save_config
|
| 22 |
+
from .data import ByteDataset, open_dataset
|
| 23 |
+
from .tokenizer import bits_per_byte, load_tokenizer
|
| 24 |
+
from .diagnostics import (aggregate_eval_stats, comparison_report, grad_report,
|
| 25 |
+
routed_report, sigma_theta_report, spectral_report,
|
| 26 |
+
theta_grad_norm)
|
| 27 |
+
|
| 28 |
+
|
| 29 |
+
# ---------------------------------------------------------------------------
|
| 30 |
+
# utils
|
| 31 |
+
# ---------------------------------------------------------------------------
|
| 32 |
+
def set_seed(seed: int, deterministic: bool = True) -> None:
|
| 33 |
+
random.seed(seed)
|
| 34 |
+
np.random.seed(seed)
|
| 35 |
+
torch.manual_seed(seed)
|
| 36 |
+
torch.cuda.manual_seed_all(seed)
|
| 37 |
+
if deterministic:
|
| 38 |
+
os.environ.setdefault("CUBLAS_WORKSPACE_CONFIG", ":4096:8")
|
| 39 |
+
torch.backends.cudnn.deterministic = True
|
| 40 |
+
torch.backends.cudnn.benchmark = False
|
| 41 |
+
try:
|
| 42 |
+
torch.use_deterministic_algorithms(True, warn_only=True)
|
| 43 |
+
except Exception:
|
| 44 |
+
pass
|
| 45 |
+
|
| 46 |
+
|
| 47 |
+
def cosine_factor(step: int, total: int, warmup: int, min_ratio: float) -> float:
|
| 48 |
+
if step < warmup:
|
| 49 |
+
return float(step + 1) / max(1, warmup)
|
| 50 |
+
if total <= warmup:
|
| 51 |
+
return 1.0
|
| 52 |
+
p = min(1.0, (step - warmup) / max(1, total - warmup))
|
| 53 |
+
return min_ratio + (1.0 - min_ratio) * 0.5 * (1.0 + math.cos(math.pi * p))
|
| 54 |
+
|
| 55 |
+
|
| 56 |
+
class CSVLogger:
|
| 57 |
+
"""Append-only CSV logger that tolerates a growing column set.
|
| 58 |
+
|
| 59 |
+
Diagnostics rows add fields that training rows do not have, so the header
|
| 60 |
+
is rewritten (from an in-memory copy) whenever the schema grows. Without
|
| 61 |
+
this, ``pandas.read_csv`` fails and the dashboard shows an empty table.
|
| 62 |
+
"""
|
| 63 |
+
|
| 64 |
+
def __init__(self, path: str | Path) -> None:
|
| 65 |
+
self.path = Path(path)
|
| 66 |
+
self.path.parent.mkdir(parents=True, exist_ok=True)
|
| 67 |
+
self._fields: List[str] = []
|
| 68 |
+
self._rows: List[Dict[str, Any]] = []
|
| 69 |
+
# A run can be resumed (or restarted in the same directory). Load the
|
| 70 |
+
# existing rows first: _rewrite() drops anything not in memory, so
|
| 71 |
+
# without this a restart silently truncated log.csv to the new rows.
|
| 72 |
+
if self.path.exists() and self.path.stat().st_size > 0:
|
| 73 |
+
with open(self.path, newline="", encoding="utf-8") as fh:
|
| 74 |
+
reader = csv.DictReader(fh)
|
| 75 |
+
self._fields = list(reader.fieldnames or [])
|
| 76 |
+
self._rows = [dict(r) for r in reader]
|
| 77 |
+
|
| 78 |
+
def log(self, row: Mapping[str, Any]) -> None:
|
| 79 |
+
new_keys = [k for k in row if k not in self._fields]
|
| 80 |
+
self._rows.append(dict(row))
|
| 81 |
+
if new_keys or not self.path.exists():
|
| 82 |
+
self._fields.extend(new_keys)
|
| 83 |
+
self._rewrite()
|
| 84 |
+
return
|
| 85 |
+
with open(self.path, "a", newline="", encoding="utf-8") as fh:
|
| 86 |
+
wr = csv.DictWriter(fh, fieldnames=self._fields,
|
| 87 |
+
extrasaction="ignore")
|
| 88 |
+
wr.writerow({k: row.get(k, "") for k in self._fields})
|
| 89 |
+
|
| 90 |
+
def _rewrite(self) -> None:
|
| 91 |
+
with open(self.path, "w", newline="", encoding="utf-8") as fh:
|
| 92 |
+
wr = csv.DictWriter(fh, fieldnames=self._fields,
|
| 93 |
+
extrasaction="ignore")
|
| 94 |
+
wr.writeheader()
|
| 95 |
+
for r in self._rows:
|
| 96 |
+
wr.writerow({k: r.get(k, "") for k in self._fields})
|
| 97 |
+
|
| 98 |
+
def close(self) -> None:
|
| 99 |
+
pass
|
| 100 |
+
|
| 101 |
+
|
| 102 |
+
# ---------------------------------------------------------------------------
|
| 103 |
+
# optimizer
|
| 104 |
+
# ---------------------------------------------------------------------------
|
| 105 |
+
def build_param_groups(model: nn.Module, tc: Mapping[str, Any]) -> List[dict]:
|
| 106 |
+
decay, no_decay, s_params, theta_params, scale_params = [], [], [], [], []
|
| 107 |
+
for name, p in model.named_parameters():
|
| 108 |
+
if not p.requires_grad:
|
| 109 |
+
continue
|
| 110 |
+
if name.endswith(".theta"):
|
| 111 |
+
theta_params.append(p)
|
| 112 |
+
elif name.endswith(".tau") or name.endswith(".gate"):
|
| 113 |
+
scale_params.append(p)
|
| 114 |
+
elif name.endswith(".s"):
|
| 115 |
+
s_params.append(p)
|
| 116 |
+
elif ("norm" in name or "embed" in name or "lm_head" in name
|
| 117 |
+
or "route" in name or "bank" in name or "pass_bias" in name):
|
| 118 |
+
no_decay.append(p)
|
| 119 |
+
else:
|
| 120 |
+
decay.append(p)
|
| 121 |
+
|
| 122 |
+
groups = [
|
| 123 |
+
{"params": decay, "lr": float(tc["lr"]),
|
| 124 |
+
"weight_decay": float(tc["weight_decay"]), "name": "decay"},
|
| 125 |
+
{"params": no_decay, "lr": float(tc["lr"]),
|
| 126 |
+
"weight_decay": 0.0, "name": "no_decay"},
|
| 127 |
+
{"params": s_params, "lr": float(tc["lr_s"]),
|
| 128 |
+
"weight_decay": 0.0, "name": "anchor_s"},
|
| 129 |
+
{"params": theta_params, "lr": float(tc["lr_theta"]),
|
| 130 |
+
"weight_decay": 0.0, "name": "theta"},
|
| 131 |
+
{"params": scale_params, "lr": float(tc["lr"]),
|
| 132 |
+
"weight_decay": 0.0, "name": "scales"},
|
| 133 |
+
]
|
| 134 |
+
return [g for g in groups if g["params"]]
|
| 135 |
+
|
| 136 |
+
|
| 137 |
+
def build_optimizer(model: nn.Module, tc: Mapping[str, Any]):
|
| 138 |
+
groups = build_param_groups(model, tc)
|
| 139 |
+
betas = tuple(tc["betas"])
|
| 140 |
+
if str(tc["optimizer"]).lower() == "adamw":
|
| 141 |
+
return torch.optim.AdamW(groups, betas=betas, eps=1e-8)
|
| 142 |
+
raise ValueError(f"unsupported optimizer {tc['optimizer']}")
|
| 143 |
+
|
| 144 |
+
|
| 145 |
+
def init_base_lrs(opt: torch.optim.Optimizer) -> None:
|
| 146 |
+
for g in opt.param_groups:
|
| 147 |
+
g["base_lr"] = float(g["lr"])
|
| 148 |
+
|
| 149 |
+
|
| 150 |
+
def set_lr(opt: torch.optim.Optimizer, factor: float) -> float:
|
| 151 |
+
lr0 = 0.0
|
| 152 |
+
for g in opt.param_groups:
|
| 153 |
+
g["lr"] = float(g["base_lr"]) * factor
|
| 154 |
+
lr0 = g["lr"]
|
| 155 |
+
return lr0
|
| 156 |
+
|
| 157 |
+
|
| 158 |
+
def named_lrs(opt: torch.optim.Optimizer) -> Dict[str, float]:
|
| 159 |
+
return {g.get("name", "group"): g["lr"] for g in opt.param_groups}
|
| 160 |
+
|
| 161 |
+
|
| 162 |
+
# ---------------------------------------------------------------------------
|
| 163 |
+
# evaluation
|
| 164 |
+
# ---------------------------------------------------------------------------
|
| 165 |
+
@torch.no_grad()
|
| 166 |
+
def evaluate(model: nn.Module, ds: ByteDataset, batch_size: int,
|
| 167 |
+
device: torch.device, max_batches: Optional[int] = None,
|
| 168 |
+
amp: str = "fp16") -> Dict[str, float]:
|
| 169 |
+
model.eval()
|
| 170 |
+
total_loss, total_tokens = 0.0, 0
|
| 171 |
+
use_amp = amp == "fp16" and device.type == "cuda"
|
| 172 |
+
for i, (x, y) in enumerate(ds.iter_blocks(batch_size, device)):
|
| 173 |
+
if max_batches is not None and i >= max_batches:
|
| 174 |
+
break
|
| 175 |
+
with torch.amp.autocast("cuda", dtype=torch.float16, enabled=use_amp):
|
| 176 |
+
out = model(x, y)
|
| 177 |
+
total_loss += float(out["loss"].detach()) * x.numel()
|
| 178 |
+
total_tokens += x.numel()
|
| 179 |
+
model.train()
|
| 180 |
+
if total_tokens == 0:
|
| 181 |
+
return {"val_loss": float("nan"), "val_ppl": float("nan")}
|
| 182 |
+
loss = total_loss / total_tokens
|
| 183 |
+
result = {"val_loss": loss, "val_ppl": math.exp(min(loss, 50.0))}
|
| 184 |
+
result.update(routed_report(model))
|
| 185 |
+
return result
|
| 186 |
+
|
| 187 |
+
|
| 188 |
+
@torch.no_grad()
|
| 189 |
+
def diagnostics(model: nn.Module, ds: ByteDataset, batch_size: int,
|
| 190 |
+
device: torch.device, spectral: bool = True,
|
| 191 |
+
amp: str = "fp16") -> Dict[str, Any]:
|
| 192 |
+
model.eval()
|
| 193 |
+
x, y = next(iter(ds.iter_blocks(batch_size, device)))
|
| 194 |
+
use_amp = amp == "fp16" and device.type == "cuda"
|
| 195 |
+
with torch.amp.autocast("cuda", dtype=torch.float16, enabled=use_amp):
|
| 196 |
+
out = model(x, y, collect=True)
|
| 197 |
+
result: Dict[str, Any] = {"batch_loss": float(out["loss"].detach())}
|
| 198 |
+
result.update(aggregate_eval_stats(out["stats"]))
|
| 199 |
+
result.update({k: v for k, v in sigma_theta_report(model).items()
|
| 200 |
+
if not isinstance(v, dict)})
|
| 201 |
+
result.update(routed_report(model))
|
| 202 |
+
result["theta_grad_norm"] = theta_grad_norm(model)
|
| 203 |
+
if spectral:
|
| 204 |
+
spec = spectral_report(model, out["trace"])
|
| 205 |
+
result["spectral"] = spec["summary"]
|
| 206 |
+
result.update({f"spec_{k}": v for k, v in spec["summary"].items()
|
| 207 |
+
if isinstance(v, (int, float))})
|
| 208 |
+
model.last_trace = None
|
| 209 |
+
model.train()
|
| 210 |
+
return result
|
| 211 |
+
|
| 212 |
+
|
| 213 |
+
# ---------------------------------------------------------------------------
|
| 214 |
+
# train
|
| 215 |
+
# ---------------------------------------------------------------------------
|
| 216 |
+
def _init_process_group(rank: int, world_size: int) -> None:
|
| 217 |
+
"""Initialise DDP, with a Windows fallback for the rendezvous store.
|
| 218 |
+
|
| 219 |
+
Some Windows wheels are built without libuv, so the default TCP store
|
| 220 |
+
aborts with "use_libuv was requested but PyTorch was build without libuv
|
| 221 |
+
support". Retrying with an explicitly constructed store fixes it; on Linux
|
| 222 |
+
(Kaggle) the default path is used unchanged.
|
| 223 |
+
"""
|
| 224 |
+
backend = "nccl" if torch.cuda.is_available() else "gloo"
|
| 225 |
+
try:
|
| 226 |
+
dist.init_process_group(backend=backend, rank=rank,
|
| 227 |
+
world_size=world_size)
|
| 228 |
+
return
|
| 229 |
+
except Exception: # noqa: BLE001
|
| 230 |
+
if dist.is_initialized():
|
| 231 |
+
dist.destroy_process_group()
|
| 232 |
+
addr = os.environ.get("MASTER_ADDR", "127.0.0.1")
|
| 233 |
+
port = int(os.environ.get("MASTER_PORT", "29500"))
|
| 234 |
+
store = dist.TCPStore(addr, port, world_size, rank == 0,
|
| 235 |
+
timeout=datetime.timedelta(minutes=30),
|
| 236 |
+
use_libuv=False)
|
| 237 |
+
dist.init_process_group(backend=backend, store=store, rank=rank,
|
| 238 |
+
world_size=world_size)
|
| 239 |
+
|
| 240 |
+
|
| 241 |
+
def train(cfg: Mapping[str, Any], run_dir: str | Path,
|
| 242 |
+
resume: Optional[str] = None) -> Dict[str, Any]:
|
| 243 |
+
tc = cfg["training"]
|
| 244 |
+
dc = cfg["data"]
|
| 245 |
+
seed = int(cfg.get("seed", 42))
|
| 246 |
+
set_seed(seed, deterministic=bool(tc.get("deterministic", True)))
|
| 247 |
+
|
| 248 |
+
# -- distributed ----------------------------------------------------
|
| 249 |
+
# `torchrun --nproc_per_node=N` sets WORLD_SIZE/RANK/LOCAL_RANK. Without
|
| 250 |
+
# it the run is the single-process path exactly as before.
|
| 251 |
+
world_size = int(os.environ.get("WORLD_SIZE", "1"))
|
| 252 |
+
ddp = world_size > 1
|
| 253 |
+
rank = int(os.environ.get("RANK", "0"))
|
| 254 |
+
local_rank = int(os.environ.get("LOCAL_RANK", "0"))
|
| 255 |
+
is_main = rank == 0
|
| 256 |
+
if ddp:
|
| 257 |
+
if not dist.is_initialized():
|
| 258 |
+
_init_process_group(rank, world_size)
|
| 259 |
+
if torch.cuda.is_available():
|
| 260 |
+
torch.cuda.set_device(local_rank)
|
| 261 |
+
|
| 262 |
+
run_dir = Path(run_dir)
|
| 263 |
+
if is_main:
|
| 264 |
+
run_dir.mkdir(parents=True, exist_ok=True)
|
| 265 |
+
save_config(cfg, run_dir / "config.yaml")
|
| 266 |
+
|
| 267 |
+
if torch.cuda.is_available():
|
| 268 |
+
device = torch.device(f"cuda:{local_rank}" if ddp
|
| 269 |
+
else tc.get("device", "cuda"))
|
| 270 |
+
else:
|
| 271 |
+
device = torch.device("cpu")
|
| 272 |
+
data_dir = Path(dc.get("data_dir", "data"))
|
| 273 |
+
train_ds = open_dataset(cfg, "train")
|
| 274 |
+
val_ds = open_dataset(cfg, "val")
|
| 275 |
+
tok = load_tokenizer(cfg)
|
| 276 |
+
bpt = tok.bytes_per_token
|
| 277 |
+
|
| 278 |
+
# -- model ---------------------------------------------------------
|
| 279 |
+
from .model import KronTransformer
|
| 280 |
+
model = KronTransformer(cfg)
|
| 281 |
+
init_from = tc.get("init_from")
|
| 282 |
+
if init_from:
|
| 283 |
+
source = Path(init_from)
|
| 284 |
+
if not source.is_absolute():
|
| 285 |
+
source = Path.cwd() / source
|
| 286 |
+
if not source.exists():
|
| 287 |
+
raise FileNotFoundError(f"training.init_from not found: {source}")
|
| 288 |
+
payload = torch.load(source, map_location="cpu", weights_only=False)
|
| 289 |
+
state = payload.get("model", payload) if isinstance(payload, dict) else payload
|
| 290 |
+
model.load_state_dict(state)
|
| 291 |
+
print(f"[init] weights <- {source} (step {payload.get('step', 'n/a') if isinstance(payload, dict) else 'n/a'})")
|
| 292 |
+
model = model.to(device)
|
| 293 |
+
raw_model = model # unwrapped: eval, diagnostics, save
|
| 294 |
+
if tc.get("compile") and hasattr(torch, "compile"):
|
| 295 |
+
model = torch.compile(model)
|
| 296 |
+
if ddp:
|
| 297 |
+
# theta is inert when sigma_mode != "learned", so the flag is needed
|
| 298 |
+
# there; the default configuration has no unused parameters.
|
| 299 |
+
kc = cfg["model"]["kron"]
|
| 300 |
+
unused = str(kc.get("sigma_mode", "learned")) != "learned"
|
| 301 |
+
model = DDP(model,
|
| 302 |
+
device_ids=[local_rank] if device.type == "cuda" else None,
|
| 303 |
+
find_unused_parameters=unused)
|
| 304 |
+
|
| 305 |
+
opt = build_optimizer(model, tc)
|
| 306 |
+
init_base_lrs(opt)
|
| 307 |
+
use_amp = str(tc["amp"]) == "fp16" and device.type == "cuda" and tc.get("amp") != "none"
|
| 308 |
+
scaler = torch.amp.GradScaler("cuda", enabled=use_amp)
|
| 309 |
+
|
| 310 |
+
bs = int(tc["batch_size"])
|
| 311 |
+
accum = int(tc["grad_accum"])
|
| 312 |
+
steps_per_epoch = max(1, len(train_ds) // (bs * accum))
|
| 313 |
+
total_steps = int(tc.get("max_steps") or steps_per_epoch * int(tc["epochs"]))
|
| 314 |
+
warmup = int(tc["warmup_steps"])
|
| 315 |
+
min_ratio = float(tc["min_lr"]) / float(tc["lr"])
|
| 316 |
+
eval_every = int(tc["eval_every"])
|
| 317 |
+
log_every = int(tc["log_every"])
|
| 318 |
+
save_every = int(tc.get("save_every", 0) or 0)
|
| 319 |
+
spectral_every = int(cfg["metrics"].get("spectral_every", 2000))
|
| 320 |
+
eval_batches = int(tc.get("eval_batches", 64))
|
| 321 |
+
patience = int(tc.get("early_stop_patience", 4))
|
| 322 |
+
random_recurse = (bool(tc.get("recurse_random"))
|
| 323 |
+
and getattr(model, "max_recurse", 1) > 1)
|
| 324 |
+
|
| 325 |
+
logger = CSVLogger(run_dir / "log.csv") if is_main else None
|
| 326 |
+
tb = None
|
| 327 |
+
if cfg.get("logging", {}).get("tensorboard"):
|
| 328 |
+
try:
|
| 329 |
+
from torch.utils.tensorboard import SummaryWriter
|
| 330 |
+
tb = SummaryWriter(run_dir / "tb")
|
| 331 |
+
except Exception as exc: # broken tensorflow/tensorboard env
|
| 332 |
+
print(f"[warn] tensorboard disabled: {exc}")
|
| 333 |
+
wb = None
|
| 334 |
+
if cfg.get("logging", {}).get("wandb"):
|
| 335 |
+
try:
|
| 336 |
+
import wandb
|
| 337 |
+
wb = wandb.init(project=cfg["logging"].get("wandb_project", "kronllm"),
|
| 338 |
+
name=f"{cfg.get('experiment', 'run')}",
|
| 339 |
+
config=json.loads(json.dumps(cfg, default=str)),
|
| 340 |
+
dir=str(run_dir))
|
| 341 |
+
except Exception as exc:
|
| 342 |
+
print(f"[warn] wandb disabled: {exc}")
|
| 343 |
+
|
| 344 |
+
eff_bs = int(tc["batch_size"]) * int(tc["grad_accum"])
|
| 345 |
+
comp = comparison_report(raw_model, int(dc["seq_len"]), batch_size=eff_bs)
|
| 346 |
+
if is_main:
|
| 347 |
+
print(f"[model] {raw_model.n_params:,} params "
|
| 348 |
+
f"({raw_model.n_params_matrix:,} in matrix layers), "
|
| 349 |
+
f"{comp['flops_per_token']/1e6:.1f} MFLOP/token "
|
| 350 |
+
f"@ batch {eff_bs}"
|
| 351 |
+
+ (f" | ddp world_size={world_size}" if ddp else ""))
|
| 352 |
+
|
| 353 |
+
gen = torch.Generator(device="cpu").manual_seed(seed + 1000 * rank)
|
| 354 |
+
start_step = 0
|
| 355 |
+
best_ppl = float("inf")
|
| 356 |
+
bad_evals = 0
|
| 357 |
+
history: List[Dict[str, Any]] = []
|
| 358 |
+
|
| 359 |
+
if resume:
|
| 360 |
+
ck = torch.load(resume, map_location="cpu")
|
| 361 |
+
raw_model.load_state_dict(ck["model"])
|
| 362 |
+
opt.load_state_dict(ck["optimizer"])
|
| 363 |
+
start_step = int(ck.get("step", 0))
|
| 364 |
+
best_ppl = float(ck.get("val_ppl", float("inf")))
|
| 365 |
+
if is_main:
|
| 366 |
+
print(f"[resume] step {start_step}, best ppl {best_ppl:.3f}")
|
| 367 |
+
|
| 368 |
+
if device.type == "cuda":
|
| 369 |
+
torch.cuda.reset_peak_memory_stats(device)
|
| 370 |
+
|
| 371 |
+
model.train()
|
| 372 |
+
t_start = time.time()
|
| 373 |
+
step = start_step
|
| 374 |
+
micro = 0
|
| 375 |
+
running = 0.0
|
| 376 |
+
running_n = 0
|
| 377 |
+
last_log = time.time()
|
| 378 |
+
stop = False
|
| 379 |
+
|
| 380 |
+
while step < total_steps and not stop:
|
| 381 |
+
for _ in range(accum):
|
| 382 |
+
if random_recurse:
|
| 383 |
+
raw_model.set_recurse(random.randint(1, raw_model.max_recurse))
|
| 384 |
+
x, y = train_ds.get_batch(bs, device, gen)
|
| 385 |
+
with torch.amp.autocast("cuda", dtype=torch.float16, enabled=use_amp):
|
| 386 |
+
out = model(x, y)
|
| 387 |
+
loss = out["loss"] / accum
|
| 388 |
+
if not torch.isfinite(loss):
|
| 389 |
+
raise FloatingPointError(f"non-finite loss at step {step}")
|
| 390 |
+
scaler.scale(loss).backward()
|
| 391 |
+
running += float(loss.detach()) * accum
|
| 392 |
+
running_n += 1
|
| 393 |
+
micro += 1
|
| 394 |
+
|
| 395 |
+
factor = cosine_factor(step, total_steps, warmup, min_ratio)
|
| 396 |
+
set_lr(opt, factor)
|
| 397 |
+
scaler.unscale_(opt)
|
| 398 |
+
grads = grad_report(model)
|
| 399 |
+
gnorm = float(torch.nn.utils.clip_grad_norm_(
|
| 400 |
+
model.parameters(), float(tc["grad_clip"])))
|
| 401 |
+
scaler.step(opt)
|
| 402 |
+
scaler.update()
|
| 403 |
+
opt.zero_grad(set_to_none=True)
|
| 404 |
+
step += 1
|
| 405 |
+
|
| 406 |
+
if save_every and step % save_every == 0 and is_main:
|
| 407 |
+
# Crash / quota resilience: a resumed run continues from here.
|
| 408 |
+
torch.save({"model": raw_model.state_dict(), "cfg": dict(cfg),
|
| 409 |
+
"optimizer": opt.state_dict(), "step": step,
|
| 410 |
+
"val_ppl": best_ppl}, run_dir / "last.pt")
|
| 411 |
+
|
| 412 |
+
if step % log_every == 0 or step == 1:
|
| 413 |
+
dt = time.time() - last_log
|
| 414 |
+
last_log = time.time()
|
| 415 |
+
train_loss = running / max(1, running_n)
|
| 416 |
+
named = named_lrs(opt)
|
| 417 |
+
row = {
|
| 418 |
+
"step": step,
|
| 419 |
+
"split": "train",
|
| 420 |
+
"loss": train_loss,
|
| 421 |
+
"ppl": math.exp(min(train_loss, 50.0)),
|
| 422 |
+
"bpb": bits_per_byte(train_loss, bpt),
|
| 423 |
+
"grad_norm": gnorm,
|
| 424 |
+
"lr": named.get("decay", float(tc["lr"])),
|
| 425 |
+
"lr_s": named.get("anchor_s", float(tc["lr_s"])),
|
| 426 |
+
"lr_theta": named.get("theta", float(tc["lr_theta"])),
|
| 427 |
+
"sec_per_step": dt / max(1, log_every),
|
| 428 |
+
}
|
| 429 |
+
row.update({k: v for k, v in grads.items()})
|
| 430 |
+
if is_main:
|
| 431 |
+
logger.log(row)
|
| 432 |
+
if tb:
|
| 433 |
+
for k, v in row.items():
|
| 434 |
+
if isinstance(v, (int, float)):
|
| 435 |
+
tb.add_scalar(f"train/{k}", v, step)
|
| 436 |
+
if wb:
|
| 437 |
+
wb.log({f"train/{k}": v for k, v in row.items()
|
| 438 |
+
if isinstance(v, (int, float))}, step=step)
|
| 439 |
+
print(f"step {step:5d} | loss {train_loss:.4f} | "
|
| 440 |
+
f"ppl {math.exp(min(train_loss,50)):.2f} | "
|
| 441 |
+
f"gnorm {gnorm:.3f} | {dt/max(1,log_every)*1000:.0f} ms/step")
|
| 442 |
+
running, running_n = 0.0, 0
|
| 443 |
+
|
| 444 |
+
if step % eval_every == 0 or step == total_steps:
|
| 445 |
+
if random_recurse:
|
| 446 |
+
raw_model.set_recurse(raw_model.max_recurse)
|
| 447 |
+
# Every rank evaluates the same fixed validation batches, so the
|
| 448 |
+
# numbers (and therefore early-stop decisions) agree without a
|
| 449 |
+
# broadcast. Only rank 0 logs and writes checkpoints.
|
| 450 |
+
ev = evaluate(raw_model, val_ds, bs, device, eval_batches, str(tc["amp"]))
|
| 451 |
+
ev["val_bpb"] = bits_per_byte(ev["val_loss"], bpt)
|
| 452 |
+
if is_main:
|
| 453 |
+
row = {"step": step, "split": "val", **ev}
|
| 454 |
+
logger.log(row)
|
| 455 |
+
if tb:
|
| 456 |
+
tb.add_scalar("val/loss", ev["val_loss"], step)
|
| 457 |
+
tb.add_scalar("val/ppl", ev["val_ppl"], step)
|
| 458 |
+
tb.add_scalar("val/bpb", ev["val_bpb"], step)
|
| 459 |
+
if wb:
|
| 460 |
+
wb.log({"val/loss": ev["val_loss"], "val/ppl": ev["val_ppl"],
|
| 461 |
+
"val/bpb": ev["val_bpb"]}, step=step)
|
| 462 |
+
print(f" [val] step {step:5d} | loss {ev['val_loss']:.4f} | "
|
| 463 |
+
f"ppl {ev['val_ppl']:.2f} | bpb {ev['val_bpb']:.3f}")
|
| 464 |
+
|
| 465 |
+
if ev["val_ppl"] < best_ppl:
|
| 466 |
+
best_ppl = ev["val_ppl"]
|
| 467 |
+
bad_evals = 0
|
| 468 |
+
if is_main:
|
| 469 |
+
torch.save({"model": raw_model.state_dict(), "cfg": dict(cfg),
|
| 470 |
+
"step": step, "val_ppl": best_ppl},
|
| 471 |
+
run_dir / "best.pt")
|
| 472 |
+
else:
|
| 473 |
+
bad_evals += 1
|
| 474 |
+
if bad_evals >= patience:
|
| 475 |
+
if is_main:
|
| 476 |
+
print(f"[early-stop] no improvement for {bad_evals} evals")
|
| 477 |
+
stop = True
|
| 478 |
+
|
| 479 |
+
if spectral_every and (step % spectral_every == 0 or step == total_steps) and is_main:
|
| 480 |
+
dg = diagnostics(raw_model, val_ds, bs, device, spectral=True,
|
| 481 |
+
amp=str(tc["amp"]))
|
| 482 |
+
logger.log({"step": step, "split": "diag", **{
|
| 483 |
+
k: v for k, v in dg.items() if isinstance(v, (int, float))}})
|
| 484 |
+
if tb:
|
| 485 |
+
for k, v in dg.items():
|
| 486 |
+
if isinstance(v, (int, float)):
|
| 487 |
+
tb.add_scalar(f"diag/{k}", v, step)
|
| 488 |
+
if wb:
|
| 489 |
+
wb.log({f"diag/{k}": v for k, v in dg.items()
|
| 490 |
+
if isinstance(v, (int, float))}, step=step)
|
| 491 |
+
history.append({"step": step, **dg})
|
| 492 |
+
|
| 493 |
+
if random_recurse:
|
| 494 |
+
raw_model.set_recurse(raw_model.max_recurse)
|
| 495 |
+
if is_main:
|
| 496 |
+
torch.save({"model": raw_model.state_dict(), "cfg": dict(cfg),
|
| 497 |
+
"optimizer": opt.state_dict(),
|
| 498 |
+
"step": step, "val_ppl": best_ppl}, run_dir / "last.pt")
|
| 499 |
+
|
| 500 |
+
elapsed = time.time() - t_start
|
| 501 |
+
peak_vram = 0.0
|
| 502 |
+
if device.type == "cuda":
|
| 503 |
+
peak_vram = torch.cuda.max_memory_allocated(device) / 1e9
|
| 504 |
+
|
| 505 |
+
final = evaluate(raw_model, val_ds, bs, device, eval_batches, str(tc["amp"]))
|
| 506 |
+
dg = diagnostics(raw_model, val_ds, bs, device, spectral=True,
|
| 507 |
+
amp=str(tc["amp"])) if is_main else {}
|
| 508 |
+
|
| 509 |
+
results = {
|
| 510 |
+
"experiment": cfg.get("experiment", "run"),
|
| 511 |
+
"desc": cfg.get("experiment_desc", ""),
|
| 512 |
+
**comp,
|
| 513 |
+
**final,
|
| 514 |
+
"val_bpb": bits_per_byte(final["val_loss"], bpt),
|
| 515 |
+
"bytes_per_token": bpt,
|
| 516 |
+
"vocab_size": tok.vocab_size,
|
| 517 |
+
"tokenizer": tok.name,
|
| 518 |
+
"best_val_ppl": best_ppl,
|
| 519 |
+
"steps": step,
|
| 520 |
+
"time_total_sec": elapsed,
|
| 521 |
+
"time_per_epoch_sec": elapsed / max(1.0, step / max(1, steps_per_epoch)),
|
| 522 |
+
"sec_per_step": elapsed / max(1, step),
|
| 523 |
+
"peak_vram_gb": peak_vram,
|
| 524 |
+
"sigma_theta_mean": dg.get("sigma_theta_mean"),
|
| 525 |
+
"sigma_theta_var": dg.get("sigma_theta_var"),
|
| 526 |
+
"attn_entropy_mean": dg.get("attn_entropy_mean"),
|
| 527 |
+
"attn_entropy_min": dg.get("attn_entropy_min"),
|
| 528 |
+
"q_sigma_min": dg.get("q_sigma_min"),
|
| 529 |
+
"k_sigma_min": dg.get("k_sigma_min"),
|
| 530 |
+
"q_drift_mean": dg.get("q_drift_mean"),
|
| 531 |
+
"h_norm_ratio_mean": dg.get("h_norm_ratio_mean"),
|
| 532 |
+
"h_norm_ratio_min": dg.get("h_norm_ratio_min"),
|
| 533 |
+
"h_norm_ratio_max": dg.get("h_norm_ratio_max"),
|
| 534 |
+
"eff_rank_mean": dg.get("spec_eff_rank_mean"),
|
| 535 |
+
"eff_rank_W": dg.get("spec_eff_rank_W"),
|
| 536 |
+
"eff_rank_D": dg.get("spec_eff_rank_D"),
|
| 537 |
+
"rank_excess": dg.get("spec_rank_excess"),
|
| 538 |
+
"eff_rank_P": dg.get("spec_eff_rank_P"),
|
| 539 |
+
"eff_rank_excess": dg.get("spec_eff_rank_excess"),
|
| 540 |
+
"eff_rank_spearman": dg.get("spec_eff_rank_spearman"),
|
| 541 |
+
"eff_rank_excess_spearman": dg.get("spec_eff_rank_excess_spearman"),
|
| 542 |
+
"eff_rank_growth": dg.get("spec_eff_rank_growth"),
|
| 543 |
+
"D_rms": dg.get("spec_D_rms"),
|
| 544 |
+
"tau_mean": dg.get("spec_tau_mean"),
|
| 545 |
+
"gate_mean": dg.get("spec_gate_mean"),
|
| 546 |
+
"sigma_min_W": dg.get("spec_sigma_min_W"),
|
| 547 |
+
"cond_max": dg.get("spec_cond_max"),
|
| 548 |
+
"cos_prev_max": dg.get("spec_cos_prev_max"),
|
| 549 |
+
"cos_prev_P_max": dg.get("spec_cos_prev_P_max"),
|
| 550 |
+
"cos_prev_P_mean": dg.get("spec_cos_prev_P_mean"),
|
| 551 |
+
"theta_grad_norm": dg.get("theta_grad_norm"),
|
| 552 |
+
"config": json.loads(json.dumps(cfg, default=str)),
|
| 553 |
+
}
|
| 554 |
+
if is_main:
|
| 555 |
+
with open(run_dir / "results.json", "w", encoding="utf-8") as fh:
|
| 556 |
+
json.dump(results, fh, indent=2)
|
| 557 |
+
with open(run_dir / "diagnostics.json", "w", encoding="utf-8") as fh:
|
| 558 |
+
json.dump({"final": dg, "history": history}, fh, indent=2)
|
| 559 |
+
|
| 560 |
+
logger.close()
|
| 561 |
+
if tb:
|
| 562 |
+
tb.close()
|
| 563 |
+
if wb:
|
| 564 |
+
wb.finish()
|
| 565 |
+
|
| 566 |
+
print(f"[done] best val ppl {best_ppl:.3f} | {elapsed/60:.1f} min | "
|
| 567 |
+
f"peak VRAM {peak_vram:.2f} GB")
|
| 568 |
+
if ddp:
|
| 569 |
+
dist.barrier()
|
| 570 |
+
dist.destroy_process_group()
|
| 571 |
+
return results
|
pyproject.toml
ADDED
|
@@ -0,0 +1,57 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
[build-system]
|
| 2 |
+
requires = ["setuptools>=68", "wheel"]
|
| 3 |
+
build-backend = "setuptools.build_meta"
|
| 4 |
+
|
| 5 |
+
[project]
|
| 6 |
+
name = "kronllm"
|
| 7 |
+
version = "2.0.0"
|
| 8 |
+
description = "A compact, fully Kronecker-parameterised language model"
|
| 9 |
+
readme = "README.md"
|
| 10 |
+
requires-python = ">=3.10"
|
| 11 |
+
license = { text = "MIT" }
|
| 12 |
+
authors = [{ name = "Ivan Kholodilo" }]
|
| 13 |
+
keywords = ["language-model", "kronecker", "parameter-efficiency", "small-language-model", "pytorch"]
|
| 14 |
+
classifiers = [
|
| 15 |
+
"MIT License",
|
| 16 |
+
"Programming Language :: Python :: 3",
|
| 17 |
+
"Programming Language :: Python :: 3.11",
|
| 18 |
+
"Topic :: Scientific/Engineering :: Artificial Intelligence",
|
| 19 |
+
]
|
| 20 |
+
dependencies = [
|
| 21 |
+
"torch>=2.4",
|
| 22 |
+
"numpy>=1.24",
|
| 23 |
+
"pyyaml>=6.0",
|
| 24 |
+
"datasets>=2.18",
|
| 25 |
+
"pandas>=2.0",
|
| 26 |
+
"matplotlib>=3.7",
|
| 27 |
+
]
|
| 28 |
+
|
| 29 |
+
[project.optional-dependencies]
|
| 30 |
+
dashboard = ["gradio>=4.0"]
|
| 31 |
+
hf = ["transformers>=4.40"]
|
| 32 |
+
dev = ["pytest>=7.4", "ruff>=0.5", "jupyter>=1.0"]
|
| 33 |
+
|
| 34 |
+
[project.urls]
|
| 35 |
+
Homepage = "https://github.com/Icold21/kron-llm"
|
| 36 |
+
Repository = "https://github.com/Icold21/kron-llm"
|
| 37 |
+
Issues = "https://github.com/Icold21/kron-llm/issues"
|
| 38 |
+
|
| 39 |
+
[project.scripts]
|
| 40 |
+
kronllm = "kronllm.cli:main"
|
| 41 |
+
|
| 42 |
+
[tool.setuptools.packages.find]
|
| 43 |
+
include = ["kronllm*"]
|
| 44 |
+
|
| 45 |
+
[tool.pytest.ini_options]
|
| 46 |
+
testpaths = ["tests"]
|
| 47 |
+
addopts = "-q --strict-markers"
|
| 48 |
+
markers = ["slow: long-running integration tests"]
|
| 49 |
+
filterwarnings = ["ignore::DeprecationWarning"]
|
| 50 |
+
|
| 51 |
+
[tool.ruff]
|
| 52 |
+
line-length = 100
|
| 53 |
+
target-version = "py310"
|
| 54 |
+
|
| 55 |
+
[tool.ruff.lint]
|
| 56 |
+
select = ["E", "F", "I", "W", "UP", "B"]
|
| 57 |
+
ignore = ["E501"]
|
requirements.txt
ADDED
|
@@ -0,0 +1,10 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
torch>=2.4
|
| 2 |
+
numpy>=1.24
|
| 3 |
+
pyyaml>=6.0
|
| 4 |
+
datasets>=2.18
|
| 5 |
+
pandas>=2.0
|
| 6 |
+
matplotlib>=3.7
|
| 7 |
+
gradio>=4.0
|
| 8 |
+
tensorboard>=2.15
|
| 9 |
+
# optional
|
| 10 |
+
# wandb>=0.16
|